def do_index(doc_type, search_api_url, search_api_access_token, data_api_url, data_api_access_token, mapping, serial, index, frameworks): logger.info("Search API URL: {search_api_url}", extra={'search_api_url': search_api_url}) logger.info("Data API URL: {data_api_url}", extra={'data_api_url': data_api_url}) if serial: mapper = map else: pool = ThreadPool(10) mapper = pool.imap_unordered indexer = indexers[doc_type]( doc_type, dmapiclient.DataAPIClient(data_api_url, data_api_access_token), dmapiclient.SearchAPIClient(search_api_url, search_api_access_token), index) if mapping and search_mapping_matches_framework(mapping, frameworks): indexer.create_index(mapping=mapping) counter = 0 start_time = datetime.utcnow() status = True items = indexer.request_items(frameworks) for result in mapper(indexer, items): counter += 1 status = status and result print_progress(counter, start_time) return status
def get_user(api_url, api_token, stage, role, framework, lot, *, brief_status=None): if not api_url: api_url = get_api_endpoint_from_stage(stage) api_token = api_token or get_auth_token('api', stage) api_client = dmapiclient.DataAPIClient(api_url, api_token) if role == 'supplier' and framework is not None: framework = get_full_framework_slug(framework) print('Framework: {}'.format(framework)) if lot is not None: print('Lot: {}'.format(lot)) supplier_id = get_supplier_id(api_client, framework, lot) print('Supplier id: {}'.format(supplier_id)) return get_random_user(api_client, None, supplier_id) if role == "buyer" and framework is not None: framework = get_full_framework_slug(framework) print('Framework: {}'.format(framework)) if framework.startswith("digital-outcomes-and-specialists"): print(f"Has requirements: {brief_status or 'True'}") return get_random_buyer_with_brief(api_client, framework, lot, brief_status=brief_status) else: return get_random_user(api_client, role)
def __call__(self, supplier): client = dmapiclient.DataAPIClient(self.endpoint, self.access_token) try: client.import_supplier(supplier['id'], self.clean_data(supplier, supplier['id'])) except dmapiclient.APIError as e: print("ERROR: {}. {} not imported".format(e.message, supplier.get('id')), file=sys.stderr) return False if not self.user_password: return True try: client.create_user({ 'role': 'supplier', 'emailAddress': CLEAN_FIELDS['email'].format( supplier['id'] ), 'password': self.user_password, 'name': supplier['name'], 'supplierId': supplier['id'], }) except dmapiclient.APIError as e: if e.status_code != 409: print("ERROR: {}. Could not create user account for {}".format( e.message, supplier.get('id')), file=sys.stderr) return False return True
def request_services(api_url, api_access_token, page=1): data_client = dmapiclient.DataAPIClient(api_url, api_access_token) while page: services_page = data_client.find_services(page=page) for service in services_page['services']: yield service if 'links' in services_page: links = services_page['links'] if isinstance(links, list): found = False for link in links: if 'rel' in link: if link['rel'] == 'next': page += 1 found = True if not found: return else: if 'next' in links: page += 1 else: return else: return
def main(data_api_url, data_api_access_token, email_api_key, stage, date_closed, dry_run): date_closed = get_date_closed(date_closed) if date_closed < (datetime.utcnow() - timedelta(days=8)).date(): logger.error( 'Not allowed to notify about briefs that closed more than a week ago' ) return False data_api_client = dmapiclient.DataAPIClient(data_api_url, data_api_access_token) closed_briefs = get_briefs_closed_on_date(data_api_client, date_closed) if not closed_briefs: logger.info("No briefs closed on {date_closed}", extra={"date_closed": date_closed}) return True logger.info("Notifying users about {briefs_count} closed briefs", extra={'briefs_count': len(closed_briefs)}) for brief in closed_briefs: if dry_run: logger.info("Would notify users about brief ID: {brief_id}", extra={'brief_id': brief['id']}) else: status = notify_users(email_api_key, stage, brief) if not status: return False return True
def main(api_url, api_access_token): client = dmapiclient.DataAPIClient(api_url, api_access_token) pool = ThreadPool(10) count = 1 for i in pool.imap_unordered(update(client), enumerate(client.find_services_iter())): count += i if count % 1000 == 0: print("** {}".format(count))
def main(api_url, api_access_token): client = dmapiclient.DataAPIClient(api_url, api_access_token) for id in [1, 2, 3]: print id try: client.req.applications(id).delete(data={"updated_by": ''}) print "deleted {}".format(id) except Exception, e: print e
def snapshot_framework_stats(api_endpoint, api_token, framework_slug): data_client = dmapiclient.DataAPIClient(api_endpoint, api_token) stats = data_client.get_framework_stats(framework_slug) data_client.create_audit_event(AuditTypes.snapshot_framework_stats, data=stats, object_type='frameworks', object_id=framework_slug) logger.info("Framework stats snapshot saved")
def main(api_url, api_access_token): client = dmapiclient.DataAPIClient(api_url, api_access_token) counters = {'total': 0, 'no_docs': 0, 'errors': 0} applications = client.req.applications().get() for application in applications['applications']: update_application_documents(client, counters, application) for k, v in counters.items(): print(k, v)
def main(api_url, api_access_token): client = dmapiclient.DataAPIClient(api_url, api_access_token) counters = {'approved': 0, 'no_app': 0, 'errors': 0, 'not_approved': 0} suppliers = client.req.suppliers().get(params='per_page=350') for supplier in suppliers['suppliers']: update_supplier_documents(client, counters, supplier['code']) for k, v in counters.items(): print(k, v)
def update_user_password(api_endpoint, api_token, user_email, new_password, unlock=False): data_client = dmapiclient.DataAPIClient(api_endpoint, api_token) try: user = data_client.get_user(email_address=user_email)['users'] data_client.update_user_password(user['id'], new_password) if unlock: data_client.update_user(user['id'], locked=False, updater=getpass.getuser()) logger.info("User unlocked") except dmapiclient.APIError: sys.exit(1) logger.info("Password updated")
def main( data_api_url, data_api_access_token, notify_api_key, notify_template_id, offset_days, dry_run=None, date_closed=None, user_id_list=None ): """ Send emails to buyers via Notify, reminding them to award their closed briefs offset_days: send emails for briefs that closed X days ago dry_run: log instead of sending emails date_closed: if supplied, send emails for briefs that closed on this date user_id_list: if supplied, only send emails to buyers with these user IDs """ logger.info("Data API URL: {data_api_url}", extra={'data_api_url': data_api_url}) date_closed = _get_brief_closing_date(offset_days, date_closed) if not date_closed: logger.error('Not allowed to notify about briefs that closed less than {} days ago', offset_days) return False data_api_client = dmapiclient.DataAPIClient(data_api_url, data_api_access_token) notify_client = scripts_notify_client(notify_api_key, logger=logger) closed_briefs = brief_data_helpers.get_briefs_closed_on_date(data_api_client, date_closed) if not closed_briefs: logger.info("No briefs closed on {date_closed}", extra={"date_closed": date_closed}) return True # If user list supplied, only continue for briefs that have at least one user in that list if user_id_list: closed_briefs = _filter_briefs_by_user_id_list(closed_briefs, user_id_list) logger.info("{briefs_count} closed brief(s) found with closing date {date_closed}", extra={ 'briefs_count': len(closed_briefs), "date_closed": date_closed }) failed_users_by_brief_id = {} for brief in closed_briefs: failed_users_for_this_brief = [] for user in brief['users']: failed_user_id = send_email_to_brief_user_via_notify( notify_client, notify_template_id, user, brief, user_id_list, dry_run ) if failed_user_id: failed_users_for_this_brief.append(failed_user_id) if failed_users_for_this_brief: failed_users_by_brief_id[brief['id']] = failed_users_for_this_brief if failed_users_by_brief_id: _log_failures(failed_users_by_brief_id, date_closed) return False return True
def __call__(self, service): client = dmapiclient.DataAPIClient(self.endpoint, self.access_token) user = self.get_user(service) fix_data = self.update_data(service) cleaned_data = self.clean_data(fix_data) try: client.import_service(service['id'], cleaned_data, user) return True except dmapiclient.APIError as e: print("ERROR: {}. {} not imported".format(e.message, service.get('id')), file=sys.stderr) return False
def request_suppliers(api_url, api_access_token, page=1): data_client = dmapiclient.DataAPIClient( api_url, api_access_token ) while page: suppliers_page = data_client.find_suppliers(page=page) for supplier in suppliers_page['suppliers']: yield supplier if suppliers_page['links'].get('next'): page += 1 else: return
from flask import Flask, request, redirect, session from flask_login import LoginManager from flask_wtf.csrf import CSRFProtect from werkzeug.local import Local, LocalProxy import dmapiclient from dmcontent.content_loader import ContentLoader from dmutils import init_app, formats from dmutils.timing import logged_duration from dmutils.user import User from govuk_frontend_jinja.flask_ext import init_govuk_frontend from config import configs csrf = CSRFProtect() data_api_client = dmapiclient.DataAPIClient() login_manager = LoginManager() # we use our own Local for objects we explicitly want to be able to retain between requests but shouldn't # share a common object between concurrent threads/contexts _local = Local() # These frameworks pre-date the introduction of the edit_service_as_admin and declaration manifests. OLD_FRAMEWORKS_WITH_MISSING_MANIFESTS = ['g-cloud-4', 'g-cloud-5', 'g-cloud-6'] def _log_missing_manifest(application, manifest_name, framework_slug): if framework_slug in OLD_FRAMEWORKS_WITH_MISSING_MANIFESTS: logger = application.logger.debug else: logger = application.logger.error
), "digital-outcomes-and-specialists": (), } _expected_framework_statuses = ("pending", "standstill",) if __name__ == '__main__': arguments = docopt(__doc__) configure_logger() STAGE = arguments['<stage>'] api_url = get_api_endpoint_from_stage(STAGE) client = dmapiclient.DataAPIClient(api_url, get_auth_token('api', STAGE)) DRAFT_BUCKET = arguments.get('<draft_bucket>') and S3(arguments['<draft_bucket>']) DOCUMENTS_BUCKET = arguments.get('<documents_bucket>') and S3(arguments['<documents_bucket>']) DRY_RUN = arguments['--dry-run'] FRAMEWORK_SLUG = arguments['<framework_slug>'] DRAFT_IDS_FILE = arguments.get('--draft-ids') and open(arguments['--draft-ids'], "r") SKIP_DOCS_IF_PUBLISHED = arguments['--skip-docs-if-published'] framework = client.get_framework(FRAMEWORK_SLUG)["frameworks"] if framework["status"] not in _expected_framework_statuses: raise ValueError( f"Framework status expected to be one of {_expected_framework_statuses}, not {framework['status']!r}" ) document_keys = _document_keys_by_framework_family.get(framework["family"])
def supplier_search(): sort_by = request.args.get('sort_by', 'a-z') sort_order = request.args.get('sort_order', 'asc') if sort_order not in ('asc', 'desc'): abort(400, 'Invalid sort_order: {}'.format(sort_order)) sort_terms = request.args.getlist('sort_term') keyword = request.args.get('keyword', None) selected_roles = set(request.args.getlist('role')) selected_seller_types = set(request.args.getlist('type')) selected_seller_types_keys = [ to_seller_type_key(x) for x in selected_seller_types ] if not sort_terms: # Sort by A-Z for default sort_terms = ['name'] data_api_client = DataAPIClient() real_data_api_client = dmapiclient.DataAPIClient() real_data_api_client.init_app(current_app) roles = get_all_domains(data_api_client) role_filters = {role: role in selected_roles for role in roles} sort_queries = [] allowed_sort_terms = set(('name', )) # Limit what can be sorted for sort_term in sort_terms: if sort_term in allowed_sort_terms: if sort_term == 'name': # Use 'name' in url to keep it clean but query needs to search on not analyzed. sort_term = 'name.not_analyzed' spec = {"order": sort_order, "mode": "min"} if sort_by: spec['sort_by'] = sort_by sort_queries.append({sort_term: spec}) else: abort(400, 'Invalid sort_term: {}'.format(sort_term)) query_contents = {} filter_terms = {} product_search_parameters = dict(sort_dir=sort_order, seller_types=selected_seller_types_keys, search_term=keyword, domains=list(selected_roles)) if selected_roles: for each in selected_roles: if each not in roles: abort(400, 'Invalid role: {}'.format(each)) filter_terms = {"domains.assessed": list(selected_roles)} if selected_seller_types: filter_terms['seller_types'] = list(selected_seller_types_keys) if filter_terms: query_contents['filtered'] = { "query": { "match_all": {} }, "filter": { "terms": filter_terms, } } if keyword: query_contents['match_phrase_prefix'] = {"name": keyword} else: query_contents['match_all'] = {} query = {"query": query_contents, "sort": sort_queries} try: page = int(request.args.get('page', 1)) except ValueError: abort(400, 'Invalid page number: {}'.format(request.args['page'])) results_from = (page - 1) * SUPPLIER_RESULTS_PER_PAGE page_params = {'from': results_from, 'size': SUPPLIER_RESULTS_PER_PAGE} query.update(product_search_parameters) products_requester = real_data_api_client.req.products().search() casestudies_requester = real_data_api_client.req.casestudies().search() with ThreadPoolExecutor(max_workers=16) as executor: futures_to_result = { executor.submit(data_api_client.find_suppliers, data=query, params=page_params): 'suppliers', executor.submit(products_requester.get, data=query, params=page_params): 'products', executor.submit(casestudies_requester.get, data=query, params=page_params): 'casestudies' } results = {} for future in as_completed(futures_to_result): results[futures_to_result[future]] = future.result() response = results['suppliers'] products_response = results['products'] casestudies_response = results.get('casestudies') results = [] services = {} for supplier in response['hits']['hits']: details = supplier['_source'] domains = details['domains'] tags = domains['assessed'] + domains['unassessed'] services = {} for tag in sorted(tags): services[tag] = True seller_type = details.get('seller_type', {}) is_recruiter = details.get('is_recruiter', False) if is_recruiter == 'true' and 'recruiter' not in seller_type.keys(): seller_type['recruitment'] = True result = { 'title': details['name'], 'description': details['summary'], 'link': url_for('.get_supplier', code=details['code']), 'services': services, 'badges': seller_type } results.append(result) num_results = response['hits']['total'] seller_type_filters = { st: to_seller_type_key(st) in selected_seller_types_keys for st in SELLER_TYPES.keys() } products_results = [] for p in products_response['hits']['hits']: details = p['_source'] supplier = dict() supplier['name'] = details.get('supplierName') supplier['profile_url'] = '/supplier/%s' % details['supplier_code'] supplier['support_url'] = details.get('support') services = {} result = { 'title': details['name'], 'description': details['summary'], 'link': details['website'], 'services': services, 'badges': details.get('seller_type', {}), 'supplier': supplier, 'pricing': details['pricing'] } products_results.append(result) num_products_results = products_response['hits']['total'] casestudies_results = None if current_user.is_authenticated and current_user.role == 'buyer': casestudies_results = [] for p in casestudies_response['hits']['hits']: details = p['_source'] domains = details.get('domains', {}) supplier = dict() supplier['name'] = details.get('supplierName') supplier['profile_url'] = '/supplier/%s' % details.get( 'supplierCode') tags = domains.get('assessed', []) + domains.get('unassessed', []) services = {} for tag in sorted(tags): services[tag] = True result = { 'title': details['title'], 'description': smart_truncate(details.get('approach', '')), 'link': url_for('.get_supplier_case_study', casestudy_id=details['id']), 'services': services, 'badges': details.get('seller_type', {}), 'supplier': supplier, 'pricing': details.get('pricing'), 'case_study_service': details.get('service') } casestudies_results.append(result) num_casestudies_results = casestudies_response['hits'][ 'total'] if casestudies_response else 0 def get_pagination(result_count): pages = get_page_list(SUPPLIER_RESULTS_PER_PAGE, result_count, page) return { 'pages': pages, 'page': page, 'pageCount': pages[-1], 'total': result_count } props = { 'form_options': { 'action': url_for('.supplier_search') }, 'search': { 'results': results[:SUPPLIER_RESULTS_PER_PAGE], 'products': products_results[:SUPPLIER_RESULTS_PER_PAGE], 'casestudies': casestudies_results[:SUPPLIER_RESULTS_PER_PAGE] if casestudies_results else None, 'keyword': keyword, 'sort_by': sort_by, 'view': request.args.get('view', 'sellers'), 'role': role_filters, 'type': seller_type_filters, 'user_role': None if current_user.is_anonymous else current_user.role }, 'pagination': { 'sellers': get_pagination(num_results), 'products': get_pagination(num_products_results), 'casestudies': get_pagination(num_casestudies_results) }, 'basename': url_for('.supplier_search') } if request_wants_json(): return jsonify(dict(props)) else: component = render_component('bundles/Search/SearchWidget.js', props) return render_template('_react.html', component=component, breadcrumb_items=[{ 'link': url_for('main.index'), 'label': 'Home' }, { 'label': 'Seller catalogue' }])
def main(data_api_url, data_api_token, email_api_key, stage, dry_run, supplier_ids=[]): logger.info("Begin to send brief update notification emails") # get today at 8 in the morning end_date = datetime.utcnow().replace(hour=8, minute=0, second=0, microsecond=0) # get yesterday at 8 in the morning start_date = end_date - timedelta(days=1) # Initialise data API client data_api_client = dmapiclient.DataAPIClient(data_api_url, data_api_token) # we want to find all questions and answers that were submitted between start and end dates briefs = get_live_briefs_with_new_questions_and_answers_between_two_dates(data_api_client, start_date, end_date) logger.info( "{} briefs with new question and answers found between {} and {}".format(len(briefs), start_date, end_date) ) # find the IDs of interested suppliers {supplier_id: [briefid1, briefid2]} interested_suppliers = get_ids_of_interested_suppliers_for_briefs(data_api_client, briefs) # Restrict suppliers to ones specified in the argument if supplier_ids: interested_suppliers = dict( (supplier_id, briefs) for supplier_id, briefs in interested_suppliers.items() if supplier_id in supplier_ids ) logger.info( "{} suppliers found interested in these briefs with the following IDs: {}".format( len(interested_suppliers), ",".join(map(str, interested_suppliers.keys())) ) ) failed_supplier_ids = [] for supplier_id, brief_ids in interested_suppliers.items(): # Get the brief objects for this supplier supplier_briefs = [b for b in briefs if b['id'] in brief_ids] # get a context for each supplier email supplier_context = create_context_for_supplier(stage, supplier_briefs) email_addresses = get_supplier_email_addresses_by_supplier_id(data_api_client, supplier_id) if not email_addresses: logger.info( "Email not sent for the following supplier ID due to no active users: {supplier_id}", extra={ 'supplier_id': supplier_id, 'brief_ids_list': ", ".join(map(str, brief_ids)) } ) elif dry_run: logger.info( "Would notify supplier ID {supplier_id} for brief IDs {brief_ids_list}", extra={ 'supplier_id': supplier_id, 'brief_ids_list': ", ".join(map(str, brief_ids)) } ) elif len(email_addresses) == 0: logger.info( "Email not sent for the following supplier ID due to no active users: {supplier_id}", extra={ 'supplier_id': supplier_id, 'brief_ids_list': ", ".join(map(str, brief_ids)) } ) else: logger.info( "Notifying supplier ID {supplier_id} of new questions/answers for brief IDs {brief_ids_list}", extra={ 'supplier_id': supplier_id, 'brief_ids_list': ", ".join(map(str, brief_ids)) } ) try: send_supplier_emails(email_api_key, email_addresses, supplier_context, logger) except EmailError: failed_supplier_ids.append(supplier_id) if failed_supplier_ids: logger.error( 'Email sending failed for the following supplier IDs: {supplier_ids}', extra={"supplier_ids": ",".join(map(str, failed_supplier_ids))} ) return False return True