def do_index(doc_type, search_api_url, search_api_access_token, data_api_url, data_api_access_token, mapping, serial,
index, frameworks):
logger.info("Search API URL: {search_api_url}", extra={'search_api_url': search_api_url})
logger.info("Data API URL: {data_api_url}", extra={'data_api_url': data_api_url})
if serial:
mapper = map
else:
pool = ThreadPool(10)
mapper = pool.imap_unordered
indexer = indexers[doc_type](
doc_type,
dmapiclient.DataAPIClient(data_api_url, data_api_access_token),
dmapiclient.SearchAPIClient(search_api_url, search_api_access_token),
index)
if mapping and search_mapping_matches_framework(mapping, frameworks):
indexer.create_index(mapping=mapping)
counter = 0
start_time = datetime.utcnow()
status = True
items = indexer.request_items(frameworks)
for result in mapper(indexer, items):
counter += 1
status = status and result
print_progress(counter, start_time)
return status
def get_user(api_url,
api_token,
stage,
role,
framework,
lot,
*,
brief_status=None):
if not api_url:
api_url = get_api_endpoint_from_stage(stage)
api_token = api_token or get_auth_token('api', stage)
api_client = dmapiclient.DataAPIClient(api_url, api_token)
if role == 'supplier' and framework is not None:
framework = get_full_framework_slug(framework)
print('Framework: {}'.format(framework))
if lot is not None:
print('Lot: {}'.format(lot))
supplier_id = get_supplier_id(api_client, framework, lot)
print('Supplier id: {}'.format(supplier_id))
return get_random_user(api_client, None, supplier_id)
if role == "buyer" and framework is not None:
framework = get_full_framework_slug(framework)
print('Framework: {}'.format(framework))
if framework.startswith("digital-outcomes-and-specialists"):
print(f"Has requirements: {brief_status or 'True'}")
return get_random_buyer_with_brief(api_client,
framework,
lot,
brief_status=brief_status)
else:
return get_random_user(api_client, role)
def __call__(self, supplier):
client = dmapiclient.DataAPIClient(self.endpoint, self.access_token)
try:
client.import_supplier(supplier['id'],
self.clean_data(supplier, supplier['id']))
except dmapiclient.APIError as e:
print("ERROR: {}. {} not imported".format(e.message,
supplier.get('id')),
file=sys.stderr)
return False
if not self.user_password:
return True
try:
client.create_user({
'role': 'supplier',
'emailAddress': CLEAN_FIELDS['email'].format(
supplier['id']
),
'password': self.user_password,
'name': supplier['name'],
'supplierId': supplier['id'],
})
except dmapiclient.APIError as e:
if e.status_code != 409:
print("ERROR: {}. Could not create user account for {}".format(
e.message, supplier.get('id')), file=sys.stderr)
return False
return True
def request_services(api_url, api_access_token, page=1):
data_client = dmapiclient.DataAPIClient(api_url, api_access_token)
while page:
services_page = data_client.find_services(page=page)
for service in services_page['services']:
yield service
if 'links' in services_page:
links = services_page['links']
if isinstance(links, list):
found = False
for link in links:
if 'rel' in link:
if link['rel'] == 'next':
page += 1
found = True
if not found:
return
else:
if 'next' in links:
page += 1
else:
return
else:
return
def main(data_api_url, data_api_access_token, email_api_key, stage,
date_closed, dry_run):
date_closed = get_date_closed(date_closed)
if date_closed < (datetime.utcnow() - timedelta(days=8)).date():
logger.error(
'Not allowed to notify about briefs that closed more than a week ago'
)
return False
data_api_client = dmapiclient.DataAPIClient(data_api_url,
data_api_access_token)
closed_briefs = get_briefs_closed_on_date(data_api_client, date_closed)
if not closed_briefs:
logger.info("No briefs closed on {date_closed}",
extra={"date_closed": date_closed})
return True
logger.info("Notifying users about {briefs_count} closed briefs",
extra={'briefs_count': len(closed_briefs)})
for brief in closed_briefs:
if dry_run:
logger.info("Would notify users about brief ID: {brief_id}",
extra={'brief_id': brief['id']})
else:
status = notify_users(email_api_key, stage, brief)
if not status:
return False
return True
def main(api_url, api_access_token):
client = dmapiclient.DataAPIClient(api_url, api_access_token)
pool = ThreadPool(10)
count = 1
for i in pool.imap_unordered(update(client),
enumerate(client.find_services_iter())):
count += i
if count % 1000 == 0:
print("** {}".format(count))
def main(api_url, api_access_token):
client = dmapiclient.DataAPIClient(api_url, api_access_token)
for id in [1, 2, 3]:
print id
try:
client.req.applications(id).delete(data={"updated_by": ''})
print "deleted {}".format(id)
except Exception, e:
print e
def snapshot_framework_stats(api_endpoint, api_token, framework_slug):
data_client = dmapiclient.DataAPIClient(api_endpoint, api_token)
stats = data_client.get_framework_stats(framework_slug)
data_client.create_audit_event(AuditTypes.snapshot_framework_stats,
data=stats,
object_type='frameworks',
object_id=framework_slug)
logger.info("Framework stats snapshot saved")
def main(api_url, api_access_token):
client = dmapiclient.DataAPIClient(api_url, api_access_token)
counters = {'total': 0, 'no_docs': 0, 'errors': 0}
applications = client.req.applications().get()
for application in applications['applications']:
update_application_documents(client, counters, application)
for k, v in counters.items():
print(k, v)
def main(api_url, api_access_token):
client = dmapiclient.DataAPIClient(api_url, api_access_token)
counters = {'approved': 0, 'no_app': 0, 'errors': 0, 'not_approved': 0}
suppliers = client.req.suppliers().get(params='per_page=350')
for supplier in suppliers['suppliers']:
update_supplier_documents(client, counters, supplier['code'])
for k, v in counters.items():
print(k, v)
def update_user_password(api_endpoint, api_token, user_email, new_password, unlock=False):
data_client = dmapiclient.DataAPIClient(api_endpoint, api_token)
try:
user = data_client.get_user(email_address=user_email)['users']
data_client.update_user_password(user['id'], new_password)
if unlock:
data_client.update_user(user['id'], locked=False, updater=getpass.getuser())
logger.info("User unlocked")
except dmapiclient.APIError:
sys.exit(1)
logger.info("Password updated")
def main(
data_api_url, data_api_access_token, notify_api_key, notify_template_id, offset_days,
dry_run=None, date_closed=None, user_id_list=None
):
"""
Send emails to buyers via Notify, reminding them to award their closed briefs
offset_days: send emails for briefs that closed X days ago
dry_run: log instead of sending emails
date_closed: if supplied, send emails for briefs that closed on this date
user_id_list: if supplied, only send emails to buyers with these user IDs
"""
logger.info("Data API URL: {data_api_url}", extra={'data_api_url': data_api_url})
date_closed = _get_brief_closing_date(offset_days, date_closed)
if not date_closed:
logger.error('Not allowed to notify about briefs that closed less than {} days ago', offset_days)
return False
data_api_client = dmapiclient.DataAPIClient(data_api_url, data_api_access_token)
notify_client = scripts_notify_client(notify_api_key, logger=logger)
closed_briefs = brief_data_helpers.get_briefs_closed_on_date(data_api_client, date_closed)
if not closed_briefs:
logger.info("No briefs closed on {date_closed}", extra={"date_closed": date_closed})
return True
# If user list supplied, only continue for briefs that have at least one user in that list
if user_id_list:
closed_briefs = _filter_briefs_by_user_id_list(closed_briefs, user_id_list)
logger.info("{briefs_count} closed brief(s) found with closing date {date_closed}", extra={
'briefs_count': len(closed_briefs), "date_closed": date_closed
})
failed_users_by_brief_id = {}
for brief in closed_briefs:
failed_users_for_this_brief = []
for user in brief['users']:
failed_user_id = send_email_to_brief_user_via_notify(
notify_client, notify_template_id, user, brief, user_id_list, dry_run
)
if failed_user_id:
failed_users_for_this_brief.append(failed_user_id)
if failed_users_for_this_brief:
failed_users_by_brief_id[brief['id']] = failed_users_for_this_brief
if failed_users_by_brief_id:
_log_failures(failed_users_by_brief_id, date_closed)
return False
return True
def __call__(self, service):
client = dmapiclient.DataAPIClient(self.endpoint, self.access_token)
user = self.get_user(service)
fix_data = self.update_data(service)
cleaned_data = self.clean_data(fix_data)
try:
client.import_service(service['id'], cleaned_data, user)
return True
except dmapiclient.APIError as e:
print("ERROR: {}. {} not imported".format(e.message,
service.get('id')),
file=sys.stderr)
return False
def request_suppliers(api_url, api_access_token, page=1):
data_client = dmapiclient.DataAPIClient(
api_url,
api_access_token
)
while page:
suppliers_page = data_client.find_suppliers(page=page)
for supplier in suppliers_page['suppliers']:
yield supplier
if suppliers_page['links'].get('next'):
page += 1
else:
return
from flask import Flask, request, redirect, session
from flask_login import LoginManager
from flask_wtf.csrf import CSRFProtect
from werkzeug.local import Local, LocalProxy
import dmapiclient
from dmcontent.content_loader import ContentLoader
from dmutils import init_app, formats
from dmutils.timing import logged_duration
from dmutils.user import User
from govuk_frontend_jinja.flask_ext import init_govuk_frontend
from config import configs
csrf = CSRFProtect()
data_api_client = dmapiclient.DataAPIClient()
login_manager = LoginManager()
# we use our own Local for objects we explicitly want to be able to retain between requests but shouldn't
# share a common object between concurrent threads/contexts
_local = Local()
# These frameworks pre-date the introduction of the edit_service_as_admin and declaration manifests.
OLD_FRAMEWORKS_WITH_MISSING_MANIFESTS = ['g-cloud-4', 'g-cloud-5', 'g-cloud-6']
def _log_missing_manifest(application, manifest_name, framework_slug):
if framework_slug in OLD_FRAMEWORKS_WITH_MISSING_MANIFESTS:
logger = application.logger.debug
else:
logger = application.logger.error
),
"digital-outcomes-and-specialists": (),
}
_expected_framework_statuses = ("pending", "standstill",)
if __name__ == '__main__':
arguments = docopt(__doc__)
configure_logger()
STAGE = arguments['<stage>']
api_url = get_api_endpoint_from_stage(STAGE)
client = dmapiclient.DataAPIClient(api_url, get_auth_token('api', STAGE))
DRAFT_BUCKET = arguments.get('<draft_bucket>') and S3(arguments['<draft_bucket>'])
DOCUMENTS_BUCKET = arguments.get('<documents_bucket>') and S3(arguments['<documents_bucket>'])
DRY_RUN = arguments['--dry-run']
FRAMEWORK_SLUG = arguments['<framework_slug>']
DRAFT_IDS_FILE = arguments.get('--draft-ids') and open(arguments['--draft-ids'], "r")
SKIP_DOCS_IF_PUBLISHED = arguments['--skip-docs-if-published']
framework = client.get_framework(FRAMEWORK_SLUG)["frameworks"]
if framework["status"] not in _expected_framework_statuses:
raise ValueError(
f"Framework status expected to be one of {_expected_framework_statuses}, not {framework['status']!r}"
)
document_keys = _document_keys_by_framework_family.get(framework["family"])
def supplier_search():
sort_by = request.args.get('sort_by', 'a-z')
sort_order = request.args.get('sort_order', 'asc')
if sort_order not in ('asc', 'desc'):
abort(400, 'Invalid sort_order: {}'.format(sort_order))
sort_terms = request.args.getlist('sort_term')
keyword = request.args.get('keyword', None)
selected_roles = set(request.args.getlist('role'))
selected_seller_types = set(request.args.getlist('type'))
selected_seller_types_keys = [
to_seller_type_key(x) for x in selected_seller_types
]
if not sort_terms: # Sort by A-Z for default
sort_terms = ['name']
data_api_client = DataAPIClient()
real_data_api_client = dmapiclient.DataAPIClient()
real_data_api_client.init_app(current_app)
roles = get_all_domains(data_api_client)
role_filters = {role: role in selected_roles for role in roles}
sort_queries = []
allowed_sort_terms = set(('name', )) # Limit what can be sorted
for sort_term in sort_terms:
if sort_term in allowed_sort_terms:
if sort_term == 'name': # Use 'name' in url to keep it clean but query needs to search on not analyzed.
sort_term = 'name.not_analyzed'
spec = {"order": sort_order, "mode": "min"}
if sort_by:
spec['sort_by'] = sort_by
sort_queries.append({sort_term: spec})
else:
abort(400, 'Invalid sort_term: {}'.format(sort_term))
query_contents = {}
filter_terms = {}
product_search_parameters = dict(sort_dir=sort_order,
seller_types=selected_seller_types_keys,
search_term=keyword,
domains=list(selected_roles))
if selected_roles:
for each in selected_roles:
if each not in roles:
abort(400, 'Invalid role: {}'.format(each))
filter_terms = {"domains.assessed": list(selected_roles)}
if selected_seller_types:
filter_terms['seller_types'] = list(selected_seller_types_keys)
if filter_terms:
query_contents['filtered'] = {
"query": {
"match_all": {}
},
"filter": {
"terms": filter_terms,
}
}
if keyword:
query_contents['match_phrase_prefix'] = {"name": keyword}
else:
query_contents['match_all'] = {}
query = {"query": query_contents, "sort": sort_queries}
try:
page = int(request.args.get('page', 1))
except ValueError:
abort(400, 'Invalid page number: {}'.format(request.args['page']))
results_from = (page - 1) * SUPPLIER_RESULTS_PER_PAGE
page_params = {'from': results_from, 'size': SUPPLIER_RESULTS_PER_PAGE}
query.update(product_search_parameters)
products_requester = real_data_api_client.req.products().search()
casestudies_requester = real_data_api_client.req.casestudies().search()
with ThreadPoolExecutor(max_workers=16) as executor:
futures_to_result = {
executor.submit(data_api_client.find_suppliers,
data=query,
params=page_params):
'suppliers',
executor.submit(products_requester.get,
data=query,
params=page_params):
'products',
executor.submit(casestudies_requester.get,
data=query,
params=page_params):
'casestudies'
}
results = {}
for future in as_completed(futures_to_result):
results[futures_to_result[future]] = future.result()
response = results['suppliers']
products_response = results['products']
casestudies_response = results.get('casestudies')
results = []
services = {}
for supplier in response['hits']['hits']:
details = supplier['_source']
domains = details['domains']
tags = domains['assessed'] + domains['unassessed']
services = {}
for tag in sorted(tags):
services[tag] = True
seller_type = details.get('seller_type', {})
is_recruiter = details.get('is_recruiter', False)
if is_recruiter == 'true' and 'recruiter' not in seller_type.keys():
seller_type['recruitment'] = True
result = {
'title': details['name'],
'description': details['summary'],
'link': url_for('.get_supplier', code=details['code']),
'services': services,
'badges': seller_type
}
results.append(result)
num_results = response['hits']['total']
seller_type_filters = {
st: to_seller_type_key(st) in selected_seller_types_keys
for st in SELLER_TYPES.keys()
}
products_results = []
for p in products_response['hits']['hits']:
details = p['_source']
supplier = dict()
supplier['name'] = details.get('supplierName')
supplier['profile_url'] = '/supplier/%s' % details['supplier_code']
supplier['support_url'] = details.get('support')
services = {}
result = {
'title': details['name'],
'description': details['summary'],
'link': details['website'],
'services': services,
'badges': details.get('seller_type', {}),
'supplier': supplier,
'pricing': details['pricing']
}
products_results.append(result)
num_products_results = products_response['hits']['total']
casestudies_results = None
if current_user.is_authenticated and current_user.role == 'buyer':
casestudies_results = []
for p in casestudies_response['hits']['hits']:
details = p['_source']
domains = details.get('domains', {})
supplier = dict()
supplier['name'] = details.get('supplierName')
supplier['profile_url'] = '/supplier/%s' % details.get(
'supplierCode')
tags = domains.get('assessed', []) + domains.get('unassessed', [])
services = {}
for tag in sorted(tags):
services[tag] = True
result = {
'title':
details['title'],
'description':
smart_truncate(details.get('approach', '')),
'link':
url_for('.get_supplier_case_study',
casestudy_id=details['id']),
'services':
services,
'badges':
details.get('seller_type', {}),
'supplier':
supplier,
'pricing':
details.get('pricing'),
'case_study_service':
details.get('service')
}
casestudies_results.append(result)
num_casestudies_results = casestudies_response['hits'][
'total'] if casestudies_response else 0
def get_pagination(result_count):
pages = get_page_list(SUPPLIER_RESULTS_PER_PAGE, result_count, page)
return {
'pages': pages,
'page': page,
'pageCount': pages[-1],
'total': result_count
}
props = {
'form_options': {
'action': url_for('.supplier_search')
},
'search': {
'results':
results[:SUPPLIER_RESULTS_PER_PAGE],
'products':
products_results[:SUPPLIER_RESULTS_PER_PAGE],
'casestudies':
casestudies_results[:SUPPLIER_RESULTS_PER_PAGE]
if casestudies_results else None,
'keyword':
keyword,
'sort_by':
sort_by,
'view':
request.args.get('view', 'sellers'),
'role':
role_filters,
'type':
seller_type_filters,
'user_role':
None if current_user.is_anonymous else current_user.role
},
'pagination': {
'sellers': get_pagination(num_results),
'products': get_pagination(num_products_results),
'casestudies': get_pagination(num_casestudies_results)
},
'basename': url_for('.supplier_search')
}
if request_wants_json():
return jsonify(dict(props))
else:
component = render_component('bundles/Search/SearchWidget.js', props)
return render_template('_react.html',
component=component,
breadcrumb_items=[{
'link': url_for('main.index'),
'label': 'Home'
}, {
'label': 'Seller catalogue'
}])
def main(data_api_url, data_api_token, email_api_key, stage, dry_run, supplier_ids=[]):
logger.info("Begin to send brief update notification emails")
# get today at 8 in the morning
end_date = datetime.utcnow().replace(hour=8, minute=0, second=0, microsecond=0)
# get yesterday at 8 in the morning
start_date = end_date - timedelta(days=1)
# Initialise data API client
data_api_client = dmapiclient.DataAPIClient(data_api_url, data_api_token)
# we want to find all questions and answers that were submitted between start and end dates
briefs = get_live_briefs_with_new_questions_and_answers_between_two_dates(data_api_client, start_date, end_date)
logger.info(
"{} briefs with new question and answers found between {} and {}".format(len(briefs), start_date, end_date)
)
# find the IDs of interested suppliers {supplier_id: [briefid1, briefid2]}
interested_suppliers = get_ids_of_interested_suppliers_for_briefs(data_api_client, briefs)
# Restrict suppliers to ones specified in the argument
if supplier_ids:
interested_suppliers = dict(
(supplier_id, briefs) for supplier_id, briefs in interested_suppliers.items()
if supplier_id in supplier_ids
)
logger.info(
"{} suppliers found interested in these briefs with the following IDs: {}".format(
len(interested_suppliers),
",".join(map(str, interested_suppliers.keys()))
)
)
failed_supplier_ids = []
for supplier_id, brief_ids in interested_suppliers.items():
# Get the brief objects for this supplier
supplier_briefs = [b for b in briefs if b['id'] in brief_ids]
# get a context for each supplier email
supplier_context = create_context_for_supplier(stage, supplier_briefs)
email_addresses = get_supplier_email_addresses_by_supplier_id(data_api_client, supplier_id)
if not email_addresses:
logger.info(
"Email not sent for the following supplier ID due to no active users: {supplier_id}",
extra={
'supplier_id': supplier_id,
'brief_ids_list': ", ".join(map(str, brief_ids))
}
)
elif dry_run:
logger.info(
"Would notify supplier ID {supplier_id} for brief IDs {brief_ids_list}",
extra={
'supplier_id': supplier_id,
'brief_ids_list': ", ".join(map(str, brief_ids))
}
)
elif len(email_addresses) == 0:
logger.info(
"Email not sent for the following supplier ID due to no active users: {supplier_id}",
extra={
'supplier_id': supplier_id,
'brief_ids_list': ", ".join(map(str, brief_ids))
}
)
else:
logger.info(
"Notifying supplier ID {supplier_id} of new questions/answers for brief IDs {brief_ids_list}",
extra={
'supplier_id': supplier_id,
'brief_ids_list': ", ".join(map(str, brief_ids))
}
)
try:
send_supplier_emails(email_api_key, email_addresses, supplier_context, logger)
except EmailError:
failed_supplier_ids.append(supplier_id)
if failed_supplier_ids:
logger.error(
'Email sending failed for the following supplier IDs: {supplier_ids}',
extra={"supplier_ids": ",".join(map(str, failed_supplier_ids))}
)
return False
return True
