def test_get_api_endpoint_for_dev_environments(self, stage): with patch.dict("os.environ", {}): assert get_api_endpoint_from_stage( stage) == "http://localhost:5000" with patch.dict("os.environ", {"DM_API_PORT": "9000"}): assert get_api_endpoint_from_stage( stage) == "http://localhost:9000"
def test_get_search_api_endpoint_for_dev_environments(self, stage): with patch.dict("os.environ", {}): assert get_api_endpoint_from_stage( stage, app="search-api") == "http://localhost:5009" with patch.dict("os.environ", {"DM_SEARCH_API_PORT": "9001"}): assert get_api_endpoint_from_stage( stage, app="search-api") == "http://localhost:9001"
def clients_in_shell(stage): print('Retrieving credentials...') api_token = 'myToken' search_api_token = 'myToken' if stage != 'development': api_token = get_auth_token('api', stage), search_api_token = get_auth_token('search_api', stage) print('Creating clients...') data = DataAPIClient(get_api_endpoint_from_stage(stage), api_token) # noqa search = SearchAPIClient(get_api_endpoint_from_stage(stage, app='search-api'), search_api_token) # noqa print('Dropping into shell...') IPython.embed()
def clients_in_shell(stage, api_url, api_token, search_api_url, search_api_token): print('Retrieving credentials...') api_token = api_token or get_auth_token('api', stage) search_api_token = search_api_token or get_auth_token('search_api', stage) print('Creating clients...') data = DataAPIClient(api_url or get_api_endpoint_from_stage(stage), api_token) # noqa search = SearchAPIClient( search_api_url or get_api_endpoint_from_stage(stage, app='search-api'), search_api_token) # noqa print('Dropping into shell...') IPython.embed()
def get_user(api_url, api_token, stage, role, framework, lot, *, brief_status=None): if not api_url: api_url = get_api_endpoint_from_stage(stage) api_token = api_token or get_auth_token('api', stage) api_client = dmapiclient.DataAPIClient(api_url, api_token) if role == 'supplier' and framework is not None: framework = get_full_framework_slug(framework) print('Framework: {}'.format(framework)) if lot is not None: print('Lot: {}'.format(lot)) supplier_id = get_supplier_id(api_client, framework, lot) print('Supplier id: {}'.format(supplier_id)) return get_random_user(api_client, None, supplier_id) if role == "buyer" and framework is not None: framework = get_full_framework_slug(framework) print('Framework: {}'.format(framework)) if framework.startswith("digital-outcomes-and-specialists"): print(f"Has requirements: {brief_status or 'True'}") return get_random_buyer_with_brief(api_client, framework, lot, brief_status=brief_status) else: return get_random_user(api_client, role)
def get_dmp_supplier_data(self, framework=None, duns_number=None, from_declaration=False): "return the DMp data for a given DUNS number and initialises the DMp client if None" # TODO: error handling if self.data_api_client is None: self.data_api_client = DataAPIClient( base_url=get_api_endpoint_from_stage(self.stage), auth_token=get_auth_token('api', self.stage)) if duns_number is not None: return self.data_api_client.find_suppliers(duns_number=duns_number) elif framework is not None: # TODO: use iter instead -> digitalmarketplace-apiclient/blob/master/dmapiclient/data.py#L119 # TODO: check pagination if from_declaration: return self.data_api_client.find_framework_suppliers(framework) return self.data_api_client.find_suppliers(framework=framework)
def inject_framework_dates(stage): data_api_token = get_auth_token( 'api', stage) if stage != 'development' else 'myToken' data_api_client = DataAPIClient(get_api_endpoint_from_stage(stage), data_api_token) for framework_slug, framework_data in FRAMEWORKS_AND_DATES.items(): print(f'Injecting dates for {framework_slug}: {framework_data}') try: data_api_client.update_framework( framework_slug=framework_slug, data=framework_data, user=f'{getpass.getuser()} - ' f'digitalmarketplace-scripts/scripts/' f'oneoff/inject-framework-dates.py') except Exception as e: print( f'Failed with {e} on {framework_slug}. Data: {framework_data}')
def reset_supplier_declaration(stage, framework_slug, reason, email, supplier_id): data_api_token = get_auth_token('api', stage) if stage != 'development' else 'myToken' data_api_client = DataAPIClient(get_api_endpoint_from_stage(stage), data_api_token) if email: user = data_api_client.get_user(email_address=email) if not user: print(f'No user found for email address `{email}`') exit(1) user_supplier_id = user['users']['supplier']['supplierId'] if user_supplier_id and supplier_id and user_supplier_id != supplier_id: print(f'Email address provided does not match with supplier provided. Email address `{email}` is ' f'associated with supplierId `{supplier_id}`. Script was called with supplierId `{supplier_id}`.') exit(2) supplier_id = user_supplier_id try: supplier_framework = data_api_client.get_supplier_framework_info(supplier_id=supplier_id, framework_slug=framework_slug) print(f"Current supplier declaration: {supplier_framework['frameworkInterest']['declaration']}") except HTTPError: print(f'No supplier framework found for supplierId `{supplier_id}` on framework `{framework_slug}`.') exit(3) if not supplier_framework: print(f'No supplier framework/interest record found for supplierId `{supplier_id}` on framework ' f'`{framework_slug}`.') exit(4) data_api_client.set_supplier_declaration(supplier_id=supplier_id, framework_slug=framework_slug, declaration={}, user=f'{getpass.getuser()} - {reason}') data_api_client.set_supplier_framework_prefill_declaration(supplier_id=supplier_id, framework_slug=framework_slug, prefill_declaration_from_framework_slug=None, user=f'{getpass.getuser()} - {reason}') print(f'Supplier declaration for supplierId `{supplier_id}` on framework `{framework_slug}` has been reset.')
def main(): args = docopt(__doc__) frameworks_repo = Path(args["--frameworks-repo"]).resolve() framework_slug = args["<framework_slug>"] stage = args["<stage>"] lot = args["<lot>"] data_api_client = DataAPIClient(get_api_endpoint_from_stage(stage), get_auth_token('api', stage)) content_loader = ContentLoader(frameworks_repo) content_loader.load_manifest(framework_slug, "services", "services_search_filters") manifest = content_loader.get_manifest(framework_slug, "services_search_filters") # FIXME there isn't a uniform way to get the lots from the framework # content repo, hard code for G-Cloud for now framework_lots = [ {"name": "Cloud hosting", "slug": "cloud-hosting"}, {"name": "Cloud software", "slug": "cloud-software"}, {"name": "Cloud support", "slug": "cloud-support"}, ] writer = csv.writer(sys.stdout) # do the thing writer.writerow(['serviceId', 'topLevelCategories']) for service in data_api_client.find_services_iter(framework=framework_slug, status='published', lot=lot): service_categories = service['serviceCategories'] if service.get('serviceCategories') else [] top_level_categories = [] for f in filters_for_lot(service['lot'], manifest, framework_lots)['categories']['filters']: children = [f['label'] for f in f['children']] if f.get('children') else [] if any(item in service_categories for item in children): top_level_categories.append(f['label']) writer.writerow([service['id'], '; '.join(top_level_categories)])
logger = logging.getLogger("script") from dmscripts.helpers.auth_helpers import get_auth_token from dmscripts.helpers import logging_helpers from dmutils.env_helpers import get_api_endpoint_from_stage from dmscripts.data_retention_remove_supplier_declarations import remove_unsuccessful_supplier_declarations if __name__ == "__main__": arguments = docopt(__doc__) # Get script arguments stage = arguments['<stage>'] dry_run = arguments['--dry-run'] framework = arguments['<framework-slug>'] verbose = arguments['--verbose'] user = arguments['<user>'] or getpass.getuser() # Set defaults, instantiate clients logging_helpers.configure_logger({"dmapiclient": logging.INFO} if verbose else {"dmapiclient": logging.WARN}) data_api_client = DataAPIClient( base_url=get_api_endpoint_from_stage(stage), auth_token=get_auth_token('api', stage)) remove_unsuccessful_supplier_declarations(data_api_client=data_api_client, logger=logger, dry_run=dry_run, framework_slug=framework, user=user)
Usage: scripts/get_active_users_csv.py <stage> Example scripts/get_active_users_csv.py preview > output.csv """ import csv import sys sys.path.insert(0, '.') from dmscripts.helpers.auth_helpers import get_auth_token from dmutils.env_helpers import get_api_endpoint_from_stage from docopt import docopt from dmapiclient import DataAPIClient if __name__ == '__main__': arguments = docopt(__doc__) stage = arguments['<stage>'] data_api_client = DataAPIClient(get_api_endpoint_from_stage(stage), get_auth_token('api', stage)) writer = csv.writer(sys.stdout) writer.writerow(['email address']) for user in filter( lambda u: u['active'], data_api_client.find_users_iter(personal_data_removed=False)): writer.writerow([user['emailAddress']])
from docopt import docopt from dmscripts.export_framework_applicant_details import get_csv_rows from dmscripts.helpers.auth_helpers import get_auth_token from dmscripts.helpers.framework_helpers import find_suppliers_with_details_and_draft_service_counts from dmscripts.helpers.supplier_data_helpers import get_supplier_ids_from_file from dmscripts.generate_framework_agreement_signature_pages import ( render_html_for_suppliers_awaiting_countersignature, render_pdf_for_each_html_page) from dmapiclient import DataAPIClient from dmutils.env_helpers import get_api_endpoint_from_stage if __name__ == '__main__': args = docopt(__doc__) framework_slug = args['<framework_slug>'] client = DataAPIClient(get_api_endpoint_from_stage(args['<stage>']), get_auth_token('api', args['<stage>'])) framework = client.get_framework(framework_slug)['frameworks'] framework_lot_slugs = tuple([ lot['slug'] for lot in client.get_framework(framework_slug)['frameworks']['lots'] ]) supplier_id_file = args['<supplier_id_file>'] supplier_ids = get_supplier_ids_from_file(supplier_id_file) html_dir = tempfile.mkdtemp() records = find_suppliers_with_details_and_draft_service_counts( client, framework_slug, supplier_ids) headers, rows = get_csv_rows(records, framework_slug,
if __name__ == '__main__': arguments = docopt(__doc__) supplier_ids = get_supplier_ids_from_args(arguments) STAGE = arguments['<stage>'] FRAMEWORK_SLUG = arguments['<framework>'] GOVUK_NOTIFY_API_KEY = arguments['<notify_api_key>'] GOVUK_NOTIFY_TEMPLATE_ID = arguments['<notify_template_id>'] CONTENT_PATH = arguments['<content_path>'] DRY_RUN = arguments['--dry-run'] content_loader = ContentLoader(CONTENT_PATH) content_loader.load_messages(FRAMEWORK_SLUG, ['e-signature']) mail_client = scripts_notify_client(GOVUK_NOTIFY_API_KEY, logger=logger) api_client = DataAPIClient(base_url=get_api_endpoint_from_stage(STAGE), auth_token=get_auth_token('api', STAGE)) context_helper = SuccessfulSupplierContextForNotify( api_client, FRAMEWORK_SLUG, supplier_ids=supplier_ids, logger=logger) context_helper.populate_data() context_data = context_helper.get_users_personalisations() framework = api_client.get_framework(FRAMEWORK_SLUG).get('frameworks') prefix = "[Dry Run] " if DRY_RUN else "" # Add in any framework-specific dates etc here extra_template_context = { "contract_title": content_loader.get_message(FRAMEWORK_SLUG, 'e-signature', 'framework_contract_title'),
service_id)) except Exception as e: if e.message == "Cannot re-publish a submitted service": print(u" > Draft {} already published".format(draft['id'])) else: print(u" > ERROR MIGRATING DRAFT {} - {}".format( draft['id'], e.message)) if __name__ == "__main__": arguments = docopt(__doc__) STAGE = arguments['<stage>'] DRY_RUN = arguments['--dry-run'] FRAMEWORK_SLUG = arguments['<framework_slug>'] api_url = get_api_endpoint_from_stage(STAGE) client = DataAPIClient(api_url, get_auth_token('api', STAGE)) print("Finding suppliers...") suppliers = find_suppliers_on_framework(client, FRAMEWORK_SLUG) print("Migrating drafts...") for supplier in suppliers: print(u"Migrating drafts for supplier {} - {}".format( supplier['supplierId'], supplier['supplierName'])) draft_services = get_submitted_drafts(client, FRAMEWORK_SLUG, supplier['supplierId']) for draft_service in draft_services: make_draft_service_live(client, draft_service, DRY_RUN)
import sys from dmapiclient import DataAPIClient from dmutils.env_helpers import get_api_endpoint_from_stage from docopt import docopt sys.path.insert(0, ".") from dmscripts.helpers.auth_helpers import get_auth_token from dmscripts.helpers.updated_by_helpers import get_user if __name__ == "__main__": args = docopt(__doc__) api_client = DataAPIClient( get_api_endpoint_from_stage(args["<stage>"]), get_auth_token("api", args["<stage>"]), user=get_user(), ) with open(args["<input_file>"]) as input_file: services = list(csv.DictReader(input_file)) missing_services = [s for s in services if not s["serviceId"]] for service in missing_services: name = service["Service Name"].replace(".csv", "")[:100] supplier_id = service["Supplier ID"] lot = { "Software": "cloud-software", "Support": "cloud-support",
def test_get_api_endpoint_for_dev_environments(self, stage): assert get_api_endpoint_from_stage(stage) == 'http://localhost:5000'
def test_get_antivirus_api_endpoint_for_dev_environments(self, stage): assert get_api_endpoint_from_stage(stage, app='antivirus-api') == 'http://localhost:5008'
def test_get_api_endpoint_for_non_dev_environments(self, stage, expected_result): assert get_api_endpoint_from_stage(stage) == expected_result
""" import sys from docopt import docopt sys.path.insert(0, '.') from dmscripts.index_to_search_service import do_index from dmscripts.helpers.auth_helpers import get_auth_token from dmutils.env_helpers import get_api_endpoint_from_stage if __name__ == "__main__": arguments = docopt(__doc__) ok = do_index( doc_type=arguments['<doc-type>'], data_api_url=arguments['--api-url'] or get_api_endpoint_from_stage(arguments['<stage>'], 'api'), data_api_access_token=arguments['--api-token'] or get_auth_token('api', arguments['<stage>']), search_api_url=arguments['--search-api-url'] or get_api_endpoint_from_stage(arguments['<stage>'], 'search-api'), search_api_access_token=arguments['--search-api-token'] or get_auth_token('search_api', arguments['<stage>']), mapping=arguments.get('--create-with-mapping'), serial=arguments['--serial'], index=arguments['--index'], frameworks=arguments['--frameworks'], ) if not ok: sys.exit(1)
./scripts/notify-suppliers-of-new-questions-answers.py preview notify-token --dry-run --supplier-ids=2,3,4 """ import sys from docopt import docopt sys.path.insert(0, '.') from dmscripts.helpers.auth_helpers import get_auth_token from dmscripts.notify_suppliers_of_new_questions_answers import main from dmutils.env_helpers import get_api_endpoint_from_stage if __name__ == "__main__": arguments = docopt(__doc__) list_of_supplier_ids = [] if arguments['--supplier-ids']: list_of_supplier_ids = list( map(int, arguments['--supplier-ids'].split(','))) ok = main(data_api_url=get_api_endpoint_from_stage(arguments['<stage>'], 'api'), data_api_token=get_auth_token('api', arguments['<stage>']), email_api_key=arguments['<notify_api_key>'], stage=arguments['<stage>'], dry_run=arguments['--dry-run'], supplier_ids=list_of_supplier_ids) if not ok: sys.exit(1)
from dmscripts.helpers.auth_helpers import get_auth_token from dmapiclient import DataAPIClient from dmscripts.mark_definite_framework_results import mark_definite_framework_results from dmscripts.helpers.logging_helpers import configure_logger from dmscripts.helpers.logging_helpers import INFO as loglevel_INFO, DEBUG as loglevel_DEBUG from dmscripts.helpers.supplier_data_helpers import get_supplier_ids_from_file from dmutils.env_helpers import get_api_endpoint_from_stage if __name__ == "__main__": from docopt import docopt args = docopt(__doc__) client = DataAPIClient(get_api_endpoint_from_stage(args["<stage>"], "api"), get_auth_token('api', args['<stage>'])) updated_by = args["--updated-by"] or getpass.getuser() declaration_definite_pass_schema = json.load(open(args["<declaration_definite_pass_schema_path>"], "r")) declaration_baseline_schema = \ (declaration_definite_pass_schema.get("definitions") or {}).get("baseline") service_schema = json.load( open(args["<draft_service_schema_path>"], "r") ) if args["<draft_service_schema_path>"] else None supplier_id_file = args["--supplier-id-file"] supplier_ids = get_supplier_ids_from_file(supplier_id_file) configure_logger({"script": loglevel_DEBUG if args["--verbose"] else loglevel_INFO})
STAGE = arguments['<stage>'] CONTENT_PATH = arguments['<content_path>'] FRAMEWORK_SLUG = arguments['<framework_slug>'] OUTPUT_DIR = arguments['--output-dir'] verbose = arguments['--verbose'] logger = logging_helpers.configure_logger( {"dmapiclient": logging.INFO} if verbose else {"dmapiclient": logging.WARN} ) if not os.path.exists(OUTPUT_DIR): logger.info("Creating {} directory".format(OUTPUT_DIR)) os.makedirs(OUTPUT_DIR) client = DataAPIClient(get_api_endpoint_from_stage(STAGE), get_auth_token('api', STAGE)) content_loader = ContentLoader(CONTENT_PATH) content_loader.load_manifest(FRAMEWORK_SLUG, "services", "edit_submission") content_manifest = content_loader.get_manifest(FRAMEWORK_SLUG, "edit_submission") capabilities = get_team_capabilities(content_manifest) locations = get_outcomes_locations(content_manifest) pool = ThreadPool(3) logger.info(f"Finding suppliers for Digital Outcomes on {FRAMEWORK_SLUG}") suppliers = find_all_outcomes(client, map_impl=pool.imap) logger.info(f"Building CSV for {len(suppliers)} Digital Outcomes suppliers") write_csv_with_make_row(
def test_get_search_api_endpoint_for_dev_environments(self, stage): assert get_api_endpoint_from_stage(stage, app='search-api') == 'http://localhost:5001'
args.env = args.env.lower() if args.env not in ['dev', 'development', 'local', 'preview', 'staging']: print( "This script can only be run against dev/preview/staging environments." ) sys.exit(1) args.lots = args.lots.lower().split(',') if set(args.lots) - set(LOTS_WHITELIST): print( "This script only allows the following lots: {}. If you need other lots, please add them to the " "whitelist (this is just a sanity-check against typos).".format( LOTS_WHITELIST)) sys.exit(1) data_api_url = get_api_endpoint_from_stage(args.env, 'api') data_api_client = DataAPIClient(data_api_url, args.data_api_token) search_api_url = get_api_endpoint_from_stage(args.env, 'search-api') search_api_client = SearchAPIClient(search_api_url, args.search_api_token) services_generated = 0 gcloud_service_faker = JsonSchemaGCloudServiceFaker() email_address = "*****@*****.**" identity = 'generate-g-cloud-services script ({})'.format( getpass.getuser()) filepath_declaration_validator = 'schemas/{}.declaration.json'.format( args.new_slug) # 0) Store the current framework state so that it can be restored at the end. current_framework_state = data_api_client.get_framework(
def test_get_antivirus_api_endpoint_for_non_dev_environments(self, stage, expected_result): assert get_api_endpoint_from_stage(stage, app='antivirus-api') == expected_result
""" import sys sys.path.insert(0, '.') from os import environ from sys import exit from dmapiclient import DataAPIClient from dmscripts.helpers.auth_helpers import get_auth_token from dmutils.env_helpers import get_api_endpoint_from_stage if __name__ == "__main__": data_api_client = DataAPIClient( get_api_endpoint_from_stage(environ["STAGE"].lower()), get_auth_token('api', environ["STAGE"].lower()), ) email_address = environ["ACCOUNT_EMAIL"] user = data_api_client.get_user(email_address=email_address) if not user: print(f"User {email_address!r} not found") exit(2) if not data_api_client.update_user_password( user["users"]["id"], environ["ACCOUNT_PASSWORD"], "set-dm-password-by-email.py", ): print(f"Failed to set password for {email_address!r}")
from dmscripts.helpers.auth_helpers import get_auth_token from dmscripts.helpers import logging_helpers from dmscripts.notify_suppliers_of_framework_application_event import \ notify_suppliers_of_framework_application_event if __name__ == "__main__": arguments = docopt(__doc__) logger = logging_helpers.configure_logger({"dmapiclient": logging.INFO}) run_id = None if not arguments.get("--resume-run-id") else UUID( arguments["--resume-run-id"]) failure_count = notify_suppliers_of_framework_application_event( data_api_client=DataAPIClient( base_url=get_api_endpoint_from_stage(arguments["<stage>"], "api"), auth_token=get_auth_token("api", arguments["<stage>"]), ), notify_client=scripts_notify_client( arguments['<govuk_notify_api_key>'], logger=logger), notify_template_id=arguments['<govuk_notify_template_id>'], framework_slug=arguments["<framework_slug>"], stage=arguments["<stage>"], dry_run=arguments["--dry-run"], logger=logger, run_id=run_id, ) if failure_count: logger.error("Failed sending {failure_count} messages", extra={"failure_count": failure_count})
if __name__ == "__main__": args = docopt(__doc__) logging.basicConfig(level=logging.INFO) FT_JOB_NAMES = ["functional-tests-preview", "functional-tests-staging"] API_USER = os.getenv("DM_JENKINS_API_USER") API_TOKEN = os.getenv("DM_JENKINS_API_TOKEN") OUTPUT_FILE = args.get("<file>") or "functional_test_report.csv" auth = HTTPBasicAuth(API_USER, API_TOKEN) # Use staging to get the framework dates because it'll be the same as production api_client = DataAPIClient(get_api_endpoint_from_stage("staging"), get_auth_token("api", "staging")) frameworks = api_client.find_frameworks()["frameworks"] build_data = [] for job in FT_JOB_NAMES: for build in get_job_build_data(job, auth): build_data.append(format_build(job, build, frameworks)) logging.info(f"Writing report to {OUTPUT_FILE}") headers = build_data[0].keys() with open(OUTPUT_FILE, "w") as f: writer = csv.DictWriter(f, headers) writer.writeheader() writer.writerows(build_data)
) for lot in lots: lot.update({"list_id": "096e52cebb"}) # Override list id if arguments.get("--list_id"): for lot in lots: lot.update({"list_id": arguments["--list_id"]}) # Override lot if arguments.get("--lot_slug"): lots = [ lot for lot in lots if lot["lot_slug"] == arguments["--lot_slug"] ] api_url = get_api_endpoint_from_stage(arguments['<stage>']) data_api_client = DataAPIClient( api_url, get_auth_token('api', arguments['<stage>'])) dm_mailchimp_client = DMMailChimpClient(arguments['<mailchimp_username>'], arguments['<mailchimp_api_key>'], logger) for lot_data in lots: ok = main( data_api_client=data_api_client, mailchimp_client=dm_mailchimp_client, lot_data=lot_data, number_of_days=number_of_days, framework_slug=framework_slug, )
("supplier_name", record["supplier"]["name"]), ("supplier_declaration_name", record["declaration"].get("supplierRegisteredName", "")), ("status", "PASSED" if record["onFramework"] else "FAILED"), ] return row + make_fields_from_content_questions(questions, record) return inner if __name__ == '__main__': arguments = docopt(__doc__) STAGE = arguments['<stage>'] CONTENT_PATH = arguments['<content_path>'] FRAMEWORK_SLUG = arguments['<framework_slug>'] client = DataAPIClient(get_api_endpoint_from_stage(STAGE), get_auth_token('api', STAGE)) content_loader = ContentLoader(CONTENT_PATH) content_loader.load_manifest(FRAMEWORK_SLUG, "services", "edit_submission") content_manifest = content_loader.get_manifest(FRAMEWORK_SLUG, "edit_submission") records = find_all_participants(client) write_csv_with_make_row( records, make_row(content_manifest), "output/{}-user-research-participants.csv".format(FRAMEWORK_SLUG))