from dmscripts.helpers.auth_helpers import get_auth_token
from dmscripts.helpers.framework_helpers import find_suppliers_with_details_and_draft_service_counts
from dmscripts.helpers.supplier_data_helpers import get_supplier_ids_from_file
from dmscripts.generate_framework_agreement_signature_pages import (
render_html_for_suppliers_awaiting_countersignature,
render_pdf_for_each_html_page)
from dmapiclient import DataAPIClient
from dmutils.env_helpers import get_api_endpoint_from_stage
if __name__ == '__main__':
args = docopt(__doc__)
framework_slug = args['<framework_slug>']
client = DataAPIClient(get_api_endpoint_from_stage(args['<stage>']),
get_auth_token('api', args['<stage>']))
framework = client.get_framework(framework_slug)['frameworks']
framework_lot_slugs = tuple([
lot['slug']
for lot in client.get_framework(framework_slug)['frameworks']['lots']
])
supplier_id_file = args['<supplier_id_file>']
supplier_ids = get_supplier_ids_from_file(supplier_id_file)
html_dir = tempfile.mkdtemp()
records = find_suppliers_with_details_and_draft_service_counts(
client, framework_slug, supplier_ids)
headers, rows = get_csv_rows(records,
framework_slug,
framework_lot_slugs,
count_statuses=("submitted", ))
map_impl = map
logger = configure_logger({
"dmapiclient.base": logging.WARNING,
"framework_helpers": logging.DEBUG if verbose >= 2 else logging.WARNING,
"script": logging.DEBUG if verbose else logging.INFO,
})
logger.debug(f"connecting to api on {stage}")
client = DataAPIClient(
get_api_endpoint_from_stage(args["<stage>"]),
get_auth_token("api", args["<stage>"]),
)
logger.debug(f"fetching lots for framework '{framework_slug}'")
framework = client.get_framework(framework_slug)["frameworks"]
suppliers = find_suppliers(client, framework, supplier_ids, map_impl, dry_run)
# create a temporary directory for the HTML files
with tempfile.TemporaryDirectory() as html_dir:
# create signature pages in HTML using Jinja templates from agreements repo
logger.debug(f"generating HTML signature pages")
render_html_for_successful_suppliers(
suppliers, framework, agreements_dir, html_dir, dry_run)
# convert HTML to PDF (this uses wkhtmltopdf under-the-hood)
logger.debug(f"generating PDF signature pages from HTML")
html_pages = os.listdir(html_dir)
html_pages.remove("framework-agreement-signature-page.css")
render_pdf_for_each_html_page(html_pages, html_dir, output_dir)
GOVUK_NOTIFY_API_KEY = arguments['<notify_api_key>']
GOVUK_NOTIFY_TEMPLATE_ID = arguments['<notify_template_id>']
CONTENT_PATH = arguments['<content_path>']
DRY_RUN = arguments['--dry-run']
content_loader = ContentLoader(CONTENT_PATH)
content_loader.load_messages(FRAMEWORK_SLUG, ['e-signature'])
mail_client = scripts_notify_client(GOVUK_NOTIFY_API_KEY, logger=logger)
api_client = DataAPIClient(base_url=get_api_endpoint_from_stage(STAGE),
auth_token=get_auth_token('api', STAGE))
context_helper = SuccessfulSupplierContextForNotify(
api_client, FRAMEWORK_SLUG, supplier_ids=supplier_ids, logger=logger)
context_helper.populate_data()
context_data = context_helper.get_users_personalisations()
framework = api_client.get_framework(FRAMEWORK_SLUG).get('frameworks')
prefix = "[Dry Run] " if DRY_RUN else ""
# Add in any framework-specific dates etc here
extra_template_context = {
"contract_title":
content_loader.get_message(FRAMEWORK_SLUG, 'e-signature',
'framework_contract_title'),
"intentionToAwardAt_dateformat":
nodaydateformat(framework['intentionToAwardAtUTC']),
"frameworkLiveAt_dateformat":
nodaydateformat(framework['frameworkLiveAtUTC'])
}
user_count = len(context_data)
logging.WARNING,
})
if __name__ == '__main__':
arguments = docopt(__doc__)
if bool(arguments.get("--notify-key")) != bool(
arguments.get("--notify-template-id")):
raise ValueError(
"Either specify both --notify-key and --notify-template-id or neither"
)
stage = arguments['<stage>']
data_api_client = DataAPIClient(get_api_endpoint_from_stage(stage),
get_auth_token('api', stage))
framework = data_api_client.get_framework(
arguments['<framework_slug>'])["frameworks"]
document_directory = arguments['<documents_directory>']
dry_run = arguments['--dry-run']
dm_notify_client = arguments.get("--notify-key") and scripts_notify_client(
arguments["--notify-key"], logger=logger)
if dry_run:
bucket = None
else:
bucket = S3(get_bucket_name(stage, "agreements"))
failure_count = 0
for file_path in get_all_files_of_type(document_directory, "pdf"):
try:
upload_counterpart_file(
data_api_url = get_api_endpoint_from_stage(args.env, 'api')
data_api_client = DataAPIClient(data_api_url, args.data_api_token)
search_api_url = get_api_endpoint_from_stage(args.env, 'search-api')
search_api_client = SearchAPIClient(search_api_url, args.search_api_token)
services_generated = 0
gcloud_service_faker = JsonSchemaGCloudServiceFaker()
email_address = "*****@*****.**"
identity = 'generate-g-cloud-services script ({})'.format(
getpass.getuser())
filepath_declaration_validator = 'schemas/{}.declaration.json'.format(
args.new_slug)
# 0) Store the current framework state so that it can be restored at the end.
current_framework_state = data_api_client.get_framework(
args.new_slug)['frameworks']['status']
# 1) Set a given (new) framework to open
data_api_client._post_with_updated_by(
url='{}/frameworks/{}'.format(data_api_url, args.new_slug),
data={"frameworks": {
"status": "open"
}},
user=identity)
# 2) Find some suppliers with services on a given (old) framework (e.g. g-cloud-7)
suppliers = data_api_client.find_framework_suppliers(
framework_slug=args.old_slug)['supplierFrameworks']
suppliers = [
x for x in suppliers if x['supplierId'] not in SUPPLIER_BLACKLIST
]
client = DataAPIClient(get_api_endpoint_from_stage(STAGE),
get_auth_token('api', STAGE))
now = datetime.datetime.now()
filename = FRAMEWORK + "-supplier-about-you-data-" + now.strftime(
"%Y-%m-%d_%H.%M-") + STAGE + ".csv"
filepath = OUTPUT_DIR + os.sep + filename
# Create output directory if it doesn't already exist
if not os.path.exists(os.path.dirname(filepath)):
try:
os.makedirs(os.path.dirname(filepath))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
framework_lot_slugs = tuple([
lot['slug']
for lot in client.get_framework(FRAMEWORK)['frameworks']['lots']
])
pool = ThreadPool(3)
export_supplier_details(client,
FRAMEWORK,
filepath,
framework_lot_slugs=framework_lot_slugs,
map_impl=pool.imap,
logger=logger)
NOTIFY_TEMPLATE_APPLICATION_MADE = 'de02a7e3-80f6-4391-818c-48326e1f4688'
NOTIFY_TEMPLATE_NO_APPLICATION = '87a126b4-7909-4b63-b981-d3c3d6a558ff'
if __name__ == '__main__':
arguments = docopt(__doc__)
STAGE = arguments['<stage>']
FRAMEWORK_SLUG = arguments['<framework_slug>']
GOVUK_NOTIFY_API_KEY = arguments['<govuk_notify_api_key>']
DRY_RUN = arguments['--dry-run']
mail_client = scripts_notify_client(GOVUK_NOTIFY_API_KEY, logger=logger)
api_client = DataAPIClient(base_url=get_api_endpoint_from_stage(STAGE),
auth_token=get_auth_token('api', STAGE))
framework_data = api_client.get_framework(FRAMEWORK_SLUG)['frameworks']
update_framework_with_formatted_dates(framework_data)
context_helper = AppliedToFrameworkSupplierContextForNotify(
api_client, FRAMEWORK_SLUG, framework_data['intentionToAwardAt'])
context_helper.populate_data()
context_data = context_helper.get_users_personalisations()
error_count = 0
for user_email, personalisation in context_data.items():
logger.info(user_email)
template_id = (NOTIFY_TEMPLATE_APPLICATION_MADE
if personalisation['applied'] else
NOTIFY_TEMPLATE_NO_APPLICATION)
if DRY_RUN:
logger.info("[Dry Run] Sending email {} to {}".format(
template_id, user_email))
def notify_suppliers_of_framework_application_event(
data_api_client: DataAPIClient,
notify_client: DMNotifyClient,
notify_template_id: str,
framework_slug: str,
stage: str,
dry_run: bool,
logger: Logger,
run_id: Optional[UUID] = None,
) -> int:
run_is_new = not run_id
run_id = run_id or uuid4()
logger.info(
f"{'Starting' if run_is_new else 'Resuming'} run id {{run_id}}",
extra={"run_id": str(run_id)})
framework = data_api_client.get_framework(framework_slug)["frameworks"]
framework_context = {
"framework_name":
framework["name"],
"updates_url":
f"{get_web_url_from_stage(stage)}/suppliers/frameworks/{framework['slug']}/updates",
"framework_dashboard_url":
f"{get_web_url_from_stage(stage)}/suppliers/frameworks/{framework['slug']}/",
"clarification_questions_closed":
"no" if framework["clarificationQuestionsOpen"] else "yes",
**_formatted_dates_from_framework(framework),
}
failure_count = 0
for supplier_framework in data_api_client.find_framework_suppliers_iter(
framework_slug):
for user in data_api_client.find_users_iter(
supplier_id=supplier_framework["supplierId"]):
if user["active"]:
# generating ref separately so we can exclude certain parameters from the context dict
notify_ref = notify_client.get_reference(
user["emailAddress"],
notify_template_id,
{
"framework_slug": framework["slug"],
"run_id": str(run_id),
},
)
if dry_run:
# Use the sent references cache unless we're re-running the script following a failure
if notify_client.has_been_sent(
notify_ref, use_recent_cache=run_is_new):
logger.debug(
"[DRY RUN] Would NOT send notification to {email_hash} (already sent)",
extra={
"email_hash": hash_string(user["emailAddress"])
},
)
else:
logger.info(
"[DRY RUN] Would send notification to {email_hash}",
extra={
"email_hash": hash_string(user["emailAddress"])
},
)
else:
try:
# Use the sent references cache unless we're re-running the script following a failure
notify_client.send_email(
user["emailAddress"],
notify_template_id,
framework_context,
allow_resend=False,
reference=notify_ref,
use_recent_cache=run_is_new,
)
except EmailError as e:
failure_count += 1
logger.error(
"Failed sending to {email_hash}: {e}",
extra={
"email_hash":
hash_string(user["emailAddress"]),
"e": str(e),
},
)
if isinstance(e, EmailTemplateError):
raise # do not try to continue
return failure_count
parser.add_argument(
'stage',
default='development',
choices=['development', 'preview', 'staging', 'production'],
help="Which stage's API to communicate with.")
args = parser.parse_args()
FILENAME = f'{args.framework_slug}-all-successful-suppliers.csv'
client = DataAPIClient(base_url=get_api_endpoint_from_stage(args.stage),
auth_token=get_auth_token('api', args.stage))
logger.info('Retrieving framework ...')
all_lot_names = [
lot['name'] for lot in client.get_framework(args.framework_slug)
['frameworks']['lots']
]
map_suppliers_to_lots = {}
logger.info('Looking up supplier services ...')
all_framework_services = client.find_services_iter(
framework=args.framework_slug)
for service in all_framework_services:
if service['supplierId'] not in map_suppliers_to_lots:
map_suppliers_to_lots[service['supplierId']] = {
'name': service['supplierName'],
'lots': set()
}
map_suppliers_to_lots[service['supplierId']]['lots'].add(
service['lotName'])
})
if __name__ == '__main__':
arguments = docopt(__doc__)
if bool(arguments.get("--notify-key")) != bool(
arguments.get("--notify-template-id")):
raise ValueError(
"Either specify both --notify-key and --notify-template-id or neither"
)
stage = arguments['<stage>']
data_api_client = DataAPIClient(get_api_endpoint_from_stage(stage),
get_auth_token('api', stage))
framework_slug = arguments['<framework_slug>']
framework = data_api_client.get_framework(framework_slug)["frameworks"]
document_directory = arguments['<documents_directory>']
content_path = arguments['<content_path>']
content_loader = ContentLoader(content_path)
if framework['isESignatureSupported']:
content_loader.load_messages(framework_slug, ['e-signature'])
contract_title = content_loader.get_message(
framework_slug, 'e-signature', 'framework_contract_title')
else:
contract_title = 'Framework Agreement'
dry_run = arguments['--dry-run']
dm_notify_client = arguments.get("--notify-key") and scripts_notify_client(
arguments["--notify-key"], logger=logger)
if dry_run:
return None
else:
raise
if __name__ == '__main__':
arguments = docopt(__doc__)
configure_logger({"script": loglevel_INFO})
client = DataAPIClient(get_api_endpoint_from_stage(arguments['<stage>']), arguments['<data_api_token>'])
user = arguments['<user>'] or getpass.getuser()
framework_slugs = arguments.get("<framework_slugs>") and arguments["<framework_slugs>"].split(",")
dry_run = bool(arguments.get("--dry-run"))
frameworks = tuple(client.get_framework(framework_slug)["frameworks"] for framework_slug in framework_slugs)
logger.info("Inspecting framework declarations in this order: %s", ", ".join(fw["slug"] for fw in frameworks))
for supplier in client.find_suppliers_iter():
logger.info("Processing supplier %s", supplier["id"])
if supplier.get("tradingStatus") and (supplier["contactInformation"][0]).get("address2") == "":
logger.info(" already done: {}".format(supplier["id"]))
continue
try:
supplier_framework = next(
sfr["frameworkInterest"] for sfr in (
_catch_404_none(
lambda: client.get_supplier_framework_info(supplier["id"], framework["slug"])
) for framework in frameworks
) if sfr and sfr["frameworkInterest"]["onFramework"]
"dmapiclient": "INFO",
"dmscripts.generate_framework_agreement_signature_pages": "DEBUG",
"framework_helpers": "DEBUG",
"script": "DEBUG",
})
else:
logger = configure_logger({
"dmapiclient": "WARN",
"dmscripts.generate_framework_agreement_signature_pages": "INFO",
"framework_helpers": "INFO",
"script": "INFO",
})
framework_slug = args['<framework_slug>']
client = DataAPIClient(get_api_endpoint_from_stage(args['<stage>']), get_auth_token('api', args['<stage>']))
framework = client.get_framework(framework_slug)['frameworks']
framework_lot_slugs = tuple([lot['slug'] for lot in client.get_framework(framework_slug)['frameworks']['lots']])
supplier_ids = get_supplier_ids_from_args(args)
html_dir = tempfile.mkdtemp()
records = find_suppliers_with_signed_framework_agreements(client, framework_slug, supplier_ids)
if framework_supports_e_signature(framework):
logger.info(
f"Framework {framework_slug} supports e-signatures, unapproved agreements will be automatically approved"
f" and suspended services unsuspended"
)
def approve_supplier_framework_agreement(record):
if not record["countersignedAt"]:
