def test_get_all_files_of_type_for_flat_folder(): temp_folder_path = tempfile.mkdtemp() pdf1 = open(os.path.join(temp_folder_path, 'test1.pdf'), 'w+') pdf2 = open(os.path.join(temp_folder_path, 'test2.pdf'), 'w+') assert len(list(get_all_files_of_type(temp_folder_path, 'pdf'))) == 2 pdf1.close() pdf2.close() shutil.rmtree(temp_folder_path)
def test_get_all_files_of_type_for_flat_folder(): temp_folder_path = tempfile.mkdtemp() pdf1 = open(os.path.join(temp_folder_path, 'test1.pdf'), 'w+') pdf2 = open(os.path.join(temp_folder_path, 'test2.pdf'), 'w+') assert len(list(get_all_files_of_type(temp_folder_path, 'pdf'))) == 2 pdf1.close() pdf2.close() shutil.rmtree(temp_folder_path)
""" import sys sys.path.insert(0, '.') from dmscripts.env import get_api_endpoint_from_stage from dmscripts.bulk_upload_documents import get_bucket_name, get_all_files_of_type from dmscripts.upload_counterpart_agreements import upload_counterpart_file from docopt import docopt from dmapiclient import DataAPIClient from dmutils.s3 import S3 if __name__ == '__main__': arguments = docopt(__doc__) stage = arguments['<stage>'] client = DataAPIClient(get_api_endpoint_from_stage(stage), arguments['<api_token>']) framework_slug = arguments['<framework_slug>'] document_directory = arguments['<documents_directory>'] dry_run = arguments['--dry-run'] if dry_run: bucket = None else: bucket = S3(get_bucket_name(stage, "agreements")) for file_path in get_all_files_of_type(document_directory, "pdf"): upload_counterpart_file(bucket, framework_slug, file_path, dry_run, client)
) from docopt import docopt from dmutils.s3 import S3 if __name__ == "__main__": arguments = docopt(__doc__) stage = arguments["<stage>"] framework_slug = arguments["<framework_slug>"] local_directory = arguments["<local_documents_directory>"] bucket_category = arguments["--bucket_category"] file_type = arguments["--file_type"] tsv_path = arguments["<tsv_path>"] dry_run = arguments["--dry-run"] if dry_run: bucket = None else: bucket = S3(get_bucket_name(stage, bucket_category)) supplier_name_dict = get_supplier_name_dict_from_tsv(tsv_path) for path in get_all_files_of_type(local_directory, file_type): try: upload_file(bucket, dry_run, path, framework_slug, bucket_category, supplier_name_dict=supplier_name_dict) except ValueError as e: print("SKIPPING: {}".format(e)) continue
get_auth_token('api', stage)) framework = data_api_client.get_framework( arguments['<framework_slug>'])["frameworks"] document_directory = arguments['<documents_directory>'] dry_run = arguments['--dry-run'] dm_notify_client = arguments.get("--notify-key") and scripts_notify_client( arguments["--notify-key"], logger=logger) if dry_run: bucket = None else: bucket = S3(get_bucket_name(stage, "agreements")) failure_count = 0 for file_path in get_all_files_of_type(document_directory, "pdf"): try: upload_counterpart_file( bucket, framework, file_path, dry_run, data_api_client, dm_notify_client=dm_notify_client, notify_template_id=arguments.get("--notify-template-id"), notify_fail_early=False, logger=logger, ) except (OSError, IOError, S3ResponseError, EmailError, APIError) as e: # upload_counterpart_file should have already logged these so no need here failure_count += 1
stage = arguments['<stage>'] framework_slug = arguments['<framework_slug>'] local_directory = arguments['<local_documents_directory>'] bucket_category = arguments['--bucket_category'] document_category = arguments['<document_category>'] document_type = arguments['--document_type'] dry_run = arguments['--dry-run'] document_categories = ['result-letter', 'framework-agreement', 'signed-framework-agreement', 'countersigned-framework-agreement'] if document_category not in document_categories: print('Document needs to be one of {}'.format(document_categories)) sys.exit(1) if dry_run: bucket = None else: bucket = S3(get_bucket_name(stage, bucket_category)) for path in get_all_files_of_type(local_directory, document_type): try: upload_file(bucket, dry_run, path, framework_slug, bucket_category, document_category, document_type) except ValueError as e: print("SKIPPING: {}".format(e)) continue
framework_slug = arguments['<framework_slug>'] local_directory = arguments['<local_documents_directory>'] bucket_category = arguments['--bucket_category'] document_category = arguments['<document_category>'] document_type = arguments['--document_type'] dry_run = arguments['--dry-run'] document_categories = ['result-letter', 'framework-agreement'] if document_category not in document_categories: print('Document needs to be one of {}'.format(document_categories)) if document_category in ('signed-framework-agreement', 'countersigned-framework-agreement'): print( 'Signed and countersigned agreement paths now need to be stored in database so cannot be uploaded ' 'using this script.') sys.exit(1) if dry_run: bucket = None else: bucket = S3(get_bucket_name(stage, bucket_category)) for path in get_all_files_of_type(local_directory, document_type): try: upload_file(bucket, dry_run, path, framework_slug, bucket_category, document_category, document_type) except ValueError as e: print("SKIPPING: {}".format(e)) continue
from dmutils.s3 import S3 if __name__ == '__main__': arguments = docopt(__doc__) stage = arguments['<stage>'] framework_slug = arguments['<framework_slug>'] local_directory = arguments['<local_documents_directory>'] bucket_category = arguments['--bucket_category'] file_type = arguments['--file_type'] tsv_path = arguments['<tsv_path>'] dry_run = arguments['--dry-run'] if dry_run: bucket = None else: bucket = S3(get_bucket_name(stage, bucket_category)) supplier_name_dict = get_supplier_name_dict_from_tsv(tsv_path) for path in get_all_files_of_type(local_directory, file_type): try: upload_file(bucket, dry_run, path, framework_slug, bucket_category, supplier_name_dict=supplier_name_dict) except ValueError as e: print("SKIPPING: {}".format(e)) continue