def export_dos_opportunities( client, logger, stage: str, output_dir, dry_run: bool = False ): output_dir = Path(output_dir) if not output_dir.exists(): logger.info(f"Creating {output_dir} directory") output_dir.mkdir(parents=True) latest_framework_slug = get_latest_dos_framework(client) communications_bucket = S3(get_bucket_name(stage, "communications")) reports_bucket = S3(get_bucket_name(stage, "reports")) logger.info("Exporting DOS opportunity data to CSV") # Get the data rows = get_brief_data(client, logger, include_buyer_user_details=True) # Construct CSV for admins write_rows_to_csv(rows, output_dir / "opportunity-data-for-admins.csv", logger) # Construct public CSV (filter out buyer details) write_rows_to_csv( [ OrderedDict((k, v) for k, v in row.items() if k in DOS_OPPORTUNITY_HEADERS) for row in rows ], output_dir / DOWNLOAD_FILE_NAME, logger ) # Upload admin CSV to reports bucket upload_file_to_s3( output_dir / "opportunity-data-for-admins.csv", reports_bucket, f"{latest_framework_slug}/reports/{DOWNLOAD_FILE_NAME}", DOWNLOAD_FILE_NAME, public=False, dry_run=dry_run, logger=logger ) # Upload public CSV to S3 upload_file_to_s3( output_dir / DOWNLOAD_FILE_NAME, communications_bucket, f"{latest_framework_slug}/communications/data/{DOWNLOAD_FILE_NAME}", DOWNLOAD_FILE_NAME, public=True, dry_run=dry_run, logger=logger )
def test_content_type_detection(self): # File extensions allowed for G6 documents: pdf, odt, ods, odp test_type = S3('test-bucket')._get_mimetype('test-file.pdf') self.assertEqual(test_type, 'application/pdf') test_type = S3('test-bucket')._get_mimetype('test-file.odt') self.assertEqual(test_type, 'application/vnd.oasis.opendocument.text') test_type = S3('test-bucket')._get_mimetype('test-file.ods') self.assertEqual(test_type, 'application/vnd.oasis.opendocument.spreadsheet') test_type = S3('test-bucket')._get_mimetype('test-file.odp') self.assertEqual(test_type, 'application/vnd.oasis.opendocument.presentation')
def test_endpoint_url_config_not_set_in_development(self, boto3): app = flask.Flask("test_endpoint_url_from_flask_config") app.env = "development" with app.app_context(): S3("bucket") assert "endpoint_url" not in boto3.resource.call_args[1]
def test_delete_key(self): mock_bucket = FakeBucket(['folder/test-file.pdf']) self.s3_mock.get_bucket.return_value = mock_bucket S3('test-bucket').delete_key('folder/test-file.pdf') assert 'folder/test-file.pdf' not in mock_bucket.keys
def test_delete_key_moves_file_with_prefix(self): mock_bucket = FakeBucket(['folder/test-file.pdf']) self.s3_mock.get_bucket.return_value = mock_bucket S3('test-bucket').delete_key('folder/test-file.pdf') assert 'folder/2015-10-10T00:00:00-test-file.pdf' in mock_bucket.keys
def test_get_signed_url_with_expires_at(self, bucket_with_file): signed_url = S3('dear-liza').get_signed_url('with/straw.dear.pdf', expires_in=10) parsed_signed_url = urlparse(signed_url) parsed_qs = parse_qs(parsed_signed_url.query) assert parsed_qs["Expires"] == ["1444435210"]
def test_save_file(self, empty_bucket, path, expected_path, expected_ct, expected_filename, expected_ext, timestamp, expected_timestamp, download_filename, expected_cd): returned_key_dict = S3("dear-liza").save( path, file_=BytesIO(b"one two three"), timestamp=timestamp, download_filename=download_filename, ) assert returned_key_dict == { "path": expected_path, "filename": expected_filename, "ext": expected_ext, "last_modified": expected_timestamp, "size": 13, } summary_list = list(empty_bucket.objects.all()) assert len(summary_list) == 1 assert summary_list[0].key == expected_path obj0 = summary_list[0].Object() assert obj0.metadata == { "timestamp": expected_timestamp, } assert obj0.content_disposition == expected_cd assert obj0.get()["Body"].read() == b"one two three" if sys.version_info < (3, 0): # moto currently has a py3 bug which makes this fail - the fix not yet upstream - perhaps next time you come # across this message try updating moto to the latest version and see if this works assert obj0.content_type == expected_ct
def test_save_existing_file(self, bucket_with_file): returned_key_dict = S3("dear-liza").save( "with/straw.dear.pdf", file_=BytesIO(b"significantly longer contents than before"), download_filename="significantly_different.pdf", ) assert returned_key_dict == { "path": "with/straw.dear.pdf", "filename": "straw.dear", "ext": "pdf", "last_modified": "2014-10-20T00:00:00.000000Z", "size": 41, } summary_list = list(bucket_with_file.objects.all()) assert len(summary_list) == 1 assert summary_list[0].key == "with/straw.dear.pdf" obj0 = summary_list[0].Object() assert obj0.metadata == { "timestamp": "2014-10-20T00:00:00.000000Z", } assert obj0.content_disposition == 'attachment; filename="significantly_different.pdf"' assert obj0.get()["Body"].read( ) == b"significantly longer contents than before" if sys.version_info < (3, 0): # moto currently has a py3 bug which makes this fail - the fix not yet upstream - perhaps next time you come # across this message try updating moto to the latest version and see if this works assert obj0.content_type == "application/pdf"
def test_endpoint_url_ignored_in_production(self, boto3): app = flask.Flask("test_endpoint_url_from_flask_config") app.env = "production" app.config["DM_S3_ENDPOINT_URL"] = "http://localhost:5100" with app.app_context(): S3("bucket") assert "endpoint_url" not in boto3.resource.call_args[1]
def test_endpoint_url_config_none_in_development(self, boto3): app = flask.Flask("test_endpoint_url_from_flask_config") app.env = "development" app.config["DM_S3_ENDPOINT_URL"] = None with app.app_context(): S3("bucket") assert "endpoint_url" not in boto3.resource.call_args[1]
def test_save_sets_timestamp_to_current_time(self): mock_bucket = FakeBucket() self.s3_mock.get_bucket.return_value = mock_bucket S3('test-bucket').save('folder/test-file.pdf', mock_file('blah', 123)) mock_bucket.s3_key_mock.set_metadata.assert_called_once_with( 'timestamp', "2015-10-10T00:00:00.000000Z")
def main(stage): drafts_bucket_name = 'digitalmarketplace-g7-draft-documents-{0}-{0}'.format( stage) communications_bucket_name = 'digitalmarketplace-communications-{0}-{0}'.format( stage) drafts_bucket = S3(drafts_bucket_name) communications_bucket = S3(communications_bucket_name) copy_file = make_copier(drafts_bucket, communications_bucket) copy_file("g-cloud-7-supplier-pack.zip", "g-cloud-7/communications/g-cloud-7-supplier-pack.zip") for file in drafts_bucket.list('g-cloud-7-updates'): newpath = "g-cloud-7/communications/updates/{}".format( re.sub(r'^g-cloud-7-updates/', '', file['path'])) copy_file(file['path'], newpath)
def test_get_key(self, bucket_with_file): assert S3('dear-liza').get_key('with/straw.dear.pdf') == { "path": "with/straw.dear.pdf", "filename": "straw.dear", "ext": "pdf", "size": 12, "last_modified": "2005-04-03T02:01:00.000000Z", }
def test_list_files(self): mock_bucket = mock.Mock() self.s3_mock.get_bucket.return_value = mock_bucket fake_key = FakeKey('dir/file 1.odt') mock_bucket.list.return_value = [fake_key] expected = [fake_key.fake_format_key(filename='file 1', ext='odt')] self.assertEqual(S3('test-bucket').list(), expected)
def test_endpoint_url_from_flask_config(self, boto3): app = flask.Flask("test_endpoint_url_from_flask_config") app.env = "development" app.config["DM_S3_ENDPOINT_URL"] = "http://localhost:5100" with app.app_context(): S3("bucket") assert boto3.resource.call_args[1][ "endpoint_url"] == "http://localhost:5100"
def test_move_existing_doesnt_delete_file(self): mock_bucket = FakeBucket(['folder/test-file.odt']) self.s3_mock.get_bucket.return_value = mock_bucket S3('test-bucket')._move_existing(existing_path='folder/test-file.odt', move_prefix='OLD') self.assertEqual(mock_bucket.keys, {'folder/test-file.odt', 'folder/OLD-test-file.odt'})
def test_bucket_short_name_invalid_format(self): bad_bucket_names = [ 'something-invalid', 'digitalmarketplace-something-environ-environ-other', 'digitalmarketplace-something-environ-other', ] for bad_bucket_name in bad_bucket_names: with pytest.raises(ValueError): S3(bad_bucket_name).bucket_short_name
def test_get_key_weird_file(self, bucket_with_weird_file): assert S3('dear-liza').get_key('.!!!.dear.pdf') == { "ext": "pdf", "filename": ".!!!.dear", "last_modified": bucket_with_weird_file["expected_returned_timestamp"], "path": ".!!!.dear.pdf", "size": 26, }
def test_get_signed_url(self, bucket_with_file): signed_url = S3('dear-liza').get_signed_url('with/straw.dear.pdf') parsed_signed_url = urlparse(signed_url) # to an extent the url format should be opaque and up to amazon so we might have to rethink these assertions if # anything changes assert "dear-liza" in parsed_signed_url.hostname assert parsed_signed_url.path == "/with/straw.dear.pdf" parsed_qs = parse_qs(parsed_signed_url.query) assert parsed_qs["AWSAccessKeyId"] == ["AKIAIABCDABCDABCDABC"] assert parsed_qs["Signature"]
def test_copy_existing_file(self, bucket_with_file): target_key = "copy/straw.dear.pdf" returned_key_dict = S3("dear-liza").copy(src_bucket="dear-liza", src_key="with/straw.dear.pdf", target_key=target_key) assert returned_key_dict['path'] == target_key assert returned_key_dict['size'] == 12 assert returned_key_dict[ 'last_modified'] == "2005-04-03T02:01:00.000000Z"
def test_save_sets_content_type_and_acl(self): mock_bucket = FakeBucket() self.s3_mock.get_bucket.return_value = mock_bucket S3('test-bucket').save('folder/test-file.pdf', mock_file('blah', 123)) self.assertEqual(mock_bucket.keys, set(['folder/test-file.pdf'])) mock_bucket.s3_key_mock.set_contents_from_file.assert_called_with( mock.ANY, headers={'Content-Type': 'application/pdf'}) mock_bucket.s3_key_mock.set_acl.assert_called_with('public-read')
def test_save_existing_file(self): mock_bucket = FakeBucket(['folder/test-file.pdf']) self.s3_mock.get_bucket.return_value = mock_bucket S3('test-bucket').save('folder/test-file.pdf', mock_file('blah', 123), move_prefix='OLD') self.assertEqual(mock_bucket.keys, {'folder/test-file.pdf', 'folder/OLD-test-file.pdf'})
def test_get_key(self): mock_bucket = mock.Mock() self.s3_mock.get_bucket.return_value = mock_bucket fake_key = FakeKey('dir/file1.pdf') mock_bucket.get_key.return_value = fake_key assert S3('test-bucket').get_key( 'dir/file1.pdf') == fake_key.fake_format_key(filename='file1', ext='pdf')
def test_list_files_with_loading_custom_timestamps(self): mock_bucket = mock.Mock() self.s3_mock.get_bucket.return_value = mock_bucket fake_key = FakeKey('dir/file 1.odt') mock_bucket.list.return_value = [fake_key] mock_bucket.get_key.return_value = FakeKey( 'dir/file 1.odt', timestamp='2015-10-10T15:00:00.0000Z') assert S3('test-bucket').list( load_timestamps=True )[0]['last_modified'] == '2015-10-10T15:00:00.000000Z'
def test_list_files_removes_directories(self): mock_bucket = mock.Mock() self.s3_mock.get_bucket.return_value = mock_bucket fake_key_directory = FakeKey('dir/', size=0) fake_key_file = FakeKey('dir/file 1.odt') mock_bucket.list.return_value = [fake_key_directory, fake_key_file] expected = [ fake_key_file.fake_format_key(filename='file 1', ext='odt') ] self.assertEqual(S3('test-bucket').list(), expected)
def test_list_files_order_by_last_modified(self): mock_bucket = mock.Mock() self.s3_mock.get_bucket.return_value = mock_bucket fake_key_later = FakeKey('dir/file 1.odt') fake_key_earlier = FakeKey('dir/file 2.odt', last_modified='2014-08-17T14:00:00.000000Z') mock_bucket.list.return_value = [fake_key_later, fake_key_earlier] expected = [ fake_key_earlier.fake_format_key(filename='file 2', ext='odt'), fake_key_later.fake_format_key(filename='file 1', ext='odt') ] self.assertEqual(S3('test-bucket').list(), expected)
def test_list_files_with_loading_custom_timestamps_sorts_by_timestamp( self): mock_bucket = mock.Mock() self.s3_mock.get_bucket.return_value = mock_bucket fake_key = FakeKey('dir/file 1.odt') mock_bucket.list.return_value = [fake_key, fake_key, fake_key] mock_bucket.get_key.side_effect = [ FakeKey('dir/file 1.odt', timestamp='2015-12-10T15:00:00.0000Z'), FakeKey('dir/file 1.odt', timestamp='2015-11-10T15:00:00.0000Z'), FakeKey('dir/file 1.odt'), ] results = S3('test-bucket').list(load_timestamps=True) assert results[0]['last_modified'] == '2015-08-17T14:00:00.000000Z' assert results[1]['last_modified'] == '2015-11-10T15:00:00.000000Z' assert results[2]['last_modified'] == '2015-12-10T15:00:00.000000Z'
def test_save_sets_content_type_and_content_disposition_header(self): mock_bucket = FakeBucket() self.s3_mock.get_bucket.return_value = mock_bucket S3('test-bucket').save('folder/test-file.pdf', mock_file('blah', 123), download_filename='new-test-file.pdf') self.assertEqual(mock_bucket.keys, set(['folder/test-file.pdf'])) mock_bucket.s3_key_mock.set_contents_from_file.assert_called_with( mock.ANY, headers={ 'Content-Type': 'application/pdf', 'Content-Disposition': 'attachment; filename="new-test-file.pdf"'.encode('utf-8') })
def update(service_id, section_id): service = data_api_client.get_service(service_id) if service is None: abort(404) service = service['services'] content = content_loader.get_builder('g-cloud-6', 'edit_service_as_admin').filter(service) section = content.get_section(section_id) if section is None or not section.editable: abort(404) errors = None posted_data = section.get_data(request.form) uploaded_documents, document_errors = upload_service_documents( S3(current_app.config['DM_S3_DOCUMENT_BUCKET']), current_app.config['DM_DOCUMENTS_URL'], service, request.files, section) if document_errors: errors = section.get_error_messages(document_errors) else: posted_data.update(uploaded_documents) if not errors and section.has_changes_to_save(service, posted_data): try: data_api_client.update_service( service_id, posted_data, current_user.email_address) except HTTPError as e: errors = section.get_error_messages(e.message) if errors: if not posted_data.get('serviceName', None): posted_data['serviceName'] = service.get('serviceName', '') return render_template( "edit_section.html", section=section, service_data=posted_data, service_id=service_id, errors=errors ), 400 return redirect(url_for(".view", service_id=service_id))
def test_default_move_prefix_is_datetime(self): mock_bucket = FakeBucket(['folder/test-file.pdf']) self.s3_mock.get_bucket.return_value = mock_bucket now = datetime.datetime(2015, 1, 1, 1, 2, 3, 4) with mock.patch.object(datetime, 'datetime', mock.Mock(wraps=datetime.datetime)) as patched: patched.utcnow.return_value = now S3('test-bucket').save( 'folder/test-file.pdf', mock_file('blah', 123), ) self.assertEqual( mock_bucket.keys, { 'folder/test-file.pdf', 'folder/2015-01-01T01:02:03.000004-test-file.pdf' })