def test_provenance_augmentation(dmf_mock, headers_mock, access_mock, builder_mock, repo_get_mock, test_fqn): builder_mock.return_value._get_obs_id.return_value = None access_mock.return_value = 'https://localhost' test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_config.task_types = [mc.TaskType.VISIT] test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) headers_mock.return_value.get_head.side_effect = _get_headers_mock dmf_mock.get.side_effect = _get_obs_id_mock repo_get_mock.side_effect = _repo_get_mock getcwd_orig = os.getcwd os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR) temp = os.path.basename(test_fqn).replace('.expected.xml', '.fits') test_storage_name = GemProcName(entry=temp) try: test_obs = mc.read_obs_from_file(test_fqn) assert not test_obs.target.moving, 'initial conditions moving target' kwargs = { 'storage_name': test_storage_name, 'working_directory': test_main_app.TEST_DATA_DIR, 'observable': test_observable, 'caom_repo_client': Mock(), } test_result = provenance_augmentation.visit(test_obs, **kwargs) assert test_result is not None, 'expect a result' assert test_result.get('provenance') == 2, 'wrong result' assert len(test_obs.members) == 1, 'wrong membership' assert test_obs.target.moving, 'should be changed' finally: os.getcwd = getcwd_orig
def test_store(put_mock): test_config = mc.Config() test_config.logging_level = 'ERROR' test_config.working_directory = '/tmp' test_url = 'https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/' \ 'T10t12/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/' \ 'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.image.' \ 'pbcor.tt0.rms.subim.fits' test_storage_name = VlassName(url=test_url, entry=test_url) transferrer = Mock() cred_param = Mock() cadc_data_client = Mock() caom_repo_client = Mock() observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'), mc.Metrics(test_config)) test_subject = ec.Store(test_config, test_storage_name, APPLICATION, cred_param, cadc_data_client, caom_repo_client, observable, transferrer) test_subject.execute(None) assert put_mock.called, 'expect a call' args, kwargs = put_mock.call_args assert args[2] == test_storage_name.file_name, 'wrong file name' assert transferrer.get.called, 'expect a transfer call' args, kwargs = transferrer.get.call_args assert args[0] == test_url, 'wrong source parameter' assert args[1] == f'/tmp/{test_storage_name.obs_id}/' \ f'{test_storage_name.file_name}',\ 'wrong destination parameter'
def test_meta_update_observation_direct(test_config): test_cred_param = '--cert /usr/src/app/cadcproxy.pem' data_client_mock = Mock() repo_client_mock = Mock() test_sn = tc.TestStorageName() test_observation = mc.read_obs_from_file( f'{tc.TEST_DATA_DIR}/fpf_start_obs.xml' ) try: test_observable = mc.Observable( mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config) ) test_executor = ec.MetaUpdateObservation( test_config, test_sn, TEST_APP, test_cred_param, data_client_mock, repo_client_mock, test_observation, [], observable=test_observable, ) test_executor.execute(None) assert repo_client_mock.update.called, 'repo update called' finally: pass
def test_preview_augment(http_mock): # this should result in two new artifacts being added to the plane # one for a thumbnail and one for a preview obs = mc.read_obs_from_file(TEST_OBS_FILE) obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow() assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition' test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_metrics = mc.Metrics(test_config) test_observable = mc.Observable(test_rejected, test_metrics) cadc_client_mock = Mock() clients_mock = Mock() clients_mock.data_client = cadc_client_mock test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits') kwargs = { 'working_directory': '/test_files', 'clients': clients_mock, 'observable': test_observable, 'storage_name': test_storage_name, } test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg' if os.path.exists(test_prev): os.unlink(test_prev) try: cadc_client_mock.get.side_effect = exceptions.UnexpectedException( 'test') http_mock.side_effect = _get_mock obs = preview_augmentation.visit(obs, **kwargs) test_url = (f'{preview_augmentation.PREVIEW_URL}' f'{TEST_PRODUCT_ID}.fits') assert http_mock.called, 'http mock should be called' http_mock.assert_called_with(test_url, test_prev), 'mock not called' assert cadc_client_mock.put.called, 'put mock not called' cadc_client_mock.put.assert_called_with( '/test_files', 'cadc:GEMINI/GN2001BQ013-04_th.jpg', ), 'wrong put arguments' assert obs is not None, 'expect a result' assert (len( obs.planes[TEST_PRODUCT_ID].artifacts) == 3), 'two new artifacts' prev_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}.jpg', SCHEME) thumb_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}_th.jpg', 'cadc') assert (prev_uri in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'no preview' assert (thumb_uri in obs.planes[TEST_PRODUCT_ID].artifacts), 'no thumbnail' finally: if os.path.exists(test_prev): os.unlink(test_prev)
def test_preview_augment_plane(): omm_name = OmmName(file_name=TEST_FILE, entry=TEST_FILE) preview = os.path.join(TEST_FILES_DIR, omm_name.prev) thumb = os.path.join(TEST_FILES_DIR, omm_name.thumb) if os.path.exists(preview): os.remove(preview) if os.path.exists(thumb): os.remove(thumb) test_fqn = os.path.join(TEST_DATA_DIR, f'{omm_name.product_id}.expected.xml') test_obs = mc.read_obs_from_file(test_fqn) assert len(test_obs.planes[omm_name.product_id].artifacts) == 1 preva = 'cadc:OMM/C170324_0054_SCI_prev.jpg' thumba = 'cadc:OMM/C170324_0054_SCI_prev_256.jpg' test_config = mc.Config() test_config.observe_execution = True test_metrics = mc.Metrics(test_config) test_observable = mc.Observable(rejected=None, metrics=test_metrics) test_kwargs = { 'working_directory': TEST_FILES_DIR, 'cadc_client': None, 'observable': test_observable, 'storage_name': omm_name, } test_result = preview_augmentation.visit(test_obs, **test_kwargs) assert test_result is not None, 'expected a visit return value' assert os.path.exists(preview) assert os.path.exists(thumb) test_plane = test_result.planes[omm_name.product_id] assert test_plane.artifacts[preva].content_checksum == ChecksumURI( 'md5:f37d21c53055498d1b5cb7753e1c6d6f'), 'prev checksum failure' assert test_plane.artifacts[thumba].content_checksum == ChecksumURI( 'md5:19661c3c2508ecc22425ee2a05881ed4'), 'thumb checksum failure' # now do updates test_obs.planes[ omm_name.product_id].artifacts[preva].content_checksum = ChecksumURI( 'de9f39804f172682ea9b001f8ca11f15') test_obs.planes[ omm_name.product_id].artifacts[thumba].content_checksum = ChecksumURI( 'cd118dae04391f6bea93ba4bf2711adf') test_result = preview_augmentation.visit(test_obs, **test_kwargs) assert test_result is not None, 'expected update visit return value' assert len(test_result.planes[omm_name.product_id].artifacts) == 3 assert os.path.exists(preview) assert os.path.exists(thumb) assert test_plane.artifacts[preva].content_checksum == ChecksumURI( 'md5:f37d21c53055498d1b5cb7753e1c6d6f'), 'prev update failed' assert test_plane.artifacts[thumba].content_checksum == ChecksumURI( 'md5:19661c3c2508ecc22425ee2a05881ed4'), 'prev_256 update failed' assert len(test_metrics.history) == 0, 'wrong history, client is not None'
def test_pull_augmentation(http_mock, json_mock, header_mock, file_type_mock): obs = mc.read_obs_from_file(TEST_OBS_AD_URI_FILE) obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow() original_uri = 'gemini:GEMINI/GN2001BQ013-04.fits' assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition' assert ( original_uri in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'initial condition' test_uri = f'{SCHEME}:{COLLECTION}/{TEST_PRODUCT_ID}.fits' test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) cadc_client_mock = Mock() cadc_client_mock.return_value.info.return_value = None clients_mock = Mock() clients_mock.data_client = cadc_client_mock json_mock.side_effect = gem_mocks.mock_retrieve_json filter_cache = svofps.FilterMetadataCache(Mock()) test_reader = gemini_metadata.GeminiFileMetadataReader( Mock(), Mock(), filter_cache) test_fqn = f'{gem_mocks.TEST_DATA_DIR}/GMOS/GN2001BQ013-04.fits.header' test_storage_name = gem_name.GemName(file_name='GN2001BQ013-04.fits') header_mock.side_effect = gem_mocks._mock_headers file_type_mock.return_values = 'application/fits' test_reader.set(test_storage_name) kwargs = { 'working_directory': TEST_DATA_DIR, 'clients': clients_mock, 'observable': test_observable, 'metadata_reader': test_reader, 'storage_name': test_storage_name, } obs = pull_augmentation.visit(obs, **kwargs) test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits' test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits' http_mock.assert_called_with(test_url, test_prev), 'mock not called' assert cadc_client_mock.put.called, 'put mock not called' cadc_client_mock.put.assert_called_with( TEST_DATA_DIR, 'gemini:GEMINI/GN2001BQ013-04.fits'), 'wrong put args' assert obs is not None, 'expect a result' assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'no new artifacts' try: ignore = obs.planes[TEST_PRODUCT_ID].artifacts[test_uri] except KeyError as ke: # because CAOM does magic result = obs.planes[TEST_PRODUCT_ID].artifacts[original_uri] assert result.uri == test_uri, f'wrong uri {result.uri}'
def test_cadc_transfer(client_mock, caps_mock): caps_mock.return_value = 'https://sc2.canfar.net/sc2repo' test_subject = tc.CadcTransfer() assert test_subject is not None, 'expect a result' test_config = mc.Config() test_config.rejected_fqn = '/tmp/rejected.yml' test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config)) test_subject.observable = test_observable test_subject.cadc_client = client_mock test_source = 'ad:TEST/test_file.fits' test_destination = '/tmp/test_file.fits' test_subject.get(test_source, test_destination) assert client_mock.get.called, 'should have been called' client_mock.get.assert_called_with( '/tmp', 'ad:TEST/test_file.fits'), 'wrong parameters'
def test_preview_augment_unknown_no_preview(): # what happens when it's not known that there's no preview obs = mc.read_obs_from_file(TEST_OBS_FILE) obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow() assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition' # make sure the rejected file is empty if os.path.exists(REJECTED_FILE): os.unlink(REJECTED_FILE) test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits') cadc_client_mock = Mock() clients_mock = Mock() clients_mock.data_client = cadc_client_mock kwargs = { 'working_directory': TEST_DATA_DIR, 'clients': clients_mock, 'stream': 'stream', 'observable': test_observable, 'storage_name': test_storage_name, } with patch( 'caom2pipe.manage_composable.http_get', side_effect=mc.CadcException( 'Not Found for url: https://archive.gemini.edu/preview'), ) as http_mock, patch( 'caom2pipe.manage_composable.data_put') as ad_put_mock, patch( 'caom2pipe.manage_composable.get_artifact_metadata' ) as art_mock, patch( 'caom2pipe.manage_composable.exec_cmd') as exec_mock: cadc_client_mock.get.side_effect = exceptions.UnexpectedException( 'test') obs = preview_augmentation.visit(obs, **kwargs) assert obs is not None, 'expect result' test_url = f'{preview_augmentation.PREVIEW_URL}{TEST_PRODUCT_ID}.fits' test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.jpg' http_mock.assert_called_with(test_url, test_prev), 'mock not called' assert not ad_put_mock.called, 'ad put mock should not be called' assert not art_mock.called, 'art mock should not be called' assert not exec_mock.called, 'exec mock should not be called'
def test_preview_augmentation(access_mock): access_mock.return_value = 'https://localhost' test_fqn = f'{TEST_DATA_DIR}/preview_augmentation_start.xml' test_science_f_name = ( 'VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.' 'tt0.subim.fits') test_storage_name = sn.VlassName(test_science_f_name) test_obs = mc.read_obs_from_file(test_fqn) test_config = mc.Config() test_rejected = mc.Rejected(f'{TEST_DATA_DIR}/rejected.yml') test_metrics = mc.Metrics(test_config) test_observable = mc.Observable(test_rejected, test_metrics) kwargs = { 'stream': None, 'observable': test_observable, 'storage_name': test_storage_name, 'working_directory': '/test_files', } test_subject = preview_augmentation.VlassPreview(**kwargs) assert test_subject is not None, 'need a test subject' assert len(test_obs.planes) == 1, 'wrong number of planes' assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 4 ), 'wrong starting # of artifacts' test_obs = test_subject.visit(test_obs) assert test_obs is not None, 'expect a result' assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 6 ), 'wrong ending # of artifacts' test_report = test_subject.report assert test_report is not None assert 'artifacts' in test_report assert test_report.get('artifacts') == 2, 'wrong report count' # does artifact re-naming work? test_url = (f'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.1/' f'T01t01/VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1/' f'{test_science_f_name}') kwargs = {'url': test_url} test_obs = cleanup_augmentation.visit(test_obs, **kwargs) test_artifacts = test_obs.planes[test_storage_name.product_id].artifacts assert len(test_artifacts) == 4, 'wrong ending conditions' assert test_storage_name.prev_uri in test_artifacts, 'missing preview' assert test_storage_name.thumb_uri in test_artifacts, 'missing thumbnail'
def test_store(): test_config = mc.Config() test_config.logging_level = 'ERROR' test_config.working_directory = '/tmp' test_url = ('https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/' 'T10t12/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/' 'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.image.' 'pbcor.tt0.rms.subim.fits') test_storage_name = VlassName(test_url) transferrer = Mock() cadc_data_client = Mock() observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'), mc.Metrics(test_config)) test_subject = ec.Store( test_config, test_storage_name, cadc_data_client, observable, transferrer, ) test_subject.execute(None) assert cadc_data_client.put.called, 'expect a call' cadc_data_client.put.assert_called_with( '/tmp/VLASS2.1.T10t12.J073401-033000', f'{SCHEME}:VLASS/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.' f'iter1.image.pbcor.tt0.rms.subim.fits', None, ), 'wrong put args' assert transferrer.get.called, 'expect a transfer call' test_f_name = ('VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.' 'image.pbcor.tt0.rms.subim.fits') transferrer.get.assert_called_with( f'https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/T10t12/' f'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/{test_f_name}', f'/tmp/VLASS2.1.T10t12.J073401-033000/{test_f_name}', ), 'wrong transferrer args'
def test_visit(): class TestVisitor(mc.PreviewVisitor): def __init__(self, **kwargs): super(TestVisitor, self).__init__(archive='VLASS', **kwargs) def generate_plots(self, obs_id): fqn = f'{self._working_dir}/{self._storage_name.prev}' with open(fqn, 'w') as f: f.write('test content') self.add_preview( self._storage_name.prev_uri, self._storage_name.prev, ProductType.THUMBNAIL, ReleaseType.META, ) return 1 class VisitStorageName(tc.TestStorageName): def __init__(self): super(VisitStorageName, self).__init__() self._source_names = [self.file_uri] @property def product_id(self): return test_product_id @property def file_uri(self): return f'ad:VLASS/{test_file_name}' test_rejected = mc.Rejected(f'{tc.TEST_DATA_DIR}/rejected.yml') test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) cadc_client_mock = Mock() test_product_id = 'VLASS1.2.T07t14.J084202-123000.quicklook.v1' test_file_name = ('VLASS1.2.ql.T07t14.J084202-123000.10.2048.v1.I.iter1.' 'image.pbcor.tt0.subim.fits') storage_name = VisitStorageName() kwargs = { 'working_directory': tc.TEST_FILES_DIR, 'cadc_client': cadc_client_mock, 'stream': 'stream', 'observable': test_observable, 'storage_name': storage_name, } obs = mc.read_obs_from_file(f'{tc.TEST_DATA_DIR}/fpf_start_obs.xml') assert len(obs.planes[test_product_id].artifacts) == 2, 'initial condition' try: test_subject = TestVisitor(**kwargs) test_result = test_subject.visit(obs) except Exception as e: assert False, f'{str(e)}' assert test_result is not None, f'expect a result' check_number = 1 end_artifact_count = 3 expected_call_count = 1 assert test_result['artifacts'] == check_number, 'artifact not added' assert (len(obs.planes[test_product_id].artifacts) == end_artifact_count ), f'new artifacts' test_preview_uri = 'cadc:TEST/test_obs_id_prev.jpg' assert (test_preview_uri in obs.planes[test_product_id].artifacts.keys()), 'no preview' assert cadc_client_mock.put.called, 'put mock not called' assert (cadc_client_mock.put.call_count == expected_call_count ), 'put called wrong number of times' # it's an ad call, so there's a stream parameter cadc_client_mock.put.assert_called_with('/test_files', 'cadc:TEST/test_obs_id_prev.jpg', 'stream')
def test_preview_augment(): # this should result in three new artifacts being added to every plane: # one for a thumbnail and two for previews (one zoom) test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) test_files = { 'visit_obs_start_wircam.xml': ['1151210o.fits.fz', '1151210s.fits.fz', '1151210m.fits.fz', '1151210p.fits.fz', '1151210w.fits.fz', '1151210y.fits.fz', '1151210g.fits.fz'], 'visit_obs_start_megacam_sci.xml': ['1927963f.fits.fz', '1927963o.fits.fz', '1927963p.fits.fz'], 'visit_obs_start_megacam_cal.xml': ['979412b.fits.fz', '979412o.fits.fz', '979412p.fits.fz'], 'visit_obs_start_sitelle_calibrated_cube.xml': ['2359320p.fits'], 'visit_obs_start_sitelle.xml': ['2359320o.fits.fz'], 'visit_obs_start_espadons.xml': ['2460606i.fits.gz', '2460606o.fits.gz'], 'visit_obs_start_espadons_cal.xml': ['1001063b.fits.gz'], 'visit_obs_start_spirou.xml': ['2401734o.fits', '2401734e.fits', '2401734r.fits', '2401734s.fits', '2401734t.fits', '2401734v.fits'], 'visit_obs_start_2hdus_mega.xml': ['1821271p.fits.fz'] } test_checksums = { 'ad:CFHT/2460606i_preview_1024.jpg': 'md5:eeb2558c76c83a1b56e0afa3d96b0fa0', 'ad:CFHT/2460606i_preview_256.jpg': 'md5:1084a1e58d246c35fae43ca021916c6a', 'ad:CFHT/2460606o_preview_1024.jpg': 'md5:eeeb436f0e6b1cb48aca5a9ba5d1ab3a', 'ad:CFHT/2460606o_preview_256.jpg': 'md5:3879c153776ee0569ee33bc8a35b5e75', 'ad:CFHT/2460606o_preview_zoom_1024.jpg': 'md5:1d5c9e5ee1586ddd6ce5c53b17f8f5ec', 'ad:CFHT/1001063b_preview_1024.jpg': 'md5:d457fa8ecdd397790de65f8bd3a80d06', 'ad:CFHT/1001063b_preview_256.jpg': 'md5:3ae90e1e120fb97216ddb2d965e9423e', 'ad:CFHT/1001063b_preview_zoom_1024.jpg': 'md5:a2d62e654838a620dcf97425753481ca', 'ad:CFHT/979412b_preview_1024.jpg': 'md5:8b3e4ff2c210b3861748423339a71cc5', 'ad:CFHT/979412b_preview_256.jpg': 'md5:893a9cd18f66fab907573bf1e664148d', 'ad:CFHT/979412b_preview_zoom_1024.jpg': 'md5:4f21d6676b0749c48b4872699f7aff7c', 'ad:CFHT/979412o_preview_1024.jpg': 'md5:379a7b21022a0e758fbdea6130c07233', 'ad:CFHT/979412o_preview_256.jpg': 'md5:59c2ecedda2a2560cafdca543533e190', 'ad:CFHT/979412o_preview_zoom_1024.jpg': 'md5:6f3a3a39155a426a365f13f6d10982a9', 'ad:CFHT/979412p_preview_1024.jpg': 'md5:d449d26987a4d9e220d249f8573025ee', 'ad:CFHT/979412p_preview_256.jpg': 'md5:102be79a6ee26a7e5c2ee7e40623d452', 'ad:CFHT/979412p_preview_zoom_1024.jpg': 'md5:7e09407afb9e8148ad36ba56933e1822', 'ad:CFHT/1927963f_preview_1024.jpg': 'md5:9afd99f2d60de3bc168832ce37200508', 'ad:CFHT/1927963f_preview_256.jpg': 'md5:5de596f4b5ab1ae8bfab1de5550ec56d', 'ad:CFHT/1927963f_preview_zoom_1024.jpg': 'md5:ca705c70eb0d320856e346662a83f1ba', 'ad:CFHT/1927963o_preview_1024.jpg': 'md5:d8a9c9c6dde50c59f75a1b384300d41a', 'ad:CFHT/1927963o_preview_256.jpg': 'md5:84cfc76e094678e61eaa9f9421e30dd9', 'ad:CFHT/1927963o_preview_zoom_1024.jpg': 'md5:843109c217104034fa3b029949fa277e', 'ad:CFHT/1927963p_preview_1024.jpg': 'md5:61b903863106438cfa2771731ed8690d', 'ad:CFHT/1927963p_preview_256.jpg': 'md5:01e4de796ff3f20949d02fe776b5f699', 'ad:CFHT/1927963p_preview_zoom_1024.jpg': 'md5:91c4a735d507d3c11482179d725a4166', 'ad:CFHT/2359320o_preview_1024.jpg': 'md5:45b164799c67751c5a90255c11ccd9e5', 'ad:CFHT/2359320o_preview_256.jpg': 'md5:dce23df89c3424bc67d733a7784a13d3', 'ad:CFHT/2359320o_preview_zoom_1024.jpg': 'md5:5cc26a79326b42c6a17bca8e0d3fb1fb', # 'ad:CFHT/2359320p_preview_1024.jpg': #### # 'md5:b09f54461fb10259f4958ff9ab1e8179', # 'ad:CFHT/2359320p_preview_256.jpg': # 'md5:b037076c0d9488d013052d6750e55aeb', # 'ad:CFHT/2359320p_preview_zoom_1024.jpg': # 'md5:514aa25fec1b74fe942ccc8ce8e552f7', 'ad:CFHT/2401734e_preview_1024.jpg': 'md5:a3b272473e959cbc5496c1d562f16c0d', 'ad:CFHT/2401734e_preview_256.jpg': 'md5:781a1c86e727f82ec17a98b5eaf2b7d0', 'ad:CFHT/2401734o_preview_1024.jpg': 'md5:1b84806bcd9ed45e850afa802499e239', 'ad:CFHT/2401734o_preview_256.jpg': 'md5:b1e8aa9b1f0275505660435d3ee6443e', 'ad:CFHT/2401734o_preview_zoom_1024.jpg': 'md5:18839f9482dd3420f0d6ca3c167e6575', 'ad:CFHT/2401734r_preview_1024.jpg': 'md5:3c0dc874898cefe2ca46195e2e19b763', 'ad:CFHT/2401734r_preview_256.jpg': 'md5:3b06668db0e8d9819cc1c50100bd85f4', 'ad:CFHT/2401734r_preview_zoom_1024.jpg': 'md5:32b54773700dd652e244632f51aca5ba', 'ad:CFHT/2401734s_preview_1024.jpg': 'md5:30a76081e4e6f5ee332e8e076feff2a7', 'ad:CFHT/2401734s_preview_256.jpg': 'md5:f32f4288aebc1f7cea577d6d217929c9', 'ad:CFHT/2401734t_preview_1024.jpg': 'md5:3a5409aa4cd728e0a92f50a3eff88427', 'ad:CFHT/2401734t_preview_256.jpg': 'md5:f90654bad03ba8c2e4796ab4acfb5264', 'ad:CFHT/1151210g_preview_1024.jpg': 'md5:ac415da2e2d9847b81c59477c73f26ce', 'ad:CFHT/1151210g_preview_256.jpg': 'md5:56ad048a81c7c16f0801ad8b22fc943f', 'ad:CFHT/1151210g_preview_zoom_1024.jpg': 'md5:0486b7e6572bb0bd8750ff16159ee5ad', 'ad:CFHT/1151210m_preview_1024.jpg': 'md5:02abd143e36acd593cab8eeea458eaf2', 'ad:CFHT/1151210m_preview_256.jpg': 'md5:99d5b739c4ab1ef69aa0b34a0696f9cf', 'ad:CFHT/1151210m_preview_zoom_1024.jpg': 'md5:d55b984b8dcb39baa0d5ba0a34051ffa', 'ad:CFHT/1151210w_preview_1024.jpg': 'md5:542b11ae4e21edc19380e0b37dec129b', 'ad:CFHT/1151210w_preview_256.jpg': 'md5:a10b7c6b8f101f52ab16d0da0649451a', 'ad:CFHT/1151210w_preview_zoom_1024.jpg': 'md5:e79bb2ac4230d5587f97a9b838d6f88d', 'ad:CFHT/1151210o_preview_1024.jpg': 'md5:302ce34edd7c69da23119d001e85d39a', 'ad:CFHT/1151210o_preview_256.jpg': 'md5:137fa8a405cdf03f827000908831ece9', 'ad:CFHT/1151210o_preview_zoom_1024.jpg': 'md5:e656f21a21a4a41e088cccf87ec76580', 'ad:CFHT/1151210p_preview_1024.jpg': 'md5:f05cb0ad85ac610ef9bc3cf1ea00c853', 'ad:CFHT/1151210p_preview_256.jpg': 'md5:73aa50e0d59a69452ae8be06de9c6317', 'ad:CFHT/1151210p_preview_zoom_1024.jpg': 'md5:e56690286391a74840bc07158cedaeec', 'ad:CFHT/1151210s_preview_1024.jpg': 'md5:7d7019efd828ff748fece2b778c5fecb', 'ad:CFHT/1151210s_preview_256.jpg': 'md5:43051edaa8dbb6436d4dcb32fc5a2bd0', 'ad:CFHT/1151210s_preview_zoom_1024.jpg': 'md5:4f20f5586eaa43abc5de977e60cea807', 'ad:CFHT/1151210y_preview_1024.jpg': 'md5:931dd9cabffe9e470a556327c08c6b96', 'ad:CFHT/1151210y_preview_256.jpg': 'md5:45df1e0a5dd4b4b058b7e20cdc55b9f7', 'ad:CFHT/1151210y_preview_zoom_1024.jpg': 'md5:53cfa0d681c46fedfc2bf719e88231b5' } kwargs = {'working_directory': TEST_FILES_DIR, 'cadc_client': None, 'stream': 'stream', 'observable': test_observable} for entry in glob.glob(f'{TEST_FILES_DIR}/*.jpg'): os.unlink(entry) checksum_failures = [] for key, value in test_files.items(): obs = mc.read_obs_from_file(f'{test_main_app.TEST_DATA_DIR}/{key}') if 'wircam' in key: instrument = md.Inst.WIRCAM elif 'mega' in key: instrument = md.Inst.MEGACAM elif 'sitelle' in key: instrument = md.Inst.SITELLE elif 'espadons' in key: instrument = md.Inst.ESPADONS elif 'spirou' in key: instrument = md.Inst.SPIROU else: assert False, 'do not understand instrument' for f_name in value: kwargs['science_file'] = f_name test_name = cfht_name.CFHTName(file_name=f_name, instrument=instrument) check_number = 1 if test_name.suffix == 'p' and instrument is md.Inst.SITELLE: check_number = 3 assert len(obs.planes[test_name.product_id].artifacts) == \ check_number, f'initial condition {f_name}' try: from datetime import datetime import logging start_ts = datetime.utcnow().timestamp() test_result = preview_augmentation.visit(obs, **kwargs) end_ts = datetime.utcnow().timestamp() logging.error(f'{f_name} execution time {end_ts - start_ts}') except Exception as e: import logging logging.error(e) import traceback logging.error(traceback.format_exc()) assert False assert test_result is not None, f'expect a result {f_name}' check_number = 3 end_artifact_count = 4 f_name_list = [test_name.prev_uri, test_name.thumb_uri, test_name.zoom_uri] if ((instrument is md.Inst.ESPADONS and test_name.suffix == 'i') or (instrument is md.Inst.SPIROU and test_name.suffix in ['e', 'p', 's', 't', 'v'])): check_number = 2 end_artifact_count = 3 f_name_list = [test_name.prev_uri, test_name.thumb_uri] assert test_result['artifacts'] == check_number, \ f'artifacts should be added {f_name}' if test_name.suffix == 'p' and instrument is md.Inst.SITELLE: end_artifact_count = 6 assert len(obs.planes[test_name.product_id].artifacts) == \ end_artifact_count, f'new artifacts {f_name}' for p in f_name_list: # assert p in \ # obs.planes[test_name.product_id].artifacts.keys(), \ # f'no preview {p}' if p in obs.planes[test_name.product_id].artifacts.keys(): artifact = obs.planes[test_name.product_id].artifacts[p] # because 2359320p_preview_1024 keeps changing .... if artifact.uri in test_checksums: if artifact.content_checksum.uri != test_checksums[p]: checksum_failures.append( f'{p} expected {test_checksums[p]} actual ' f'{artifact.content_checksum.uri}') assert len(checksum_failures) == 0, \ '\n'.join(ii for ii in checksum_failures)