def test_provenance_augmentation(dmf_mock, headers_mock, access_mock, builder_mock, repo_get_mock, test_fqn): builder_mock.return_value._get_obs_id.return_value = None access_mock.return_value = 'https://localhost' test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_config.task_types = [mc.TaskType.VISIT] test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) headers_mock.return_value.get_head.side_effect = _get_headers_mock dmf_mock.get.side_effect = _get_obs_id_mock repo_get_mock.side_effect = _repo_get_mock getcwd_orig = os.getcwd os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR) temp = os.path.basename(test_fqn).replace('.expected.xml', '.fits') test_storage_name = GemProcName(entry=temp) try: test_obs = mc.read_obs_from_file(test_fqn) assert not test_obs.target.moving, 'initial conditions moving target' kwargs = { 'storage_name': test_storage_name, 'working_directory': test_main_app.TEST_DATA_DIR, 'observable': test_observable, 'caom_repo_client': Mock(), } test_result = provenance_augmentation.visit(test_obs, **kwargs) assert test_result is not None, 'expect a result' assert test_result.get('provenance') == 2, 'wrong result' assert len(test_obs.members) == 1, 'wrong membership' assert test_obs.target.moving, 'should be changed' finally: os.getcwd = getcwd_orig
def test_pull_augmentation(): obs = mc.read_obs_from_file(TEST_OBS_FILE) obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow() assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition' test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) cadc_client_mock = Mock() kwargs = {'working_directory': TEST_DATA_DIR, 'cadc_client': cadc_client_mock, 'stream': 'stream', 'observable': test_observable} with patch('caom2pipe.manage_composable.http_get') as http_mock, \ patch('caom2pipe.manage_composable.data_put') as ad_put_mock: cadc_client_mock.return_value.data_get.return_value = mc.CadcException( 'test') # no scheme from cadc client cadc_client_mock.get_file_info.return_value = {'md5sum': '1234'} result = pull_augmentation.visit(obs, **kwargs) test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits' test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits' http_mock.assert_called_with(test_url, test_prev), 'mock not called' assert ad_put_mock.called, 'ad put mock not called' assert result is not None, 'expect a result' assert result['observation'] == 0, 'no updated metadata' assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, \ 'no new artifacts'
def test_pull_v_augmentation(put_mock, http_mock): obs = mc.read_obs_from_file(TEST_OBS_FILE) obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow() assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition' test_uri = f'{SCHEME}:{COLLECTION}/{TEST_PRODUCT_ID}.fits' for plane in obs.planes.values(): for artifact in plane.artifacts.values(): artifact.uri = test_uri test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) cadc_client_mock = Mock() kwargs = {'working_directory': TEST_DATA_DIR, 'cadc_client': cadc_client_mock, 'observable': test_observable} result = pull_v_augmentation.visit(obs, **kwargs) test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits' test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits' http_mock.assert_called_with(test_url, test_prev), 'mock not called' assert put_mock.called, 'put mock not called' args, kwargs = put_mock.call_args assert args[1] == TEST_DATA_DIR, 'wrong working dir' assert args[2] == f'{TEST_PRODUCT_ID}.fits', 'wrong file name' assert args[3] == test_uri, 'wrong storage name' assert result is not None, 'expect a result' assert result['observation'] == 0, 'no updated metadata' assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, \ 'no new artifacts'
def test_store(put_mock): test_config = mc.Config() test_config.logging_level = 'ERROR' test_config.working_directory = '/tmp' test_url = 'https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/' \ 'T10t12/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/' \ 'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.image.' \ 'pbcor.tt0.rms.subim.fits' test_storage_name = VlassName(url=test_url, entry=test_url) transferrer = Mock() cred_param = Mock() cadc_data_client = Mock() caom_repo_client = Mock() observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'), mc.Metrics(test_config)) test_subject = ec.Store(test_config, test_storage_name, APPLICATION, cred_param, cadc_data_client, caom_repo_client, observable, transferrer) test_subject.execute(None) assert put_mock.called, 'expect a call' args, kwargs = put_mock.call_args assert args[2] == test_storage_name.file_name, 'wrong file name' assert transferrer.get.called, 'expect a transfer call' args, kwargs = transferrer.get.call_args assert args[0] == test_url, 'wrong source parameter' assert args[1] == f'/tmp/{test_storage_name.obs_id}/' \ f'{test_storage_name.file_name}',\ 'wrong destination parameter'
def test_preview_augment_delete_preview(): # plane starts with a preview artifact, but it represents a non-existent # file, so remove the artifact from the CAOM observation test_product_id = 'S20080610S0045' fqn = os.path.join(TEST_DATA_DIR, 'GS-2008A-C-5-35-002.fits.xml') obs = mc.read_obs_from_file(fqn) assert len(obs.planes[test_product_id].artifacts) == 2, 'initial condition' test_rejected = mc.Rejected('/tmp/nonexistent') test_rejected.content = { 'bad_metadata': [], 'no_preview': ['S20080610S0043.jpg', 'S20080610S0041.jpg', 'S20080610S0044.jpg', 'S20080610S0045.jpg']} test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) kwargs = {'working_directory': TEST_DATA_DIR, 'cadc_client': None, 'stream': 'stream', 'observable': test_observable} result = preview_augmentation.visit(obs, **kwargs) assert result is not None, 'expect a result' assert result['artifacts'] == 1, 'wrong result' assert len(obs.planes[test_product_id].artifacts) == 1, 'post condition'
def test_meta_update_observation_direct(test_config): test_cred_param = '--cert /usr/src/app/cadcproxy.pem' data_client_mock = Mock() repo_client_mock = Mock() test_sn = tc.TestStorageName() test_observation = mc.read_obs_from_file( f'{tc.TEST_DATA_DIR}/fpf_start_obs.xml' ) try: test_observable = mc.Observable( mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config) ) test_executor = ec.MetaUpdateObservation( test_config, test_sn, TEST_APP, test_cred_param, data_client_mock, repo_client_mock, test_observation, [], observable=test_observable, ) test_executor.execute(None) assert repo_client_mock.update.called, 'repo update called' finally: pass
def test_store(put_mock): test_config = mc.Config() test_config.logging_level = 'ERROR' test_config.working_directory = '/tmp' test_fqn = '/users/OpenData_DonneesOuvertes/pub/NEOSSAT/ASTRO/2019/' \ '268/NEOS_SCI_2019268004930_clean.fits' test_storage_name = NEOSSatName(file_name=test_fqn, entry=test_fqn) transferrer = Mock() cred_param = Mock() cadc_data_client = Mock() caom_repo_client = Mock() observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'), mc.Metrics(test_config)) test_subject = ec.Store(test_config, test_storage_name, APPLICATION, cred_param, cadc_data_client, caom_repo_client, observable, transferrer) test_subject.execute(None) assert put_mock.called, 'expect a call' args, kwargs = put_mock.call_args assert args[2] == test_storage_name.file_name, 'wrong file name' assert transferrer.get.called, 'expect a transfer call' args, kwargs = transferrer.get.call_args import logging logging.error(args) assert args[0] == test_fqn, 'wrong source parameter' assert args[1] == f'/tmp/{test_storage_name.obs_id}/' \ f'{test_storage_name.file_name}', \ 'wrong destination parameter'
def test_look_pull_and_put_v(http_mock, mock_client): stat_orig = os.stat os.stat = Mock() os.stat.return_value = Mock(st_size=1234) try: test_storage_name = 'cadc:GEMINI/TEST.fits' f_name = 'test_f_name.fits' url = f'https://localhost/{f_name}' test_config = mc.Config() test_config.observe_execution = True test_metrics = mc.Metrics(test_config) mock_client.get_node.side_effect = tc.mock_get_node mock_client.copy.return_value = 1234 assert len(test_metrics.history) == 0, 'initial history conditions' assert len(test_metrics.failures) == 0, 'initial failure conditions' mc.look_pull_and_put_v( test_storage_name, f_name, tc.TEST_DATA_DIR, url, mock_client, 'md5:01234', test_metrics, ) test_fqn = os.path.join(tc.TEST_DATA_DIR, f_name) mock_client.copy.assert_called_with( test_fqn, destination=test_storage_name), 'mock not called' http_mock.assert_called_with(url, os.path.join( tc.TEST_DATA_DIR, f_name)), 'http mock not called' assert len(test_metrics.history) == 1, 'history conditions' assert len(test_metrics.failures) == 0, 'failure conditions' finally: os.stat = stat_orig
def test_provenance_augmentation(obs_id_mock, repo_get_mock, headers_mock, test_fqn): test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) headers_mock.side_effect = _get_headers_mock repo_get_mock.side_effect = _repo_get_mock obs_id_mock.side_effect = _get_obs_id_mock getcwd_orig = os.getcwd os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR) try: test_obs = mc.read_obs_from_file(test_fqn) assert not test_obs.target.moving, 'initial conditions moving target' kwargs = { 'science_file': os.path.basename(test_fqn).replace('.expected.xml', '.fits'), 'working_directory': test_main_app.TEST_DATA_DIR, 'observable': test_observable, 'caom_repo_client': Mock() } test_result = provenance_augmentation.visit(test_obs, **kwargs) assert test_result is not None, 'expect a result' assert test_result.get('provenance') == 2, 'wrong result' assert len(test_obs.members) == 1, 'wrong membership' assert test_obs.target.moving, 'should be changed' finally: os.getcwd = getcwd_orig
def _filter_result(self): config = mc.Config() config.get_executors() subject = clc.define_subject(config) caom_client = CAOM2RepoClient(subject) metrics = mc.Metrics(config) for entry in self._source: if VlassValidator._later_version_at_cadc(entry, caom_client, metrics): self._source.remove(entry)
def test_preview_augment(http_mock): # this should result in two new artifacts being added to the plane # one for a thumbnail and one for a preview obs = mc.read_obs_from_file(TEST_OBS_FILE) obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow() assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition' test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_metrics = mc.Metrics(test_config) test_observable = mc.Observable(test_rejected, test_metrics) cadc_client_mock = Mock() clients_mock = Mock() clients_mock.data_client = cadc_client_mock test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits') kwargs = { 'working_directory': '/test_files', 'clients': clients_mock, 'observable': test_observable, 'storage_name': test_storage_name, } test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg' if os.path.exists(test_prev): os.unlink(test_prev) try: cadc_client_mock.get.side_effect = exceptions.UnexpectedException( 'test') http_mock.side_effect = _get_mock obs = preview_augmentation.visit(obs, **kwargs) test_url = (f'{preview_augmentation.PREVIEW_URL}' f'{TEST_PRODUCT_ID}.fits') assert http_mock.called, 'http mock should be called' http_mock.assert_called_with(test_url, test_prev), 'mock not called' assert cadc_client_mock.put.called, 'put mock not called' cadc_client_mock.put.assert_called_with( '/test_files', 'cadc:GEMINI/GN2001BQ013-04_th.jpg', ), 'wrong put arguments' assert obs is not None, 'expect a result' assert (len( obs.planes[TEST_PRODUCT_ID].artifacts) == 3), 'two new artifacts' prev_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}.jpg', SCHEME) thumb_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}_th.jpg', 'cadc') assert (prev_uri in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'no preview' assert (thumb_uri in obs.planes[TEST_PRODUCT_ID].artifacts), 'no thumbnail' finally: if os.path.exists(test_prev): os.unlink(test_prev)
def test_preview_augment_plane(): omm_name = OmmName(file_name=TEST_FILE, entry=TEST_FILE) preview = os.path.join(TEST_FILES_DIR, omm_name.prev) thumb = os.path.join(TEST_FILES_DIR, omm_name.thumb) if os.path.exists(preview): os.remove(preview) if os.path.exists(thumb): os.remove(thumb) test_fqn = os.path.join(TEST_DATA_DIR, f'{omm_name.product_id}.expected.xml') test_obs = mc.read_obs_from_file(test_fqn) assert len(test_obs.planes[omm_name.product_id].artifacts) == 1 preva = 'cadc:OMM/C170324_0054_SCI_prev.jpg' thumba = 'cadc:OMM/C170324_0054_SCI_prev_256.jpg' test_config = mc.Config() test_config.observe_execution = True test_metrics = mc.Metrics(test_config) test_observable = mc.Observable(rejected=None, metrics=test_metrics) test_kwargs = { 'working_directory': TEST_FILES_DIR, 'cadc_client': None, 'observable': test_observable, 'storage_name': omm_name, } test_result = preview_augmentation.visit(test_obs, **test_kwargs) assert test_result is not None, 'expected a visit return value' assert os.path.exists(preview) assert os.path.exists(thumb) test_plane = test_result.planes[omm_name.product_id] assert test_plane.artifacts[preva].content_checksum == ChecksumURI( 'md5:f37d21c53055498d1b5cb7753e1c6d6f'), 'prev checksum failure' assert test_plane.artifacts[thumba].content_checksum == ChecksumURI( 'md5:19661c3c2508ecc22425ee2a05881ed4'), 'thumb checksum failure' # now do updates test_obs.planes[ omm_name.product_id].artifacts[preva].content_checksum = ChecksumURI( 'de9f39804f172682ea9b001f8ca11f15') test_obs.planes[ omm_name.product_id].artifacts[thumba].content_checksum = ChecksumURI( 'cd118dae04391f6bea93ba4bf2711adf') test_result = preview_augmentation.visit(test_obs, **test_kwargs) assert test_result is not None, 'expected update visit return value' assert len(test_result.planes[omm_name.product_id].artifacts) == 3 assert os.path.exists(preview) assert os.path.exists(thumb) assert test_plane.artifacts[preva].content_checksum == ChecksumURI( 'md5:f37d21c53055498d1b5cb7753e1c6d6f'), 'prev update failed' assert test_plane.artifacts[thumba].content_checksum == ChecksumURI( 'md5:19661c3c2508ecc22425ee2a05881ed4'), 'prev_256 update failed' assert len(test_metrics.history) == 0, 'wrong history, client is not None'
def declare_client(config): """Common code to set the client used for interacting with CADC storage.""" subject = define_subject(config) cadc_client = StorageClientWrapper( using_storage_inventory=config.features.supports_latest_client, resource_id=config.storage_inventory_resource_id, subject=subject, metrics=mc.Metrics(config), ) return cadc_client
def __init__(self, config): super(CFHTBuilder, self).__init__() self._config = config self._data_client = None self._repo_client = None self._metrics = mc.Metrics(self._config) if not self._config.use_local_files: subject = mc.define_subject(self._config) self._data_client = CadcDataClient(subject) self._repo_client = CAOM2RepoClient( subject, resource_id=self._config.resource_id) self._logger = logging.getLogger(__name__)
def test_client_get_failure(mock_client): test_config = mc.Config() test_config.observe_execution = True test_metrics = mc.Metrics(test_config) with pytest.raises(mc.CadcException): mc.client_get( mock_client, tc.TEST_DATA_DIR, 'TEST_get.fits', 'TEST', test_metrics, ) assert len(test_metrics.failures) == 1, 'wrong failures' assert test_metrics.failures['vos']['copy']['TEST_get.fits'] == 1, 'count'
def test_client_get_failure(mock_client): test_config = mc.Config() test_config.observe_execution = True test_metrics = mc.Metrics(test_config) with pytest.raises(mc.CadcException): clc.si_client_get( mock_client, os.path.join(tc.TEST_DATA_DIR, 'TEST_get.fits'), 'cadc:TEST/TEST_get.fits', test_metrics, ) assert len(test_metrics.failures) == 1, 'wrong failures' assert test_metrics.failures['si']['cadcget'][ 'TEST_get.fits'] == 1, 'count'
def test_preview_augment_known_no_preview(): # rejected file exists that says there's a preview known to not # exist, so trying to generate a thumbnail will result in no # change to the plane/artifact structure try: obs = mc.read_obs_from_file(TEST_OBS_FILE) obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow() assert (len( obs.planes[TEST_PRODUCT_ID].artifacts) == 1), 'initial condition' if os.path.exists(REJECTED_FILE): os.unlink(REJECTED_FILE) test_rejected = mc.Rejected(REJECTED_FILE) test_rejected.record(mc.Rejected.NO_PREVIEW, f'{TEST_PRODUCT_ID}.jpg') test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) test_storage_name = gem_name.GemName(file_name=TEST_FP_FILE) cadc_client_mock = Mock() clients_mock = Mock() clients_mock.data_client = cadc_client_mock kwargs = { 'working_directory': TEST_DATA_DIR, 'clients': clients_mock, 'stream': 'stream', 'observable': test_observable, 'storage_name': test_storage_name, } with patch('caom2pipe.manage_composable.http_get') as http_mock, patch( 'caom2pipe.manage_composable.data_put') as ad_put_mock, patch( 'caom2pipe.manage_composable.get_artifact_metadata' ) as art_mock, patch( 'caom2pipe.manage_composable.exec_cmd') as exec_mock: cadc_client_mock.return_value.data_get.return_value = ( mc.CadcException('test')) obs = preview_augmentation.visit(obs, **kwargs) assert not http_mock.called, 'http mock should not be called' assert not ad_put_mock.called, 'ad put mock should not be called' assert not art_mock.called, 'art mock should not be called' assert not exec_mock.called, 'exec mock should not be called' assert obs is not None, 'expect a result' assert (len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1 ), 'no new artifacts' test_rejected.persist_state() assert os.path.exists(REJECTED_FILE) finally: if os.path.exists(REJECTED_FILE): os.unlink(REJECTED_FILE)
def test_preview_augment_v(put_mock, http_mock): # this should result in two new artifacts being added to the plane # one for a thumbnail and one for a preview obs = mc.read_obs_from_file(TEST_OBS_FILE) obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow() assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition' test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_metrics = mc.Metrics(test_config) test_observable = mc.Observable(test_rejected, test_metrics) cadc_client_mock = Mock() kwargs = {'working_directory': '/test_files', 'cadc_client': cadc_client_mock, 'observable': test_observable} test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg' if os.path.exists(test_prev): os.unlink(test_prev) try: cadc_client_mock.return_value.data_get.return_value = \ mc.CadcException('test') http_mock.side_effect = _get_mock result = preview_v_augmentation.visit(obs, **kwargs) test_url = f'{preview_augmentation.PREVIEW_URL}' \ f'{TEST_PRODUCT_ID}.fits' assert http_mock.called, 'http mock should be called' http_mock.assert_called_with(test_url, test_prev), \ 'mock not called' assert put_mock.called, 'put mock not called' put_mock.assert_called_with( cadc_client_mock, '/test_files', 'GN2001BQ013-04_th.jpg', 'ad:GEM/GN2001BQ013-04_th.jpg', metrics=test_metrics) assert result is not None, 'expect a result' assert result['artifacts'] == 2, 'artifacts should be added' assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 3, \ 'two new artifacts' prev_uri = mc.build_uri( ARCHIVE, f'{TEST_PRODUCT_ID}.jpg', SCHEME) thumb_uri = mc.build_uri(ARCHIVE, f'{TEST_PRODUCT_ID}_th.jpg') assert prev_uri in obs.planes[TEST_PRODUCT_ID].artifacts.keys(), \ 'no preview' assert thumb_uri in obs.planes[TEST_PRODUCT_ID].artifacts, \ 'no thumbnail' finally: if os.path.exists(test_prev): os.unlink(test_prev)
def test_pull_augmentation(http_mock, json_mock, header_mock, file_type_mock): obs = mc.read_obs_from_file(TEST_OBS_AD_URI_FILE) obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow() original_uri = 'gemini:GEMINI/GN2001BQ013-04.fits' assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition' assert ( original_uri in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'initial condition' test_uri = f'{SCHEME}:{COLLECTION}/{TEST_PRODUCT_ID}.fits' test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) cadc_client_mock = Mock() cadc_client_mock.return_value.info.return_value = None clients_mock = Mock() clients_mock.data_client = cadc_client_mock json_mock.side_effect = gem_mocks.mock_retrieve_json filter_cache = svofps.FilterMetadataCache(Mock()) test_reader = gemini_metadata.GeminiFileMetadataReader( Mock(), Mock(), filter_cache) test_fqn = f'{gem_mocks.TEST_DATA_DIR}/GMOS/GN2001BQ013-04.fits.header' test_storage_name = gem_name.GemName(file_name='GN2001BQ013-04.fits') header_mock.side_effect = gem_mocks._mock_headers file_type_mock.return_values = 'application/fits' test_reader.set(test_storage_name) kwargs = { 'working_directory': TEST_DATA_DIR, 'clients': clients_mock, 'observable': test_observable, 'metadata_reader': test_reader, 'storage_name': test_storage_name, } obs = pull_augmentation.visit(obs, **kwargs) test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits' test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits' http_mock.assert_called_with(test_url, test_prev), 'mock not called' assert cadc_client_mock.put.called, 'put mock not called' cadc_client_mock.put.assert_called_with( TEST_DATA_DIR, 'gemini:GEMINI/GN2001BQ013-04.fits'), 'wrong put args' assert obs is not None, 'expect a result' assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'no new artifacts' try: ignore = obs.planes[TEST_PRODUCT_ID].artifacts[test_uri] except KeyError as ke: # because CAOM does magic result = obs.planes[TEST_PRODUCT_ID].artifacts[original_uri] assert result.uri == test_uri, f'wrong uri {result.uri}'
def test_cadc_transfer(client_mock, caps_mock): caps_mock.return_value = 'https://sc2.canfar.net/sc2repo' test_subject = tc.CadcTransfer() assert test_subject is not None, 'expect a result' test_config = mc.Config() test_config.rejected_fqn = '/tmp/rejected.yml' test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config)) test_subject.observable = test_observable test_subject.cadc_client = client_mock test_source = 'ad:TEST/test_file.fits' test_destination = '/tmp/test_file.fits' test_subject.get(test_source, test_destination) assert client_mock.get.called, 'should have been called' client_mock.get.assert_called_with( '/tmp', 'ad:TEST/test_file.fits'), 'wrong parameters'
def _init(self, config): if mc.TaskType.SCRAPE in config.task_types: self._logger.info( f'SCRAPE\'ing data - no clients will be initialized.' ) else: subject = define_subject(config) self._metadata_client = CAOM2RepoClient( subject, config.logging_level, config.resource_id ) self._data_client = declare_client(config) if config.tap_id is not None: self._query_client = CadcTapClient( subject=subject, resource_id=config.tap_id ) self._metrics = mc.Metrics(config)
def test_preview_augment_unknown_no_preview(): # what happens when it's not known that there's no preview obs = mc.read_obs_from_file(TEST_OBS_FILE) obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow() assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition' # make sure the rejected file is empty if os.path.exists(REJECTED_FILE): os.unlink(REJECTED_FILE) test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits') cadc_client_mock = Mock() clients_mock = Mock() clients_mock.data_client = cadc_client_mock kwargs = { 'working_directory': TEST_DATA_DIR, 'clients': clients_mock, 'stream': 'stream', 'observable': test_observable, 'storage_name': test_storage_name, } with patch( 'caom2pipe.manage_composable.http_get', side_effect=mc.CadcException( 'Not Found for url: https://archive.gemini.edu/preview'), ) as http_mock, patch( 'caom2pipe.manage_composable.data_put') as ad_put_mock, patch( 'caom2pipe.manage_composable.get_artifact_metadata' ) as art_mock, patch( 'caom2pipe.manage_composable.exec_cmd') as exec_mock: cadc_client_mock.get.side_effect = exceptions.UnexpectedException( 'test') obs = preview_augmentation.visit(obs, **kwargs) assert obs is not None, 'expect result' test_url = f'{preview_augmentation.PREVIEW_URL}{TEST_PRODUCT_ID}.fits' test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.jpg' http_mock.assert_called_with(test_url, test_prev), 'mock not called' assert not ad_put_mock.called, 'ad put mock should not be called' assert not art_mock.called, 'art mock should not be called' assert not exec_mock.called, 'exec mock should not be called'
def test_cadc_transfer(data_get_mock, caps_mock): caps_mock.return_value = 'https://sc2.canfar.net/sc2repo' data_get_mock.side_effect = Mock(autospec=True) test_subject = tc.CadcTransfer() assert test_subject is not None, 'expect a result' test_config = mc.Config() test_config.rejected_fqn = '/tmp/rejected.yml' test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config)) test_subject.observable = test_observable test_subject.cadc_client = Mock() test_source = 'ad:TEST/test_file.fits' test_destination = '/tmp/test_file.fits' test_subject.get(test_source, test_destination) assert data_get_mock.called, 'should have been called' args, kwargs = data_get_mock.call_args assert args[1] == '/tmp', 'wrong dir name' assert args[2] == 'test_file.fits', 'wrong file name' assert args[3] == 'TEST', 'wrong archive name'
def test_preview_augmentation(access_mock): access_mock.return_value = 'https://localhost' test_fqn = f'{TEST_DATA_DIR}/preview_augmentation_start.xml' test_science_f_name = ( 'VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.' 'tt0.subim.fits') test_storage_name = sn.VlassName(test_science_f_name) test_obs = mc.read_obs_from_file(test_fqn) test_config = mc.Config() test_rejected = mc.Rejected(f'{TEST_DATA_DIR}/rejected.yml') test_metrics = mc.Metrics(test_config) test_observable = mc.Observable(test_rejected, test_metrics) kwargs = { 'stream': None, 'observable': test_observable, 'storage_name': test_storage_name, 'working_directory': '/test_files', } test_subject = preview_augmentation.VlassPreview(**kwargs) assert test_subject is not None, 'need a test subject' assert len(test_obs.planes) == 1, 'wrong number of planes' assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 4 ), 'wrong starting # of artifacts' test_obs = test_subject.visit(test_obs) assert test_obs is not None, 'expect a result' assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 6 ), 'wrong ending # of artifacts' test_report = test_subject.report assert test_report is not None assert 'artifacts' in test_report assert test_report.get('artifacts') == 2, 'wrong report count' # does artifact re-naming work? test_url = (f'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.1/' f'T01t01/VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1/' f'{test_science_f_name}') kwargs = {'url': test_url} test_obs = cleanup_augmentation.visit(test_obs, **kwargs) test_artifacts = test_obs.planes[test_storage_name.product_id].artifacts assert len(test_artifacts) == 4, 'wrong ending conditions' assert test_storage_name.prev_uri in test_artifacts, 'missing preview' assert test_storage_name.thumb_uri in test_artifacts, 'missing thumbnail'
def test_preview_augmentation(data_client_mock, tap_mock): getcwd_orig = os.getcwd os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR) tap_mock.side_effect = _tap_mock data_client_mock.return_value.info.side_effect = ( test_main_app._get_file_info) test_f_id = 'rnN20140428S0181_ronchi' test_f_name = f'{test_f_id}.fits' test_obs = mc.read_obs_from_file( f'{test_main_app.TEST_DATA_DIR}/{test_f_id}.expected.xml') test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_config.get_executors() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) external_metadata.init_global(test_config) test_builder = builder.GemProcBuilder(test_config) test_fqn = os.path.join(test_main_app.TEST_DATA_DIR, test_f_name) test_storage_name = test_builder.build(test_fqn) kwargs = { 'working_directory': TEST_FILES_DIR, 'cadc_client': None, 'stream': 'stream', 'observable': test_observable, 'storage_name': test_storage_name, } try: start_ts = datetime.utcnow().timestamp() test_result = preview_augmentation.visit(test_obs, **kwargs) end_ts = datetime.utcnow().timestamp() logging.error(f'{test_f_name} execution time {end_ts - start_ts}') except Exception as e: logging.error(e) logging.error(traceback.format_exc()) assert False finally: os.getcwd = getcwd_orig assert test_result is not None, 'expect a result' assert test_result.get('artifacts') == 2, 'wrong result'
def _update_ngvs_time(chunk, provenance, obs_id): logging.debug(f'Begin _update_ngvs_time for {obs_id}') if (chunk is not None and provenance is not None and len(provenance.inputs) > 0): # bounds = ctor config = mc.Config() config.get_executors() subject = mc.define_subject(config) client = CAOM2RepoClient( subject, config.logging_level, 'ivo://cadc.nrc.ca/ams') metrics = mc.Metrics(config) bounds = CoordBounds1D() min_date = 0 max_date = sys.float_info.max exposure = 0 for entry in provenance.inputs: ip_obs_id, ip_product_id = mc.CaomName.decompose_provenance_input( entry.uri) logging.info(f'Retrieving provenance metadata for {ip_obs_id}.') ip_obs = mc.repo_get(client, 'CFHT', ip_obs_id, metrics) if ip_obs is not None: ip_plane = ip_obs.planes.get(ip_product_id) if (ip_plane is not None and ip_plane.time is not None and ip_plane.time.bounds is not None): bounds.samples.append(CoordRange1D( RefCoord(pix=0.5, val=ip_plane.time.bounds.lower), RefCoord(pix=1.5, val=ip_plane.time.bounds.upper))) min_date = min(ip_plane.time.bounds.lower, min_date) max_date = max(ip_plane.time.bounds.upper, max_date) exposure += ip_plane.time.exposure axis = Axis(ctype='TIME', cunit='d') time_axis = CoordAxis1D(axis=axis, error=None, range=None, bounds=bounds, function=None) temporal_wcs = TemporalWCS(axis=time_axis, timesys=None, trefpos=None, mjdref=None, exposure=mc.to_float(exposure), resolution=None) chunk.time = temporal_wcs logging.debug(f'End _update_ngvs_time.')
def test_repo_delete(mock_client): test_config = mc.Config() test_config.observe_execution = True test_metrics = mc.Metrics(test_config) assert len(test_metrics.history) == 0, 'initial history conditions' assert len(test_metrics.failures) == 0, 'initial failure conditions' mc.repo_delete(mock_client, 'coll', 'test_id', test_metrics) mock_client.delete.assert_called_with('coll', 'test_id'), 'mock not called' assert len(test_metrics.history) == 1, 'history conditions' assert len(test_metrics.failures) == 0, 'failure conditions' assert 'caom2' in test_metrics.history, 'history' assert 'delete' in test_metrics.history['caom2'], 'delete' assert 'test_id' in test_metrics.history['caom2']['delete'], 'obs id' mock_client.reset_mock() mock_client.delete.side_effect = Exception('boo') with pytest.raises(mc.CadcException): mc.repo_delete(mock_client, 'coll', 'test_id', test_metrics) assert len(test_metrics.failures) == 1, 'should have failure counts'
def test_http_transfer(get_mock): test_source = 'http://localhost/test_file.fits' test_destination = '/tmp/test_file.fits' if not os.path.exists(test_destination): with open(test_destination, 'w') as f: f.write('test content') get_mock.side_effect = Mock(autospec=True) test_config = mc.Config() test_config.working_directory = test_conf.TEST_DATA_DIR test_config.netrc_file = 'test_netrc' test_config.rejected_fqn = '/tmp/rejected.yml' test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config)) test_subject = tc.HttpTransfer() assert test_subject is not None, 'expect a result' test_subject.observable = test_observable with pytest.raises(mc.CadcException): test_subject.get(test_source, test_destination) assert get_mock.called, 'should have been called' args, kwargs = get_mock.call_args assert args[1] == test_source, 'wrong source name' assert args[2] == test_destination, 'wrong dest name'
def test_repo_update(mock_client): test_obs = mc.read_obs_from_file(TEST_OBS_FILE) test_config = mc.Config() test_config.observe_execution = True test_metrics = mc.Metrics(test_config) assert len(test_metrics.history) == 0, 'initial history conditions' assert len(test_metrics.failures) == 0, 'initial failure conditions' mc.repo_update(mock_client, test_obs, test_metrics) mock_client.update.assert_called_with(test_obs), 'mock not called' assert len(test_metrics.history) == 1, 'history conditions' assert len(test_metrics.failures) == 0, 'failure conditions' assert 'caom2' in test_metrics.history, 'history' assert 'update' in test_metrics.history['caom2'], 'update' assert 'test_obs_id' in test_metrics.history['caom2']['update'], 'obs id' mock_client.reset_mock() mock_client.update.side_effect = Exception('boo') with pytest.raises(mc.CadcException): mc.repo_update(mock_client, test_obs, test_metrics) assert len(test_metrics.failures) == 1, 'should have failure counts'
def test_preview_augment_unknown_no_preview(): # what happens when it's not known that there's no preview obs = mc.read_obs_from_file(TEST_OBS_FILE) obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow() assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition' # make sure the rejected file is empty if os.path.exists(REJECTED_FILE): os.unlink(REJECTED_FILE) test_rejected = mc.Rejected(REJECTED_FILE) test_config = mc.Config() test_observable = mc.Observable(test_rejected, mc.Metrics(test_config)) cadc_client_mock = Mock() kwargs = {'working_directory': TEST_DATA_DIR, 'cadc_client': cadc_client_mock, 'stream': 'stream', 'observable': test_observable} with patch('caom2pipe.manage_composable.http_get', side_effect=mc.CadcException( 'Internal Server Error for url: ' 'https://archive.gemini.edu/preview')) as http_mock, \ patch('caom2pipe.manage_composable.data_put') as ad_put_mock, \ patch('caom2pipe.manage_composable.get_artifact_metadata') as \ art_mock, \ patch('caom2pipe.manage_composable.exec_cmd') as exec_mock: cadc_client_mock.return_value.data_get.return_value = mc.CadcException( 'test') result = preview_augmentation.visit(obs, **kwargs) assert result is not None, 'expect result' # 0 because the preview artifact doesn't already exist assert result['artifacts'] == 0, 'wrong result' test_url = f'{preview_augmentation.PREVIEW_URL}{TEST_PRODUCT_ID}.fits' test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.jpg' http_mock.assert_called_with(test_url, test_prev), 'mock not called' assert not ad_put_mock.called, 'ad put mock should not be called' assert not art_mock.called, 'art mock should not be called' assert not exec_mock.called, 'exec mock should not be called'