def _mock_file_info(uri): sci_result = FileInfo(id=uri, lastmod='Fri, 28 Dec 2018 01:43:28 GMT') reject_result = FileInfo(id=uri, lastmod='Thu, 14 May 2020 20:29:02 GMT') result = reject_result if '_SCI' in uri: result = sci_result return result
def test_visitor(test_name): storage_name = OmmName(file_name=basename(test_name).replace( '.header', '.gz'), ) file_info = FileInfo(id=storage_name.file_uri, file_type='application/fits') headers = ac.make_headers_from_file(test_name) metadata_reader = rdc.FileMetadataReader() metadata_reader._headers = {storage_name.file_uri: headers} metadata_reader._file_info = {storage_name.file_uri: file_info} kwargs = { 'storage_name': storage_name, 'metadata_reader': metadata_reader, } observation = None input_file = f'{TEST_DATA_DIR}/in.{storage_name.product_id}.fits.xml' if exists(input_file): observation = mc.read_obs_from_file(input_file) observation = fits2caom2_augmentation.visit(observation, **kwargs) expected_fqn = (f'{TEST_DATA_DIR}/{storage_name.file_id}.expected.xml') expected = mc.read_obs_from_file(expected_fqn) compare_result = get_differences(expected, observation) if compare_result is not None: actual_fqn = expected_fqn.replace('expected', 'actual') mc.write_obs_to_file(observation, actual_fqn) compare_text = '\n'.join([r for r in compare_result]) msg = (f'Differences found in observation {expected.observation_id}\n' f'{compare_text}') raise AssertionError(msg)
def test_todo_local_common( caom_mock, access_mock, fits2caom2_mock, test_input_name, ): if test_input_name not in ['OMM_TODO_LOCAL', 'NEOSSAT_TODO_LOCAL']: return test_input = INPUTS.get(test_input_name) _cleanup() _setup(test_input, local=True) access_mock.return_value = 'https://localhost' getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') caom_mock.return_value.data_client.info.side_effect = [ None, FileInfo( id=test_input.test_uri, md5sum='3d29f0edd984065a044d1376a11c6f08', ), ] caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', test_input.collection, Algorithm(name='exposure'), ), ] def _info(uri): assert (uri == test_input.test_uri), 'wrong info uri' return FileInfo( id=uri, md5sum='abc', size=42, ) fits2caom2_mock.return_value.info.side_effect = _info try: test_result = test_module._run() assert test_result is not None, f'expect a result {test_input_name}' assert test_result == 0, f'wrong test result {test_input_name}' assert (caom_mock.return_value.data_client.put.called ), f'{test_input_name} put not called' caom_mock.return_value.data_client.put.assert_called_with( '/usr/src/app/integration_test/mock_test/data/test_files', test_input.test_uri, None, ), f'{test_input_name} wrong put args' finally: os.getcwd = getcwd_orig
def _mock_get_file_info(file_id): return FileInfo( id=file_id, size=10290, md5sum='{}'.format(md5('-37'.encode()).hexdigest()), file_type='image/jpeg', )
def info(self, uri): """ Retrieve the descriptive metdata associated with a file. :param uri: str that is an Artifact URI, representing the file for which to retrieve metadata :return: cadcdata.FileInfo instance, no scheme for md5sum """ self._logger.debug(f'Begin info for {uri}') try: if self._use_si: result = self._cadc_client.cadcinfo(uri) # make the result look like the other possible ways to # obtain metadata result.md5sum = result.md5sum.replace('md5:', '') else: archive, f_name = StorageClientWrapper._decompose(uri) temp = self._cadc_client.get_file_info(archive, f_name) result = FileInfo( id=uri, size=temp.get('size'), file_type=temp.get('type'), md5sum=temp.get('md5sum').replace('md5:', '') ) except exceptions.NotFoundException: self._logger.info(f'cadcinfo:: {uri} not found') result = None self._logger.debug('End info') return result
def _get_uris(collection, fnames, obs): # NOTE: this function makes the assumption that the collection is # the same with the AD archive, which in many cases is not true (CFHT # for example has multiple collections) uris = [] file_meta = {} if fnames: for fname in fnames: f = os.path.basename(fname).replace('.header', '') for p in obs.planes.values(): for a in p.artifacts.values(): if (f'ad:{collection}/{f}' in a.uri or (a.uri.startswith('vos') and f in a.uri)): uris.append(a.uri) meta = FileInfo(id=a.uri, file_type=a.content_type, size=a.content_length, md5sum=a.content_checksum.checksum) file_url = urlparse(a.uri) if file_url.scheme not in ['ad', 'vos', 'cadc']: # TODO add hook to support other service providers raise NotImplementedError( 'Only ad, vos type URIs supported') archive, file_id = file_url.path.split('/')[-2:] file_meta[(archive, file_id)] = meta return uris, file_meta else: return None
def _info(uri): assert uri == 'ad:CFHT/2460606o.fits.gz', 'wrong info uri' return FileInfo( id=uri, md5sum='abc', size=42, )
def _info(uri): assert (uri == 'gemini:GEMINI/S20191214S0301.fits'), 'wrong info uri' return FileInfo( id=uri, md5sum='abc', size=42, )
def _info(uri): assert (uri == test_input.test_uri), 'wrong info uri' return FileInfo( id=uri, md5sum='abc', size=42, )
def mock_get_file_info(file_id): if isinstance(file_id, GemName): # the case if StorageClientReader is being mocked file_id = file_id.source_names[0] if '_prev' in file_id: return FileInfo( size=10290, md5sum='{}'.format(md5(b'-37').hexdigest()), file_type='image/jpeg', id=file_id, ) else: return FileInfo( size=665345, md5sum='a347f2754ff2fd4b6209e7566637efad', file_type='application/fits', id=file_id, )
def test_time_nan(): test_obs = 'C120712_NGC7790_H_SCIRED' test_file = f'file://{TESTDATA_DIR}/{test_obs}.fits.header' omm_name = OmmName(file_name=f'{test_obs}.fits', entry=test_file) test_file_info = FileInfo(id=omm_name.file_uri) test_xml = f'{TESTDATA_DIR}/{test_obs}.xml' obs = mc.read_obs_from_file(test_xml) headers = data_util.get_local_file_headers(test_file) telescope = Telescope(omm_name.file_uri, headers) with pytest.raises(mc.CadcException): result = telescope.update(obs, omm_name, test_file_info) assert result is None, 'should have returned nothing'
def get_local_file_info(fqn): """ Gets descriptive metadata for a file on disk. :param fqn: Fully-qualified name of the file on disk. :return: FileInfo """ s = stat(fqn) meta = FileInfo( id=path.basename(fqn), size=s.st_size, md5sum=md5(open(fqn, 'rb').read()).hexdigest(), file_type=get_file_type(fqn), ) return meta
def add_file_info_record(self, uri): """add_json_record has already been successfully called""" if uri not in self._file_info.keys(): record = self._json_metadata[uri] self._file_info[uri] = FileInfo( id=uri, size=record.get('data_size'), name=record.get('filename'), md5sum=record.get('data_md5'), lastmod=mc.make_time_tz(record.get('lastmod')), file_type=data_util.get_file_type(record.get('filename')), encoding=data_util.get_file_encoding(record.get('filename')), ) self._logger.debug(f'Adding FileInfo for {uri}')
def vault_info(client, uri): """ Translate Node properties into FileInfo. :param client: Vault client :param uri: VOS URI :return: an instance of FileInfo """ node = client.get_node(uri, limit=None, force=False) return FileInfo( id=uri, size=node.props.get('length'), md5sum=node.props.get('MD5'), lastmod=node.props.get('lastmod'), )
def test_failure(): test_fqn = ( f'{test_fits2caom2_augmentation.TEST_DATA_DIR}/broken_data/' f'dao_c122_2001_006946.fits.header' ) dao_name = DAOName(basename(test_fqn).replace('.header', '.gz')) file_info = FileInfo(id=dao_name.file_uri, file_type='application/fits') headers = ac.make_headers_from_file(test_fqn) metadata_reader = rdc.FileMetadataReader() metadata_reader._headers = {dao_name.file_uri: headers} metadata_reader._file_info = {dao_name.file_uri: file_info} kwargs = { 'storage_name': dao_name, 'metadata_reader': metadata_reader, } observation = None observation = fits2caom2_augmentation.visit(observation, **kwargs) assert observation is None, 'failure, no observation creation'
def get_local_file_info(fqn): """ Gets descriptive metadata for a file on disk. :param fqn: Fully-qualified name of the file on disk. :return: FileInfo, no scheme on the md5sum value. """ s = stat(fqn) # copy and paste from cadcdata/storageinventory.py hash_md5 = md5() with open(fqn, 'rb') as f: for chunk in iter(lambda: f.read(4096), b''): hash_md5.update(chunk) meta = FileInfo( id=path.basename(fqn), size=s.st_size, md5sum=hash_md5.hexdigest(), file_type=get_file_type(fqn), ) return meta
def test_look_pull_and_put(http_mock, mock_client): test_storage_name = 'cadc:GEMINI/TEST.fits' mock_client.info.return_value = FileInfo( id=test_storage_name, size=1234, md5sum='9473fdd0d880a43c21b7778d34872157', ) f_name = 'TEST.fits' url = f'https://localhost/{f_name}' test_config = mc.Config() test_config.observe_execution = True mock_client.info.return_value = None test_fqn = os.path.join(TEST_DATA_DIR, f_name) pull_augmentation.look_pull_and_put( test_storage_name, test_fqn, url, mock_client, 'md5:01234', ) mock_client.data_client.put.assert_called_with( TEST_DATA_DIR, test_storage_name), 'mock not called' http_mock.assert_called_with(url, test_fqn), 'http mock not called'
def test_store_newer_files_only_flag(client_mock, test_config): # first test case # flag set to True, file is older at CADC, supports_latest_client = False test_config.working_directory = tc.TEST_DATA_DIR test_config.use_local_files = True test_config.features.supports_latest_client = False test_f_name = '1000003f.fits.fz' sn = [os.path.join('/caom2pipe_test', test_f_name)] du = [f'cadc:TEST/{test_f_name}'] test_sn = FlagStorageName(test_f_name, sn, du) observable_mock = Mock(autospec=True) client_mock.info.return_value = FileInfo( id=du[0], lastmod='Mon, 4 Mar 2019 19:05:41 GMT') test_subject = ec.LocalStore( test_config, test_sn, 'TEST_STORE', client_mock, observable_mock, ) test_subject.execute(None) assert client_mock.put.called, 'expect put call'
def test_store_newer_files_only_flag_client(client_mock, test_config): # just like the previous test, except supports_latest_client = True # first test case # flag set to True, file is older at CADC, supports_latest_client = False test_config.working_directory = tc.TEST_DATA_DIR test_config.use_local_files = True test_config.features.supports_latest_client = True test_f_name = '1000003f.fits.fz' sn = [os.path.join('/caom2pipe_test', test_f_name)] du = [f'cadc:TEST/{test_f_name}'] test_sn = FlagStorageName(test_f_name, sn, du) observable_mock = Mock(autospec=True) client_mock.cadcinfo.return_value = FileInfo( id=du[0], md5sum='d41d8cd98f00b204e9800998ecf8427e') test_subject = ec.LocalStore( test_config, test_sn, 'TEST_STORE', client_mock, observable_mock, ) test_subject.execute(None) assert client_mock.put.called, 'expect copy call'
def test_si_client_get(mock_metrics, mock_client): test_fqn = f'{tc.TEST_FILES_DIR}/TEST.fits' if os.path.exists(test_fqn): os.unlink(test_fqn) test_source = 'gemini:GEMINI/TEST.fits' test_fqn = os.path.join(tc.TEST_FILES_DIR, 'TEST.fits') mock_client.cadcget.side_effect = tc.mock_si_get mock_client.cadcinfo.return_value = FileInfo( test_source, md5sum='9473fdd0d880a43c21b7778d34872157') clc.si_client_get( mock_client, test_fqn, test_source, metrics=mock_metrics, ) mock_client.cadcget.assert_called_with(test_source, dest=test_fqn), 'mock not called' assert mock_metrics.observe.called, 'mock not called' args, kwargs = mock_metrics.observe.call_args assert args[2] == 12, 'wrong size' assert args[3] == 'cadcget', 'wrong endpoint' assert args[4] == 'si', 'wrong service' assert args[5] == 'TEST.fits', 'wrong id'
def test_dao_transfer_check_fits_verify(vault_info_mock): test_match_file_info = FileInfo( id='vos:abc/def.fits', md5sum='ghi', ) test_different_file_info = FileInfo( id='vos:abc/def.fits', md5sum='ghi', ) test_file_info = [test_match_file_info, test_different_file_info] test_data_client = Mock(autospec=True) test_vos_client = Mock(autospec=True) test_config = mc.Config() test_config.data_source_extensions = ['.fits.gz'] test_config.data_sources = ['vos:DAO/Archive/Incoming'] test_config.cleanup_failure_destination = 'vos:DAO/failure' test_config.cleanup_success_destination = 'vos:DAO/success' test_config.recurse_data_sources = False def _mock_listdir(entry): if entry.endswith('Incoming'): return [ 'dao123.fits.gz', 'dao456.fits', 'Yesterday', '.dot.fits.gz' ] else: return [] test_vos_client.listdir.side_effect = _mock_listdir test_vos_client.isdir.side_effect = [ False, False, True, False, False, False, True, False ] vault_info_mock.return_value = test_file_info test_data_client.info.return_value = test_file_info for case in [True, False]: test_config.cleanup_files_when_storing = case test_subject = data_source.DAOVaultDataSource(test_config, test_vos_client, test_data_client) assert test_subject is not None, 'expect ctor to work' test_result = test_subject.get_work() assert test_result is not None, 'expect a work list' assert len(test_result) == 1, 'wrong work list entries' assert (test_result[0] == 'vos:DAO/Archive/Incoming/dao123.fits.gz' ), 'wrong work entry' assert test_vos_client.isdir.call_count == 4, 'wrong is_dir count' test_vos_client.isdir.reset_mock() # test the case when the md5sums are the same, so the transfer does # not occur, but the file ends up in the success location test_vos_client.isdir.side_effect = [False, False, True, False] test_config.cleanup_files_when_storing = True test_config.store_modified_files_only = True vault_info_mock.return_value = test_match_file_info test_data_client.info.return_value = test_different_file_info test_vos_client.status.raises = exceptions.NotFoundException second_test_subject = data_source.DAOVaultDataSource( test_config, test_vos_client, test_data_client) assert second_test_subject is not None, 'second ctor fails' second_test_result = second_test_subject.get_work() assert second_test_result is not None, 'expect a second result' assert len(second_test_result) == 0, 'should be no successes' assert test_vos_client.move.called, 'expect a success move call' test_vos_client.move.assert_called_with( 'vos:DAO/Archive/Incoming/dao123.fits.gz', 'vos:DAO/success/dao123.fits.gz', ), 'wrong success move args' assert test_vos_client.status.called, 'expect a status call'
def _file_info_mock(uri): return FileInfo( id=uri, file_type='application/fits', md5sum='md5:def', )
def _get_info(uri): assert uri == 'ad:DAO/dest_fqn.fits', f'wrong storage check {uri}' return FileInfo( id=uri, md5sum='ghi', )
def test_todo_local_move( caom_mock, access_mock, fits2caom2_mock, test_input_name, ): if 'MOVE' not in test_input_name: return test_input = INPUTS.get(test_input_name) _cleanup() new_dir = TEST_DATA_DIR / 'new' fail_dir = TEST_DATA_DIR / 'failure' success_dir = TEST_DATA_DIR / 'success' for entry in [new_dir, fail_dir, success_dir]: for listing in entry.iterdir(): listing.unlink() shutil.copy(test_input.test_file, new_dir / '2460606o.fits.gz') access_mock.return_value = 'https://localhost' # make sure the working directory TEST_EXEC_DIR has the correct things # in it config_file_target = TEST_EXEC_DIR / 'config.yml' shutil.copy(test_input.config_file, config_file_target) cache_file_target = TEST_EXEC_DIR / 'cache.yml' shutil.copy(test_input.cache_file, cache_file_target) with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f: f.write('test content') getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') caom_mock.return_value.data_client.info.side_effect = [ None, FileInfo( id='ad:CFHT/2460606o.fits.gz', md5sum='3d29f0edd984065a044d1376a11c6f08', ), ] caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', 'CFHT', Algorithm(name='exposure'), instrument=Instrument(name='ESPaDOnS'), ), ] def _info(uri): assert uri == 'ad:CFHT/2460606o.fits.gz', 'wrong info uri' return FileInfo( id=uri, md5sum='abc', size=42, ) fits2caom2_mock.return_value.info.side_effect = _info try: test_result = test_module._run_by_builder() assert test_result is not None, f'expect a result {test_input_name}' assert test_result == 0, f'wrong test result {test_input_name}' assert (caom_mock.return_value.data_client.put.called ), f'{test_input_name} put not called' caom_mock.return_value.data_client.put.assert_called_with( '/usr/src/app/integration_test/mock_test/data/test_files/new', 'ad:CFHT/2460606o.fits.gz', 'raw', ), f'{test_input_name} wrong put args' count = 0 for entry in new_dir.iterdir(): count += 1 assert count == 0, 'wrong new dir content' for entry in success_dir.iterdir(): count += 1 assert count == 1, 'wrong success dir content' count = 0 for entry in fail_dir.iterdir(): logging.error(f'fail entry {entry}') count += 1 assert count == 0, f'wrong fail dir content {count}' finally: os.getcwd = getcwd_orig del sys.modules['composable']
def test_run_state_v(client_mock): client_mock.metadata_client.read.side_effect = tc.mock_read client_mock.data_client.info.return_value = FileInfo( id='cadc:TEST/anything.fits', size=42, md5sum='9473fdd0d880a43c21b7778d34872157', ) test_wd = '/usr/src/app/caom2pipe/int_test' caom2pipe_bookmark = 'caom2_timestamp' test_config = mc.Config() test_config.working_directory = test_wd test_config.collection = 'TEST' test_config.interval = 10 test_config.log_file_directory = f'{test_wd}/logs' test_config.failure_fqn = ( f'{test_config.log_file_directory}/failure_log.txt') test_config.log_to_file = True test_config.logging_level = 'INFO' test_config.progress_file_name = 'progress.txt' test_config.proxy_file_name = f'{test_wd}/cadcproxy.pem' test_config.rejected_file_name = 'rejected.yml' test_config.rejected_directory = f'{test_wd}/rejected' test_config._report_fqn = ( f'{test_config.log_file_directory}/app_report.txt') test_config.resource_id = 'ivo://cadc.nrc.ca/sc2repo' test_config.retry_file_name = 'retries.txt' test_config.retry_fqn = ( f'{test_config.log_file_directory}/{test_config.retry_file_name}') test_config.state_file_name = 'state.yml' test_config.success_fqn = ( f'{test_config.log_file_directory}/success_log.txt') test_config.tap_id = 'ivo://cadc.nrc.ca/sc2tap' test_config.task_types = [ mc.TaskType.STORE, mc.TaskType.INGEST, mc.TaskType.MODIFY, ] test_config.features.use_file_names = True test_config.features.use_urls = False test_config.features.supports_latest_client = True test_config.use_local_files = False test_config.storage_inventory_resource_id = 'ivo://cadc.nrc.ca/test' if not os.path.exists(test_wd): os.mkdir(test_wd) test_start_time, test_end_time = _get_times(test_config, caom2pipe_bookmark) with open(test_config.proxy_fqn, 'w') as f: f.write('test content\n') test_data_source = TestListDirTimeBoxDataSource() test_builder = nbc.GuessingBuilder(tc.TestStorageName) transferrer = TestTransfer() try: test_result = rc.run_by_state( bookmark_name=caom2pipe_bookmark, command_name='collection2caom2', config=test_config, end_time=test_end_time, name_builder=test_builder, source=test_data_source, modify_transfer=transferrer, store_transfer=transferrer, clients=client_mock, ) assert test_result is not None, 'expect a result' assert test_result == 0, 'expect success' assert client_mock.data_client.put.called, 'expect put call' client_mock.data_client.put.assert_called_with( '/usr/src/app/caom2pipe/int_test/test_obs_id', 'cadc:TEST/test_file.fits.gz', None, ), 'wrong call args' # state file checking test_state = mc.State(test_config.state_fqn) assert test_state is not None, 'expect state content' test_checkpoint = test_state.get_bookmark(caom2pipe_bookmark) assert test_checkpoint == test_end_time, 'wrong bookmark' # success file testing assert os.path.exists(test_config.log_file_directory), 'log directory' assert os.path.exists(test_config.success_fqn), 'success fqn' assert os.path.exists(test_config.progress_fqn), 'progress fqn' log_file = f'{test_config.log_file_directory}/test_obs_id.log' actual = glob.glob(f'{test_config.log_file_directory}/**') assert os.path.exists(log_file), f'specific log file {actual}' xml_file = f'{test_config.log_file_directory}/test_obs_id.xml' assert os.path.exists(xml_file), f'xml file {actual}' # reporting testing report_file = f'{test_config.log_file_directory}/app_report.txt' assert os.path.exists(report_file), f'report file {actual}' pass_through_test = False with open(report_file, 'r') as f: for line in f: pass_through_test = True if 'Number' in line: bits = line.split(':') found = False if 'Inputs' in bits[0]: assert bits[1].strip() == '1', 'wrong inputs' found = True elif 'Successes' in bits[0]: assert bits[1].strip() == '1', 'wrong successes' found = True elif 'Timeouts' in bits[0]: assert bits[1].strip() == '0', 'wrong timeouts' found = True elif 'Retries' in bits[0]: assert bits[1].strip() == '0', 'wrong retries' found = True elif 'Errors' in bits[0]: assert bits[1].strip() == '0', 'wrong errors' found = True elif 'Rejections' in bits[0]: assert bits[1].strip() == '0', 'wrong rejections' found = True assert found, f'{line}' assert pass_through_test, 'found a report file and checked it' finally: f_list = glob.glob(f'{test_wd}/**', recursive=True) for entry in f_list: try: if os.path.isdir(entry): os.rmdir(entry) else: os.unlink(entry) except OSError as e: logging.error(f'failed to delete {e}')
def info_si_mock(ignore): return FileInfo(id=test_uri, file_type='application/fits', md5sum='abc', size=42)
def _info(uri): return FileInfo( id=uri, md5sum='abc', size=42, )
def _info_mock(uri): return FileInfo(id=uri, size=12, file_type='application/fits', md5sum='abc')
def _get_file_info(uri): return FileInfo(id=uri, file_type='application/fits')
def test_todo_vos( vo_mock, caom_mock, access_mock, transfer_mock, fits2caom2_mock, test_input_name, ): if 'VOS' not in test_input_name: return test_input = INPUTS.get(test_input_name) _cleanup() access_mock.return_value = 'https://localhost' # make sure the working directory TEST_EXEC_DIR has the correct things # in it config_file_target = TEST_EXEC_DIR / 'config.yml' shutil.copy(test_input.config_file, config_file_target) with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f: f.write('test content') # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') reload(test_module) logging.error(dir(test_module)) def _vo_listdir(entry): assert entry == 'vos:goliaths/test', 'wrong parameter' return ['a2021_08_17_19_30_01.fits.gz'] vo_mock.return_value.listdir.side_effect = _vo_listdir vo_mock.return_value.isdir.return_value = False def _vo_get_node(uri, limit=None, force=False): assert (uri == 'vos:goliaths/test/a2021_08_17_19_30_01.fits.gz' ), f'wrong get node uri {uri}' test_start_time = (datetime.now(tz=dateutil.tz.UTC) - timedelta(minutes=5)) node = type('', (), {})() node.props = { 'length': 42, 'MD5': '1234', 'lastmod': test_start_time.isoformat(), } return node vo_mock.return_value.get_node.side_effect = _vo_get_node caom_mock.return_value.data_client.info.side_effect = [ None, FileInfo(id='ad:DAO/a2021_08_17_19_30_01.fits.gz'), ] caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', 'DAO', Algorithm(name='exposure'), ), ] def _transfer_get(src, dst): logging.error(src) logging.error(dst) assert (src == 'vos:goliaths/test/a2021_08_17_19_30_01.fits.gz' ), 'wrong source' assert ( dst == '/usr/src/app/integration_test/mock_test/data/execution/' 'a2021_08_17_19_30_01/a2021_08_17_19_30_01.fits.gz'), 'wrong dst' with open(dst, 'w') as f2: f2.write('test content') transfer_mock.return_value.get.side_effect = _transfer_get def _get_head(ignore): x = """SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017 BITPIX = -32 / Bits per pixel NAXIS = 2 / Number of dimensions NAXIS1 = 2048 / NAXIS2 = 2048 / EXPTIME = 1.23 NCOMBINE= 1 DATATYPE= 'REDUC ' /Data type, SCIENCE/CALIB/REJECT/FOCUS/TEST END """ delim = '\nEND' extensions = \ [e + delim for e in x.split(delim) if e.strip()] headers = [fits.Header.fromstring(e, sep='\n') for e in extensions] return headers fits2caom2_mock.return_value.get_head.side_effect = _get_head def _info(uri): return FileInfo( id=uri, md5sum='abc', size=42, ) fits2caom2_mock.return_value.info.side_effect = _info getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) try: test_result = test_module._run_vo() assert test_result is not None, f'expect a result {test_input_name}' assert test_result == 0, f'wrong test result {test_input_name}' assert (caom_mock.return_value.data_client.put.called ), f'{test_input_name} put not called' caom_mock.return_value.data_client.put.assert_called_with( '/usr/src/app/integration_test/mock_test/data/execution/' 'a2021_08_17_19_30_01', 'ad:DAO/a2021_08_17_19_30_01.fits.gz', 'raw', ), f'{test_input_name} wrong put args' finally: os.getcwd = getcwd_orig del sys.modules['composable']