def test_augment_observation(test_file, test_file_uri): test_fitsparser = FitsParser(test_file) test_obs = Observation('collection', 'observation_id', Algorithm('algorithm')) test_fitsparser.augment_observation(test_obs, test_file_uri) assert test_obs is not None assert test_obs.planes is not None assert len(test_obs.planes) == 1 test_plane = test_obs.planes['HI-line'] assert test_plane.artifacts is not None assert len(test_plane.artifacts) == 1 test_artifact = test_plane.artifacts[test_file_uri] assert test_artifact is not None test_part = test_artifact.parts['0'] # remove the chunk bit, as it's part of other tests - # results in <caom2:chunks/> xml output test_part.chunks.pop() # set the ids to expected values test_obs._id = uuid.UUID('00000000000000001234567812345678') test_plane._id = uuid.UUID('00000000000000001234567812345678') test_artifact._id = uuid.UUID('00000000000000001234567812345678') test_part._id = uuid.UUID('00000000000000001234567812345678') output = BytesIO() ow = ObservationWriter(False, False, "caom2", obs_reader_writer.CAOM20_NAMESPACE) ow.write(test_obs, output) result = output.getvalue().decode('UTF-8') output.close() assert result == EXPECTED_OBS_XML # , result
def _cmd_direct_mock(): from caom2 import SimpleObservation, Algorithm obs = SimpleObservation(observation_id='VLASS1.2.T07t13.J083838-153000', collection=COLLECTION, algorithm=Algorithm(name='testing')) mc.write_obs_to_file( obs, '/usr/src/app/logs/VLASS1.2.T07t13.J083838-153000.fits.xml')
def test_run_todo_file_data_source(clients_mock, test_config): clients_mock.return_value.data_client.get_file_info.return_value = None clients_mock.return_value.metadata_client.read.return_value = ( SimpleObservation( collection=test_config.collection, observation_id='def', algorithm=Algorithm(str('test')), )) if os.path.exists(test_config.success_fqn): os.unlink(test_config.success_fqn) test_config.work_fqn = f'{TEST_DIR}/todo.txt' test_config.task_types = [mc.TaskType.VISIT] test_config.log_to_file = True test_chooser = ec.OrganizeChooser() test_result = rc.run_by_todo(config=test_config, chooser=test_chooser, command_name=TEST_COMMAND) assert test_result is not None, 'expect a result' assert test_result == 0, 'expect success' assert os.path.exists(test_config.success_fqn), 'expect success file' with open(test_config.success_fqn) as f: content = f.read() # the obs id and file name assert 'def def.fits' in content, 'wrong success message'
def test_run_todo_file_data_source_v( repo_read_mock, set_clients_mock, test_config ): set_clients_mock.side_effect = _clients_mock test_config.features.supports_latest_client = True test_cert_file = os.path.join(TEST_DIR, 'test_proxy.pem') test_config.proxy_fqn = test_cert_file repo_read_mock.return_value = SimpleObservation( collection=test_config.collection, observation_id='def', algorithm=Algorithm(str('test')), ) if os.path.exists(test_config.success_fqn): os.unlink(test_config.success_fqn) test_config.work_fqn = f'{TEST_DIR}/todo.txt' test_config.task_types = [mc.TaskType.VISIT] test_config.log_to_file = True test_chooser = ec.OrganizeChooser() test_result = rc.run_by_todo( config=test_config, chooser=test_chooser, command_name=TEST_COMMAND ) assert test_result is not None, 'expect a result' assert test_result == 0, 'expect success' assert os.path.exists(test_config.success_fqn), 'expect success file' with open(test_config.success_fqn) as f: content = f.read() # the obs id and file name assert 'def def.fits' in content, 'wrong success message' assert repo_read_mock.called, 'expect e call' repo_read_mock.assert_called_with(), 'wrong e args'
def test_todo_local_common( caom_mock, access_mock, fits2caom2_mock, test_input_name, ): if test_input_name not in ['OMM_TODO_LOCAL', 'NEOSSAT_TODO_LOCAL']: return test_input = INPUTS.get(test_input_name) _cleanup() _setup(test_input, local=True) access_mock.return_value = 'https://localhost' getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') caom_mock.return_value.data_client.info.side_effect = [ None, FileInfo( id=test_input.test_uri, md5sum='3d29f0edd984065a044d1376a11c6f08', ), ] caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', test_input.collection, Algorithm(name='exposure'), ), ] def _info(uri): assert (uri == test_input.test_uri), 'wrong info uri' return FileInfo( id=uri, md5sum='abc', size=42, ) fits2caom2_mock.return_value.info.side_effect = _info try: test_result = test_module._run() assert test_result is not None, f'expect a result {test_input_name}' assert test_result == 0, f'wrong test result {test_input_name}' assert (caom_mock.return_value.data_client.put.called ), f'{test_input_name} put not called' caom_mock.return_value.data_client.put.assert_called_with( '/usr/src/app/integration_test/mock_test/data/test_files', test_input.test_uri, None, ), f'{test_input_name} wrong put args' finally: os.getcwd = getcwd_orig
def _write_obs_mock(): args = get_gen_proc_arg_parser().parse_args() obs = SimpleObservation( collection=args.observation[0], observation_id=args.observation[1], algorithm=Algorithm(name='exposure'), ) mc.write_obs_to_file(obs, args.out_obs_xml)
def _cmd_direct_mock(): from caom2 import SimpleObservation, Algorithm obs = SimpleObservation( observation_id='VLASS1.2.T07t13.J083838-153000', collection=COLLECTION, algorithm=Algorithm(name='testing'), ) return obs
def _update_observation_type(observation): """For the case where a SimpleObservation needs to become a CompositeObservation.""" return CompositeObservation( observation.collection, observation.observation_id, observation.algorithm, observation.sequence_number, observation.intent, Algorithm('composite'), observation.proposal, observation.telescope, observation.instrument, observation.target, observation.meta_release, observation.planes, observation.environment, observation.target_position)
def to_caom2(): plugin = ('/usr/local/lib/python3.9/site-packages/test_execute_composable/' 'test_execute_composable.py') assert sys.argv is not None, 'expect sys.argv to be set' local_meta_create_answer = [ 'test_execute_composable', '--verbose', '--observation', 'OMM', 'test_obs_id', '--local', f'{tc.TEST_DATA_DIR}/test_file.fits.gz', '--out', f'{tc.THIS_DIR}/test_obs_id/test_obs_id.xml', '--plugin', f'{plugin}', '--module', f'{plugin}', '--lineage', 'test_obs_id/cadc:TEST/test_obs_id.fits.gz', ] scrape_answer = [ 'test_execute_composable', '--verbose', '--not_connected', '--observation', 'OMM', 'test_obs_id', '--local', f'{tc.TEST_DATA_DIR}/test_file.fits.gz', '--out', f'{tc.TEST_DATA_DIR}/test_obs_id/test_obs_id.xml', '--plugin', f'{plugin}', '--module', f'{plugin}', '--lineage', 'test_obs_id/cadc:TEST/test_obs_id.fits.gz', ] # TaskType.SCRAPE (Scrape) if sys.argv != scrape_answer: # TaskType.INGEST (LocalMetaCreate) if sys.argv != local_meta_create_answer: assert False, (f'wrong sys.argv values \n{sys.argv} ' f'\n{local_meta_create_answer}') fqn_index = sys.argv.index('--out') + 1 fqn = sys.argv[fqn_index] mc.write_obs_to_file( SimpleObservation( collection='test_collection', observation_id='test_obs_id', algorithm=Algorithm(str('exposure')), ), fqn, )
def _mock_write(): fqn = f'{tc.THIS_DIR}/NEOS_SCI_2015347000000_clean/' \ f'NEOS_SCI_2015347000000_clean.fits.xml' mc.write_obs_to_file( SimpleObservation( collection='test_collection', observation_id='ghi', algorithm=Algorithm(str('test')) ), fqn, )
def visit(self): for uri, file_info in self._metadata_reader.file_info.items(): headers = self._metadata_reader.headers.get(uri) telescope_data = Telescope(uri, headers) blueprint = ObsBlueprint(instantiated_class=telescope_data) telescope_data.accumulate_bp(blueprint) if len(headers) == 0: parser = GenericParser(blueprint, uri) else: parser = FitsParser(headers, blueprint, uri) if self._dump_config: print(f'Blueprint for {uri}: {blueprint}') if self._observation is None: if blueprint._get('DerivedObservation.members') is None: self._logger.debug('Build a SimpleObservation') self._observation = SimpleObservation( collection=self._storage_name.collection, observation_id=self._storage_name.obs_id, algorithm=Algorithm('exposure'), ) else: self._logger.debug('Build a DerivedObservation') self._observation = DerivedObservation( collection=self._storage_name.collection, observation_id=self._storage_name.obs_id, algorithm=Algorithm('composite'), ) parser.augment_observation( observation=self._observation, artifact_uri=uri, product_id=self._storage_name.product_id, ) self._observation = telescope_data.update(self._observation, self._storage_name, file_info) return self._observation
def test_validate_observation(): obs = SimpleObservation('test_collection', 'test_obs_id', Algorithm('test_name')) validate(obs) obs = DerivedObservation('test_collection', 'test_obs_id', Algorithm('test_name'), proposal=Proposal('test_proposal'), telescope=Telescope('test_telescope'), instrument=Instrument('test_instrument'), target=Target('test_targets')) obs.algorithm.keywords = 'foo' obs.proposal.keywords = set('foo=42') obs.telescope.keywords = set('foo:42') obs.instrument.keywords.add("tick'marks") obs.target.keywords = set('has multiple spaces') test_plane = Plane('test_plane') test_plane.provenance = Provenance('test_provenance') test_plane.provenance.keywords.add('pipe|denied') obs.planes['test_plane'] = test_plane with pytest.raises(AssertionError): validate(obs)
def change_to_simple(observation): """For the case where a DerivedObservation needs to become a SimpleObservation.""" temp = SimpleObservation( observation.collection, observation.observation_id, Algorithm('exposure'), observation.sequence_number, observation.intent, observation.type, observation.proposal, observation.telescope, observation.instrument, observation.target, observation.meta_release, observation.meta_read_groups, observation.planes, observation.environment, observation.target_position) temp.last_modified = datetime.utcnow() temp._id = observation._id return temp
def test_read_write_obs_with_file(): if os.path.exists(TEST_OBS_FILE): os.unlink(TEST_OBS_FILE) mc.write_obs_to_file( SimpleObservation( collection='test_collection', observation_id='test_obs_id', algorithm=Algorithm(str('exposure')), ), TEST_OBS_FILE, ) test_subject = mc.read_obs_from_file(TEST_OBS_FILE) assert test_subject is not None, 'expect a result' assert isinstance(test_subject, SimpleObservation), 'wrong read'
def test_augment_observation(): test_obs_blueprint = ObsBlueprint(position_axes=(1, 2)) test_obs_blueprint.set('Observation.target.name', 'CGPS Mosaic MA1') test_obs_blueprint.set('Observation.target.standard', False) test_obs_blueprint.set('Observation.telescope.name', 'DRAO-ST') test_obs_blueprint.set('Observation.instrument.name', 'DRAO-ST') test_obs_blueprint.set('Observation.telescope.geoLocationX', '-2100330.87517') test_obs_blueprint.set('Observation.telescope.geoLocationY', '-3694247.82445') test_obs_blueprint.set('Observation.telescope.geoLocationZ', '4741018.33097') test_obs_blueprint.set('Plane.dataProductType', 'cube') test_obs_blueprint.set('Artifact.productType', 'info') test_obs_blueprint.set('Artifact.releaseType', 'data') test_obs_blueprint.set('Plane.calibrationLevel', '2') test_fitsparser = FitsParser(sample_file_4axes_obs, test_obs_blueprint) test_fitsparser.blueprint = test_obs_blueprint test_obs = SimpleObservation('collection', 'MA1_DRAO-ST', Algorithm('exposure')) test_fitsparser.augment_observation(test_obs, sample_file_4axes_uri, product_id='HI-line') assert test_obs is not None assert test_obs.planes is not None assert len(test_obs.planes) == 1 test_plane = test_obs.planes['HI-line'] assert test_plane.artifacts is not None assert len(test_plane.artifacts) == 1 test_artifact = test_plane.artifacts[sample_file_4axes_uri] assert test_artifact is not None test_part = test_artifact.parts['0'] # remove the chunk bit, as it's part of other tests - # results in <caom2:chunks/> xml output test_part.chunks.pop() output = BytesIO() ow = ObservationWriter(False, False, "caom2", obs_reader_writer.CAOM20_NAMESPACE) ow.write(test_obs, output) result = output.getvalue().decode('UTF-8') output.close() expected = _get_obs(EXPECTED_OBS_XML) actual = _get_obs(result) diff_result = get_differences(expected, actual, 'Observation') assert diff_result is None
def change_to_composite(observation, algorithm_name='composite', collection=None, features=None): """For the case where a SimpleObservation needs to become a DerivedObservation.""" temp = DerivedObservation( observation.collection, observation.observation_id, Algorithm(algorithm_name), observation.sequence_number, observation.intent, observation.type, observation.proposal, observation.telescope, observation.instrument, observation.target, observation.meta_release, observation.meta_read_groups, observation.planes, observation.environment, observation.target_position) temp.meta_producer = observation.meta_producer temp.last_modified = datetime.utcnow() temp._id = observation._id return temp
def test_augment_observation(test_file, test_file_uri): # logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) test_obs_blueprint = ObsBlueprint(position_axis=(1, 2)) test_obs_blueprint.set('Observation.target.name', 'CGPS Mosaic MA1') test_obs_blueprint.set('Observation.telescope.name', 'DRAO-ST') test_obs_blueprint.set('Observation.instrument.name', 'DRAO-ST') test_obs_blueprint.set('Observation.telescope.geoLocationX', '-2100330.87517') test_obs_blueprint.set('Observation.telescope.geoLocationY', '-3694247.82445') test_obs_blueprint.set('Observation.telescope.geoLocationZ', '4741018.33097') test_obs_blueprint.set('Plane.dataProductType', 'cube') test_obs_blueprint.set('Plane.calibrationLevel', '2') test_fitsparser = FitsParser(test_file, test_obs_blueprint) test_fitsparser.blueprint = test_obs_blueprint test_obs = Observation('collection', 'MA1_DRAO-ST', Algorithm('exposure')) test_fitsparser.augment_observation(test_obs, test_file_uri, product_id='HI-line') assert test_obs is not None assert test_obs.planes is not None assert len(test_obs.planes) == 1 test_plane = test_obs.planes['HI-line'] assert test_plane.artifacts is not None assert len(test_plane.artifacts) == 1 test_artifact = test_plane.artifacts[test_file_uri] assert test_artifact is not None test_part = test_artifact.parts['0'] # remove the chunk bit, as it's part of other tests - # results in <caom2:chunks/> xml output test_part.chunks.pop() output = BytesIO() ow = ObservationWriter(False, False, "caom2", obs_reader_writer.CAOM20_NAMESPACE) ow.write(test_obs, output) result = output.getvalue().decode('UTF-8') output.close() compare = re.sub(r'caom2:id=".*"', 'caom2:id=""', result) assert compare == EXPECTED_OBS_XML # , result
def test_run_todo_file_data_source( caps_mock, ad_mock, data_client_mock, set_clients_mock, test_config ): set_clients_mock.side_effect = _clients_mock caps_mock.return_value = 'https://sc2.canfar.net/sc2repo' response = Mock() response.status_code = 200 response.iter_content.return_value = [b'fileName\n'] ad_mock.return_value.__enter__.return_value = response data_client_mock.return_value = SimpleObservation( collection=test_config.collection, observation_id='def', algorithm=Algorithm(str('test')) ) if os.path.exists(test_config.success_fqn): os.unlink(test_config.success_fqn) test_config.work_fqn = f'{TEST_DIR}/todo.txt' test_config.task_types = [mc.TaskType.VISIT] test_config.log_to_file = True test_chooser = ec.OrganizeChooser() test_result = rc.run_by_todo( config=test_config, chooser=test_chooser, command_name=TEST_COMMAND ) assert test_result is not None, 'expect a result' assert test_result == 0, 'expect success' assert os.path.exists(test_config.success_fqn), 'expect success file' with open(test_config.success_fqn) as f: content = f.read() # the obs id and file name assert 'def def.fits' in content, 'wrong success message'
def test_todo_local_move( caom_mock, access_mock, fits2caom2_mock, test_input_name, ): if 'MOVE' not in test_input_name: return test_input = INPUTS.get(test_input_name) _cleanup() new_dir = TEST_DATA_DIR / 'new' fail_dir = TEST_DATA_DIR / 'failure' success_dir = TEST_DATA_DIR / 'success' for entry in [new_dir, fail_dir, success_dir]: for listing in entry.iterdir(): listing.unlink() shutil.copy(test_input.test_file, new_dir / '2460606o.fits.gz') access_mock.return_value = 'https://localhost' # make sure the working directory TEST_EXEC_DIR has the correct things # in it config_file_target = TEST_EXEC_DIR / 'config.yml' shutil.copy(test_input.config_file, config_file_target) cache_file_target = TEST_EXEC_DIR / 'cache.yml' shutil.copy(test_input.cache_file, cache_file_target) with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f: f.write('test content') getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') caom_mock.return_value.data_client.info.side_effect = [ None, FileInfo( id='ad:CFHT/2460606o.fits.gz', md5sum='3d29f0edd984065a044d1376a11c6f08', ), ] caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', 'CFHT', Algorithm(name='exposure'), instrument=Instrument(name='ESPaDOnS'), ), ] def _info(uri): assert uri == 'ad:CFHT/2460606o.fits.gz', 'wrong info uri' return FileInfo( id=uri, md5sum='abc', size=42, ) fits2caom2_mock.return_value.info.side_effect = _info try: test_result = test_module._run_by_builder() assert test_result is not None, f'expect a result {test_input_name}' assert test_result == 0, f'wrong test result {test_input_name}' assert (caom_mock.return_value.data_client.put.called ), f'{test_input_name} put not called' caom_mock.return_value.data_client.put.assert_called_with( '/usr/src/app/integration_test/mock_test/data/test_files/new', 'ad:CFHT/2460606o.fits.gz', 'raw', ), f'{test_input_name} wrong put args' count = 0 for entry in new_dir.iterdir(): count += 1 assert count == 0, 'wrong new dir content' for entry in success_dir.iterdir(): count += 1 assert count == 1, 'wrong success dir content' count = 0 for entry in fail_dir.iterdir(): logging.error(f'fail entry {entry}') count += 1 assert count == 0, f'wrong fail dir content {count}' finally: os.getcwd = getcwd_orig del sys.modules['composable']
def test_gem_todo_local( caom_mock, access_mock, fits2caom2_mock, tap_mock, test_input_name, ): if 'GEM_TODO_LOCAL' not in test_input_name: return test_input = INPUTS.get(test_input_name) _cleanup() shutil.copy(test_input.test_file, TEST_DATA_DIR / 'S20191214S0301.fits') access_mock.return_value = 'https://localhost' _setup(test_input) getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') caom_mock.return_value.data_client.info.side_effect = [ None, FileInfo( id='gemini:GEMINI/S20191214S0301.fits', md5sum='3d29f0edd984065a044d1376a11c6f08', ), ] caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', 'GEMINI', Algorithm(name='exposure'), ), ] def _info(uri): assert (uri == 'gemini:GEMINI/S20191214S0301.fits'), 'wrong info uri' return FileInfo( id=uri, md5sum='abc', size=42, ) fits2caom2_mock.return_value.info.side_effect = _info def _tap_query( ignore_query, output_file, data_only=True, response_format='csv', ): output_file.write( 'observationID,instrument_name\n' 'GS-CAL20191214-1-029,F2\n', ) # caom_mock.return_value.query_client.query.side_effect = _tap_query tap_mock.return_value.query.side_effect = _tap_query try: test_result = test_module._run() assert test_result is not None, f'expect a result {test_input_name}' assert test_result == 0, f'wrong test result {test_input_name}' # Gemini local should not be checking archive.gemini.edu for a # newer version of the file assert not caom_mock.return_value.data_client.put.called, 'no put' finally: os.getcwd = getcwd_orig del sys.modules['composable']
def _read_mock(ignore_fqn): return SimpleObservation( collection='TEST', observation_id='TEST_OBS_ID', algorithm=Algorithm('exposure'), )
def test_gem_todo( data_mock, caom_mock, json_mock, filter_mock, http_get_mock, tap_mock, external_header_mock, test_input_name, ): if 'GEM_TODO' != test_input_name: return test_input = INPUTS.get(test_input_name) _cleanup() getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) _setup(test_input) todo_fqn = TEST_EXEC_DIR / 'todo.txt' with open(todo_fqn, 'w') as f1: f1.write('S20191214S0301.fits\n') def _json_mock(url, ignore_session): response = Mock() response.close = Mock() fqn = test_input.input_dir / 'input.json' with open(fqn) as f: response.text = f.read() def x(): return json.loads(response.text) response.json = x return response json_mock.side_effect = _json_mock def _filter_mock(): from astropy.table import parse_single_table fqn = test_input.input_dir / 'filter.xml' content = parse_single_table(fqn) return content, None filter_mock.side_effect = _filter_mock caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', 'GEMINI', Algorithm(name='exposure'), ), ] def _tap_query( ignore_query, output_file, data_only=True, response_format='csv', ): output_file.write( 'observationID,instrument_name\n' 'GS-CAL20191214-1-029,F2\n', ) # caom_mock.return_value.query_client.query.side_effect = _tap_query tap_mock.return_value.query.side_effect = _tap_query def _get_mock(ignore, uri): if uri == 'gemini:GEMINI/S20191214S0301.jpg': raise exceptions.UnexpectedException('') caom_mock.return_value.data_client.get.side_effect = _get_mock def _http_get_mock(ignore_url, fqn): if (ignore_url != 'https://archive.gemini.edu/file/S20191214S0301.fits' and ignore_url != 'https://archive.gemini.edu/preview/S20191214S0301.fits'): assert False, f'bad http get mock url {ignore_url}' for ext in ['fits', 'jpg']: test_src_fqn = test_input.input_dir / f'S20191214S0301.{ext}' shutil.copy(test_src_fqn, fqn) http_get_mock.side_effect = _http_get_mock def _external_header(ignore_url): assert (ignore_url == 'https://archive.gemini.edu/fullheader/S20191214S0301.fits' ), 'wrong file header url' fqn = test_input.input_dir / 'S20191214S0301.fits' return caom2utils.data_util.get_local_file_headers(fqn.as_posix()) external_header_mock.side_effect = _external_header def _info(uri): return FileInfo( id=uri, md5sum='abc', size=42, ) data_mock.return_value.info.side_effect = _info # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') try: test_result = test_module._run() assert test_result is not None, f'expect a result {test_input_name}' assert test_result == 0, f'wrong test result {test_input_name}' assert (caom_mock.return_value.data_client.put.called ), f'{test_input_name} put not called' assert (caom_mock.return_value.data_client.put.call_count == 3 ), 'wrong put call count' put_calls = [ call( '/usr/src/app/integration_test/mock_test/data/execution/' 'GS-CAL20191214-1-029', 'gemini:GEMINI/S20191214S0301.jpg', ), call( '/usr/src/app/integration_test/mock_test/data/execution/' 'GS-CAL20191214-1-029', 'cadc:GEMINI/S20191214S0301_th.jpg', ), call( '/usr/src/app/integration_test/mock_test/data/execution/' 'GS-CAL20191214-1-029', 'gemini:GEMINI/S20191214S0301.fits', ), ] caom_mock.return_value.data_client.put.assert_has_calls( put_calls), f'{test_input_name} wrong put args' assert http_get_mock.called, 'expect http get call' assert http_get_mock.call_count == 2, 'wrong http get call count' http_get_calls = [ call( 'https://archive.gemini.edu/preview/S20191214S0301.fits', '/usr/src/app/integration_test/mock_test/data/execution/' 'GS-CAL20191214-1-029/S20191214S0301.jpg', ), call( 'https://archive.gemini.edu/file/S20191214S0301.fits', '/usr/src/app/integration_test/mock_test/data/execution/' 'GS-CAL20191214-1-029/S20191214S0301.fits', ), ] http_get_mock.assert_has_calls(http_get_calls), 'wrong http get args' except Exception as e: logging.error(traceback.format_exc()) raise e finally: os.getcwd = getcwd_orig del sys.modules['composable']
def mock_read(collection, obs_id): return SimpleObservation( collection=collection, observation_id=obs_id, algorithm=Algorithm('exposure'), )
def test_state( data_mock, web_log_mock, nrao_mock, caom_mock, transfer_mock, local_header_mock, qa_mock, test_input_name, ): if 'TODO' in test_input_name: return test_input = INPUTS.get(test_input_name) # make sure the working directory TEXT_EXEC_DIR has nothing in it for child in TEST_EXEC_DIR.iterdir(): if child == TEST_EXEC_DIR: continue if child.is_dir(): for child_2 in child.iterdir(): child_2.unlink() child.rmdir() else: child.unlink() # make sure the working directory TEST_EXEC_DIR has the correct things # in it config_file_target = TEST_EXEC_DIR / 'config.yml' shutil.copy(test_input.config_file, config_file_target) state_file_target = TEST_EXEC_DIR / 'state.yml' shutil.copy(test_input.state_file, state_file_target) with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f: f.write('test content') # make the state file won't take decades to execute test_start_time = datetime.now(tz=dateutil.tz.UTC) - timedelta(minutes=5) state = mc.State(state_file_target.as_posix()) state.save_state(test_input.bookmark, test_start_time) # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') nrao_mock.side_effect = _nrao_mock def _web_log_init(ignore): global web_log_content web_log_content = { 'VLASS1.1_T07t13.J083838-153000_P68878v1_2020_08_29T21_' '48_48.092': '2020-09-09 07:53', } web_log_mock.side_effect = _web_log_init def _transfer_get(src, dst): assert (src == 'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.1/T07t13/' 'VLASS1.1.ql.T07t13.J083838-153000.10.2048.v1.I.iter1.image.' 'pbcor.tt0.rms.subim.fits'), 'wrong source' assert ( dst == '/usr/src/app/integration_test/mock_test/data/execution/' 'VLASS1.1.T07t13.J083838-153000/' 'VLASS1.1.ql.T07t13.J083838-153000.10.2048.v1.I.iter1.image.' 'pbcor.tt0.rms.subim.fits'), 'wrong dst' with open(dst, 'w') as f2: f2.write('test content') transfer_mock.return_value.get.side_effect = _transfer_get caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', 'VLASS', Algorithm(name='exposure'), ), ] def _local_header(ignore): x = """SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017 BITPIX = -32 / Bits per pixel NAXIS = 2 / Number of dimensions NAXIS1 = 2048 / NAXIS2 = 2048 / TYPE = 'image' BMAJ = 1.09 BMIN = 0.19 DATATYPE= 'REDUC ' /Data type, SCIENCE/CALIB/REJECT/FOCUS/TEST END """ delim = '\nEND' extensions = \ [e + delim for e in x.split(delim) if e.strip()] headers = [fits.Header.fromstring(e, sep='\n') for e in extensions] return headers local_header_mock.side_effect = _local_header def _info(uri): return FileInfo( id=uri, md5sum='abc', size=42, ) data_mock.return_value.info.side_effect = _info qa_mock.return_value = False getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) logging.getLogger('StorageClientWrapper').setLevel(logging.DEBUG) try: test_result = test_module._run_state() assert test_result is not None, f'expect a result {test_input_name}' assert test_result == 0, f'wrong test result {test_input_name}' # was state updated? post_state = mc.State(state_file_target.as_posix()) assert (post_state.get_bookmark(test_input.bookmark) > test_start_time), f'state not updated {test_input_name}' assert (caom_mock.return_value.data_client.put.called ), f'{test_input_name} put not called' caom_mock.return_value.data_client.put.assert_called_with( '/usr/src/app/integration_test/mock_test/data/execution/' 'VLASS1.1.T07t13.J083838-153000', 'nrao:VLASS/VLASS1.1.ql.T07t13.J083838-153000.10.2048.v1.I.' 'iter1.image.pbcor.tt0.rms.subim.fits', None, ), f'{test_input_name} wrong put args' finally: os.getcwd = getcwd_orig del sys.modules['composable']
def test_omm_retry( caom_mock, access_mock, fits2caom2_mock, test_input_name, ): if 'OMM_RETRY' != test_input_name: return access_mock.return_value = 'https://localhost' test_input = INPUTS.get(test_input_name) _cleanup() getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) _setup(test_input) shutil.copy(f'{TEST_DIR}/vlass/footprintfinder.py', TEST_EXEC_DIR) retry_fqn = Path(f'{TEST_EXEC_DIR.as_posix()}_0') todo_fqn = TEST_EXEC_DIR / 'todo.txt' with open(todo_fqn, 'w') as f1: f1.write(f'{test_input.test_file.name}\n') caom_mock.return_value.data_client.info.side_effect = [ None, FileInfo( id=test_input.test_uri, md5sum='3d29f0edd984065a044d1376a11c6f08', ), FileInfo( id=test_input.test_uri, md5sum='3d29f0edd984065a044d1376a11c6f08', ), FileInfo( id=test_input.test_uri, md5sum='3d29f0edd984065a044d1376a11c6f08', ), ] test_obs = mc.read_obs_from_file(test_input.obs_xml.as_posix()) caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', test_input.collection, Algorithm(name='exposure'), ), test_obs, test_obs, ] caom_mock.return_value.metadata_client.update.side_effect = [ mc.CadcException('pretend there is a timeout'), None, None, None, ] test_exec_fqn = Path(TEST_EXEC_DIR / 'C170324_0054') def _get_mock(ignore, uri): assert uri == test_input.test_uri, 'wrong uri' assert ignore == test_exec_fqn.as_posix(), 'wrong working directory' shutil.copy(test_input.test_file, test_exec_fqn) caom_mock.return_value.data_client.get.side_effect = _get_mock def _info(uri): return FileInfo( id=uri, md5sum='abc', size=42, ) fits2caom2_mock.return_value.info.side_effect = _info def _get_head(uri): assert uri == test_input.test_uri, 'wrong get_head parameter' return caom2utils.data_util.get_local_file_headers( test_input.test_file.as_posix()) fits2caom2_mock.return_value.get_head.side_effect = _get_head # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') try: test_result = test_module._run() assert test_result is not None, f'expect a result {test_input_name}' # expect a non-zero return, because evidence of the first failure # is preserved even through the retry assert test_result == -1, f'wrong test result {test_input_name}' assert (caom_mock.return_value.data_client.get.call_count == 2 ), 'wrong get call count' assert retry_fqn.exists(), 'expect directory to exist' assert retry_fqn.is_dir(), 'expect it to be a directory' assert (caom_mock.return_value.data_client.put.call_count == 2 ), 'wrong put call count' put_calls = [ call( test_exec_fqn.as_posix(), 'cadc:OMM/C170324_0054_SCI_prev_256.jpg', None, ), call( test_exec_fqn.as_posix(), 'cadc:OMM/C170324_0054_SCI_prev.jpg', None, ), ] caom_mock.return_value.data_client.put.assert_has_calls( put_calls), 'wrong put calls' except Exception as e: logging.error(traceback.format_exc()) raise e finally: os.getcwd = getcwd_orig if retry_fqn.exists(): logging.error(f'Cleaning up {retry_fqn.as_posix()}') for child in retry_fqn.iterdir(): child.unlink() retry_fqn.rmdir()
def _mock_repo_read(ignore_client, collection, obs_id, ignore_metrics): return SimpleObservation(obs_id, collection, Algorithm(name='exposure'))
def test_gem_state( data_mock, caom_mock, local_header_mock, json_mock, filter_mock, http_get_mock, endpoint_mock, tap_mock, external_header_mock, test_input_name, ): if 'GEM_STATE' not in test_input_name: return test_input = INPUTS.get(test_input_name) _cleanup() getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) test_start_time, state_file_target = _setup(test_input) def _json_mock(url, ignore_session): response = Mock() response.close = Mock() fqn = test_input.input_dir / 'input.json' with open(fqn) as f: response.text = f.read() def x(): return json.loads(response.text) response.json = x return response json_mock.side_effect = _json_mock def _endpoint_mock(ignore): assert (ignore.startswith( 'https://archive.gemini.edu/jsonsummary/canonical/NotFail/' 'notengineering/entrytimedaterange') ), 'wrong url for incremental querying' return _json_mock(ignore, None) endpoint_mock.side_effect = _endpoint_mock def _filter_mock(): from astropy.table import parse_single_table fqn = test_input.input_dir / 'filter.xml' content = parse_single_table(fqn) return content, None filter_mock.side_effect = _filter_mock caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', 'GEMINI', Algorithm(name='exposure'), ), ] def _tap_query( ignore_query, output_file, data_only=True, response_format='csv', ): output_file.write( 'observationID,instrument_name\n' 'GS-CAL20191214-1-029,F2\n', ) # caom_mock.return_value.query_client.query.side_effect = _tap_query tap_mock.return_value.query.side_effect = _tap_query def _local_header(ignore): x = """SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017 BITPIX = -32 / Bits per pixel NAXIS = 2 / Number of dimensions NAXIS1 = 14 / NAXIS2 = 24 / INSTRUME= 'F2' DATALAB = 'GS-CAL20191214-1-029 END """ delim = '\nEND' extensions = \ [e + delim for e in x.split(delim) if e.strip()] headers = [fits.Header.fromstring(e, sep='\n') for e in extensions] return headers local_header_mock.side_effect = _local_header external_header_mock.side_effect = _local_header def _info(uri): return FileInfo( id=uri, md5sum='abc', size=42, ) data_mock.return_value.info.side_effect = _info # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') try: test_result = test_module._run_state() assert test_result is not None, f'expect a result {test_input_name}' assert test_result == 0, f'wrong test result {test_input_name}' # was state updated? post_state = mc.State(state_file_target.as_posix()) assert (post_state.get_bookmark(test_input.bookmark) > test_start_time), f'state not updated {test_input_name}' assert (caom_mock.return_value.data_client.put.called ), f'{test_input_name} put not called' caom_mock.return_value.data_client.put.assert_called_with( '/usr/src/app/integration_test/mock_test/data/execution/' 'GS-CAL20191214-1-029', 'gemini:GEMINI/S20191214S0301.fits', ), f'{test_input_name} wrong put args' assert http_get_mock.called, 'expect http get call' http_get_mock.assert_called_with( 'https://archive.gemini.edu/file/S20191214S0301.fits', '/usr/src/app/integration_test/mock_test/data/execution/' 'GS-CAL20191214-1-029/S20191214S0301.fits', ), 'wrong http get args' except Exception as e: logging.error(traceback.format_exc()) raise e finally: os.getcwd = getcwd_orig del sys.modules['composable']
def _mock_read(ignore_fqn): return SimpleObservation( collection='test_collection', observation_id='ghi', algorithm=Algorithm(str('test')), )
def test_todo_vos( vo_mock, caom_mock, access_mock, transfer_mock, fits2caom2_mock, test_input_name, ): if 'VOS' not in test_input_name: return test_input = INPUTS.get(test_input_name) _cleanup() access_mock.return_value = 'https://localhost' # make sure the working directory TEST_EXEC_DIR has the correct things # in it config_file_target = TEST_EXEC_DIR / 'config.yml' shutil.copy(test_input.config_file, config_file_target) with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f: f.write('test content') # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') reload(test_module) logging.error(dir(test_module)) def _vo_listdir(entry): assert entry == 'vos:goliaths/test', 'wrong parameter' return ['a2021_08_17_19_30_01.fits.gz'] vo_mock.return_value.listdir.side_effect = _vo_listdir vo_mock.return_value.isdir.return_value = False def _vo_get_node(uri, limit=None, force=False): assert (uri == 'vos:goliaths/test/a2021_08_17_19_30_01.fits.gz' ), f'wrong get node uri {uri}' test_start_time = (datetime.now(tz=dateutil.tz.UTC) - timedelta(minutes=5)) node = type('', (), {})() node.props = { 'length': 42, 'MD5': '1234', 'lastmod': test_start_time.isoformat(), } return node vo_mock.return_value.get_node.side_effect = _vo_get_node caom_mock.return_value.data_client.info.side_effect = [ None, FileInfo(id='ad:DAO/a2021_08_17_19_30_01.fits.gz'), ] caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', 'DAO', Algorithm(name='exposure'), ), ] def _transfer_get(src, dst): logging.error(src) logging.error(dst) assert (src == 'vos:goliaths/test/a2021_08_17_19_30_01.fits.gz' ), 'wrong source' assert ( dst == '/usr/src/app/integration_test/mock_test/data/execution/' 'a2021_08_17_19_30_01/a2021_08_17_19_30_01.fits.gz'), 'wrong dst' with open(dst, 'w') as f2: f2.write('test content') transfer_mock.return_value.get.side_effect = _transfer_get def _get_head(ignore): x = """SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017 BITPIX = -32 / Bits per pixel NAXIS = 2 / Number of dimensions NAXIS1 = 2048 / NAXIS2 = 2048 / EXPTIME = 1.23 NCOMBINE= 1 DATATYPE= 'REDUC ' /Data type, SCIENCE/CALIB/REJECT/FOCUS/TEST END """ delim = '\nEND' extensions = \ [e + delim for e in x.split(delim) if e.strip()] headers = [fits.Header.fromstring(e, sep='\n') for e in extensions] return headers fits2caom2_mock.return_value.get_head.side_effect = _get_head def _info(uri): return FileInfo( id=uri, md5sum='abc', size=42, ) fits2caom2_mock.return_value.info.side_effect = _info getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) try: test_result = test_module._run_vo() assert test_result is not None, f'expect a result {test_input_name}' assert test_result == 0, f'wrong test result {test_input_name}' assert (caom_mock.return_value.data_client.put.called ), f'{test_input_name} put not called' caom_mock.return_value.data_client.put.assert_called_with( '/usr/src/app/integration_test/mock_test/data/execution/' 'a2021_08_17_19_30_01', 'ad:DAO/a2021_08_17_19_30_01.fits.gz', 'raw', ), f'{test_input_name} wrong put args' finally: os.getcwd = getcwd_orig del sys.modules['composable']
def test_neoss_state( data_mock, csa_mock, caom_mock, transfer_mock, local_header_mock, test_input_name, ): if 'NEOSS' not in test_input_name: return test_input = INPUTS.get(test_input_name) # make sure the working directory TEXT_EXEC_DIR has nothing in it _cleanup() # make sure the working directory TEST_EXEC_DIR has the correct things # in it config_file_target = TEST_EXEC_DIR / 'config.yml' shutil.copy(test_input.config_file, config_file_target) state_file_target = TEST_EXEC_DIR / 'state.yml' shutil.copy(test_input.state_file, state_file_target) with open(TEST_EXEC_DIR / 'cadcproxy.pem', 'w') as f: f.write('test content') # make the state file won't take decades to execute test_start_time = datetime.now(tz=dateutil.tz.UTC) - timedelta(minutes=5) state = mc.State(state_file_target.as_posix()) state.save_state(test_input.bookmark, test_start_time) def _csa_mock(start_date, ign1, ign2, ign3, ign4, ign5): return { '/users/OpenData_DonneesOuvertes/pub/NEOSSAT/ASTRO/2019/256/' 'NEOS_SCI_2019213215700.fits': [False, start_date + timedelta(minutes=5).total_seconds()], } csa_mock.side_effect = _csa_mock def _transfer_get(src, dst): assert (src == '/users/OpenData_DonneesOuvertes/pub/NEOSSAT/ASTRO/2019/256/' 'NEOS_SCI_2019213215700.fits'), 'wrong source' assert ( dst == '/usr/src/app/integration_test/mock_test/data/execution/' '2019213215700/NEOS_SCI_2019213215700.fits'), 'wrong dst' with open(dst, 'w') as f2: f2.write('test content') transfer_mock.return_value.get.side_effect = _transfer_get caom_mock.return_value.metadata_client.read.side_effect = [ None, SimpleObservation( 'obs_id', 'NEOSSAT', Algorithm(name='exposure'), ), ] def _local_header(ignore): x = """SIMPLE = T / Written by IDL: Fri Oct 6 01:48:35 2017 BITPIX = -32 / Bits per pixel NAXIS = 2 / Number of dimensions NAXIS1 = 14 / NAXIS2 = 24 / RA = '22:53:27.5' DEC = '-30:04:37.6' MODE = '14 - FINE_SETTLE' OBJECT = '2020-P4-C' EXPOSURE= 128.0311 DATATYPE= 'REDUC ' /Data type, SCIENCE/CALIB/REJECT/FOCUS/TEST END """ delim = '\nEND' extensions = \ [e + delim for e in x.split(delim) if e.strip()] headers = [fits.Header.fromstring(e, sep='\n') for e in extensions] return headers local_header_mock.side_effect = _local_header def _info(uri): return FileInfo( id=uri, md5sum='abc', size=42, ) data_mock.return_value.info.side_effect = _info # import the module for execution sys.path.append(test_input.test_path) test_module = import_module('composable') getcwd_orig = os.getcwd os.getcwd = Mock(return_value=TEST_EXEC_DIR) try: test_result = test_module._run_state() assert test_result is not None, f'expect a result {test_input_name}' assert test_result == 0, f'wrong test result {test_input_name}' # was state updated? post_state = mc.State(state_file_target.as_posix()) assert (post_state.get_bookmark(test_input.bookmark) > test_start_time), f'state not updated {test_input_name}' assert (caom_mock.return_value.data_client.put.called ), f'{test_input_name} put not called' caom_mock.return_value.data_client.put.assert_called_with( '/usr/src/app/integration_test/mock_test/data/execution/' '2019213215700', 'cadc:NEOSSAT/NEOS_SCI_2019213215700.fits', None, ), f'{test_input_name} wrong put args' except Exception as e: logging.error(traceback.format_exc()) raise e finally: os.getcwd = getcwd_orig del sys.modules['composable']