def _check_remote(self, uri): self._logger.debug(f'Begin _check_remote for {uri}') result = None json = retrieve_json(uri, self._logger, self._gemini_session) f_id = obs_file_relationship.remove_extensions(uri.split('/')[-1]) for ii in json: y = obs_file_relationship.remove_extensions(ii.get('name')) if y == f_id: result = ii.get('data_label') break self._logger.debug(f'End _check_remote with result {result}') return result
def test_visitor( pf_mock, get_pi_mock, svofps_mock, headers_mock, json_mock, file_type_mock, test_name, ): test_file_id = obs_file_relationship.remove_extensions( os.path.basename(test_name) ) expected_fqn = f'{os.path.dirname(test_name)}/{test_file_id}.expected.xml' gem_mocks._run_test_common( data_sources=[os.path.dirname(test_name)], get_pi_mock=get_pi_mock, svofps_mock=svofps_mock, headers_mock=headers_mock, pf_mock=pf_mock, json_mock=json_mock, file_type_mock=file_type_mock, test_set=[test_name], expected_fqn=expected_fqn, )
def _check_local(self, f_name): self._logger.debug(f'Begin _check_local for {f_name}') file_id = obs_file_relationship.remove_extensions(f_name) try_these = [ f'{file_id}.fits', f'{file_id}.fits.header', f'{file_id}.fits.bz2', f'{file_id}.fits.gz', ] result = None for data_source in self._data_sources: for f_name in try_these: fqn = path.join(data_source, f_name) if path.exists(fqn): headers = data_util.get_local_file_headers(fqn) temp = headers[0].get('DATALAB').upper() if temp is not None: result = headers[0].get('DATALAB') self._logger.debug( f'Found observation ID {result} for {f_name} on ' f'disk.' ) break self._logger.debug('End _check_local') return result
def visit(observation, **kwargs): """ If there are artifacts with the same name, but different case, prefer the lower case artifact, and remove the upper-case one. :param observation: Observation instance - check all it's artifacts :param kwargs: """ mc.check_param(observation, Observation) artifact_count = 0 plane_count = 0 if len(observation.planes.values()) > 1: all_artifact_keys = cc.get_all_artifact_keys(observation) all_artifact_keys_lower = [ii.lower() for ii in all_artifact_keys] set_artifact_keys_lower = set(all_artifact_keys_lower) delete_these_artifacts = [] if len(all_artifact_keys) != len(set_artifact_keys_lower): for entry in set_artifact_keys_lower: ignore_scheme, ignore_path, file_name = mc.decompose_uri(entry) file_id = obs_file_relationship.remove_extensions(file_name) # it's the suffix that has the different case, so use it # to figure out which artifacts shouldn't exist suffixes = obs_file_relationship.get_suffix( file_id, observation.observation_id) for key in all_artifact_keys: for suffix in suffixes: if suffix.upper() in key: # get the fits, previews, thumbnails as well delete_these_artifacts.append(key) delete_these_planes = [] for entry in delete_these_artifacts: for plane in observation.planes.values(): if entry in plane.artifacts.keys(): plane.artifacts.pop(entry) logging.info(f'Removing {entry} from {plane.product_id}.') artifact_count += 1 if len(plane.artifacts.keys()) == 0: delete_these_planes.append(plane.product_id) for entry in set(delete_these_planes): observation.planes.pop(entry) logging.info( f'Removing {entry} from {observation.observation_id}.') plane_count += 1 logging.info( f'Completed cleanup for {observation.observation_id}. Removed ' f'{artifact_count} artifacts and {plane_count} planes.') result = { 'artifacts': artifact_count, 'planes': plane_count, } return observation
def mock_get_data_label(uri): ignore_scheme, ignore_collection, f_name = mc.decompose_uri(uri) file_id = GemName.remove_extensions(f_name) temp = mock_get_obs_metadata(file_id) result = None for ii in temp: y = obs_file_relationship.remove_extensions(ii.get('filename')) if y == file_id: result = ii.get('data_label') break return result
def mock_get_obs_metadata(file_id): file_id = obs_file_relationship.remove_extensions(file_id.split('/')[-1]) try: fname = f'{TEST_DATA_DIR}/json/{file_id}.json' if os.path.exists(fname): with open(fname) as f: y = json.loads(f.read()) else: # TODO y = [ { 'data_label': TAP_QUERY_LOOKUP.get(file_id, 'test_data_label'), 'filename': f'{file_id}.fits.bz2', 'name': f'{file_id}.fits.bz2', 'lastmod': '2020-02-25T20:36:31.230', 'instrument': 'GMOS-S', }, ] return y except Exception as e: logging.error(e) tb = traceback.format_exc() logging.error(tb)
def set_file_id(self): self._file_id = remove_extensions(self._file_name)