def _check_remote(self, uri):
self._logger.debug(f'Begin _check_remote for {uri}')
result = None
json = retrieve_json(uri, self._logger, self._gemini_session)
f_id = obs_file_relationship.remove_extensions(uri.split('/')[-1])
for ii in json:
y = obs_file_relationship.remove_extensions(ii.get('name'))
if y == f_id:
result = ii.get('data_label')
break
self._logger.debug(f'End _check_remote with result {result}')
return result
def test_visitor(
pf_mock,
get_pi_mock,
svofps_mock,
headers_mock,
json_mock,
file_type_mock,
test_name,
):
test_file_id = obs_file_relationship.remove_extensions(
os.path.basename(test_name)
)
expected_fqn = f'{os.path.dirname(test_name)}/{test_file_id}.expected.xml'
gem_mocks._run_test_common(
data_sources=[os.path.dirname(test_name)],
get_pi_mock=get_pi_mock,
svofps_mock=svofps_mock,
headers_mock=headers_mock,
pf_mock=pf_mock,
json_mock=json_mock,
file_type_mock=file_type_mock,
test_set=[test_name],
expected_fqn=expected_fqn,
)
def _check_local(self, f_name):
self._logger.debug(f'Begin _check_local for {f_name}')
file_id = obs_file_relationship.remove_extensions(f_name)
try_these = [
f'{file_id}.fits',
f'{file_id}.fits.header',
f'{file_id}.fits.bz2',
f'{file_id}.fits.gz',
]
result = None
for data_source in self._data_sources:
for f_name in try_these:
fqn = path.join(data_source, f_name)
if path.exists(fqn):
headers = data_util.get_local_file_headers(fqn)
temp = headers[0].get('DATALAB').upper()
if temp is not None:
result = headers[0].get('DATALAB')
self._logger.debug(
f'Found observation ID {result} for {f_name} on '
f'disk.'
)
break
self._logger.debug('End _check_local')
return result
def visit(observation, **kwargs):
"""
If there are artifacts with the same name, but different case, prefer
the lower case artifact, and remove the upper-case one.
:param observation: Observation instance - check all it's artifacts
:param kwargs:
"""
mc.check_param(observation, Observation)
artifact_count = 0
plane_count = 0
if len(observation.planes.values()) > 1:
all_artifact_keys = cc.get_all_artifact_keys(observation)
all_artifact_keys_lower = [ii.lower() for ii in all_artifact_keys]
set_artifact_keys_lower = set(all_artifact_keys_lower)
delete_these_artifacts = []
if len(all_artifact_keys) != len(set_artifact_keys_lower):
for entry in set_artifact_keys_lower:
ignore_scheme, ignore_path, file_name = mc.decompose_uri(entry)
file_id = obs_file_relationship.remove_extensions(file_name)
# it's the suffix that has the different case, so use it
# to figure out which artifacts shouldn't exist
suffixes = obs_file_relationship.get_suffix(
file_id, observation.observation_id)
for key in all_artifact_keys:
for suffix in suffixes:
if suffix.upper() in key:
# get the fits, previews, thumbnails as well
delete_these_artifacts.append(key)
delete_these_planes = []
for entry in delete_these_artifacts:
for plane in observation.planes.values():
if entry in plane.artifacts.keys():
plane.artifacts.pop(entry)
logging.info(f'Removing {entry} from {plane.product_id}.')
artifact_count += 1
if len(plane.artifacts.keys()) == 0:
delete_these_planes.append(plane.product_id)
for entry in set(delete_these_planes):
observation.planes.pop(entry)
logging.info(
f'Removing {entry} from {observation.observation_id}.')
plane_count += 1
logging.info(
f'Completed cleanup for {observation.observation_id}. Removed '
f'{artifact_count} artifacts and {plane_count} planes.')
result = {
'artifacts': artifact_count,
'planes': plane_count,
}
return observation
def mock_get_data_label(uri):
ignore_scheme, ignore_collection, f_name = mc.decompose_uri(uri)
file_id = GemName.remove_extensions(f_name)
temp = mock_get_obs_metadata(file_id)
result = None
for ii in temp:
y = obs_file_relationship.remove_extensions(ii.get('filename'))
if y == file_id:
result = ii.get('data_label')
break
return result
def mock_get_obs_metadata(file_id):
file_id = obs_file_relationship.remove_extensions(file_id.split('/')[-1])
try:
fname = f'{TEST_DATA_DIR}/json/{file_id}.json'
if os.path.exists(fname):
with open(fname) as f:
y = json.loads(f.read())
else:
# TODO
y = [
{
'data_label': TAP_QUERY_LOOKUP.get(file_id,
'test_data_label'),
'filename': f'{file_id}.fits.bz2',
'name': f'{file_id}.fits.bz2',
'lastmod': '2020-02-25T20:36:31.230',
'instrument': 'GMOS-S',
},
]
return y
except Exception as e:
logging.error(e)
tb = traceback.format_exc()
logging.error(tb)
def set_file_id(self):
self._file_id = remove_extensions(self._file_name)