def init_global(config):
global om
om = gom.GeminiObsMetadata()
get_gofr(config)
global gofr
if gofr.tap_client is None and config.is_connected:
subject = mc.define_subject(config)
tap_client = CadcTapClient(subject=subject, resource_id=config.tap_id)
gofr.tap_client = tap_client
def _filter_result(self):
config = mc.Config()
config.get_executors()
subject = mc.define_subject(config)
caom_client = CAOM2RepoClient(subject)
metrics = mc.Metrics(config)
for entry in self._source:
if VlassValidator._later_version_at_cadc(
entry, caom_client, metrics):
self._source.remove(entry)
def __init__(self, config):
super(CFHTBuilder, self).__init__()
self._config = config
self._data_client = None
self._repo_client = None
self._metrics = mc.Metrics(self._config)
if not self._config.use_local_files:
subject = mc.define_subject(self._config)
self._data_client = CadcDataClient(subject)
self._repo_client = CAOM2RepoClient(
subject, resource_id=self._config.resource_id)
self._logger = logging.getLogger(__name__)
def get_obs_id(self):
obs_id = external_metadata.get_obs_id_from_headers(self._file_id)
if obs_id is None:
config = mc.Config()
config.get_executors()
subject = mc.define_subject(config)
tap_client = CadcTapClient(subject=subject,
resource_id=config.tap_id)
obs_id = external_metadata.get_obs_id_from_cadc(
self._file_id, tap_client, COLLECTION)
if obs_id is None:
headers = fits2caom2.get_cadc_headers(
f'ad:GEMINI/{self._file_name}', subject)
obs_id = headers[0].get('DATALAB')
if obs_id is None:
raise mc.CadcException(f'No obs id for {self._file_name}')
return obs_id
def visit(observation, **kwargs):
mc.check_param(observation, Observation)
working_directory = kwargs.get('working_directory', './')
science_file = kwargs.get('science_file')
if science_file is None:
raise mc.CadcException(
f'Must have a science_file parameter for provenance_augmentation '
f'for {observation.observation_id}')
config = mc.Config()
config.get_executors()
subject = mc.define_subject(config)
tap_client = CadcTapClient(subject, config.tap_id)
count = 0
storage_name = GemProcName(science_file, entry=science_file)
obs_members = TypedSet(ObservationURI, )
for plane in observation.planes.values():
plane_inputs = TypedSet(PlaneURI, )
for artifact in plane.artifacts.values():
if storage_name.file_uri == artifact.uri:
count = _do_provenance(
working_directory, science_file, observation,
tap_client, plane_inputs, obs_members)
if plane.provenance is not None:
plane.provenance.inputs.update(plane_inputs)
if isinstance(observation, DerivedObservation):
observation.members.update(obs_members)
if len(observation.members) > 0:
observable = kwargs.get('observable')
caom_repo_client = kwargs.get('caom_repo_client')
if caom_repo_client is None:
logging.warning(f'Warning: Must have a caom_repo_client for '
f'members metadata for '
f'{observation.observation_id}.')
_do_members_metadata(
observation, caom_repo_client, observation.members,
observable.metrics)
logging.info(
f'Done provenance_augmentation for {observation.observation_id}')
return {'provenance': count}
def _update_ngvs_time(chunk, provenance, obs_id):
logging.debug(f'Begin _update_ngvs_time for {obs_id}')
if (chunk is not None and provenance is not None and
len(provenance.inputs) > 0):
# bounds = ctor
config = mc.Config()
config.get_executors()
subject = mc.define_subject(config)
client = CAOM2RepoClient(
subject, config.logging_level, 'ivo://cadc.nrc.ca/ams')
metrics = mc.Metrics(config)
bounds = CoordBounds1D()
min_date = 0
max_date = sys.float_info.max
exposure = 0
for entry in provenance.inputs:
ip_obs_id, ip_product_id = mc.CaomName.decompose_provenance_input(
entry.uri)
logging.info(f'Retrieving provenance metadata for {ip_obs_id}.')
ip_obs = mc.repo_get(client, 'CFHT', ip_obs_id, metrics)
if ip_obs is not None:
ip_plane = ip_obs.planes.get(ip_product_id)
if (ip_plane is not None and ip_plane.time is not None and
ip_plane.time.bounds is not None):
bounds.samples.append(CoordRange1D(
RefCoord(pix=0.5, val=ip_plane.time.bounds.lower),
RefCoord(pix=1.5, val=ip_plane.time.bounds.upper)))
min_date = min(ip_plane.time.bounds.lower, min_date)
max_date = max(ip_plane.time.bounds.upper, max_date)
exposure += ip_plane.time.exposure
axis = Axis(ctype='TIME', cunit='d')
time_axis = CoordAxis1D(axis=axis,
error=None,
range=None,
bounds=bounds,
function=None)
temporal_wcs = TemporalWCS(axis=time_axis, timesys=None, trefpos=None,
mjdref=None, exposure=mc.to_float(exposure),
resolution=None)
chunk.time = temporal_wcs
logging.debug(f'End _update_ngvs_time.')
def test_define_subject():
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=tc.TEST_DATA_DIR)
try:
test_config = mc.Config()
test_config.get_executors()
test_config.proxy_fqn = None
test_config.netrc_file = 'test_netrc'
test_netrc_fqn = os.path.join(test_config.working_directory,
test_config.netrc_file)
if not os.path.exists(test_netrc_fqn):
with open(test_netrc_fqn, 'w') as f:
f.write(
'machine www.example.com login userid password userpass')
test_subject = mc.define_subject(test_config)
assert test_subject is not None, 'expect a netrc subject'
test_config.netrc_file = 'nonexistent'
test_subject = mc.define_subject(test_config)
assert test_subject is None, 'expect no subject, cannot find content'
test_config.netrc_file = None
# proxy pre-condition
test_config.proxy_fqn = f'{tc.TEST_DATA_DIR}/proxy.pem'
if not os.path.exists(test_config.proxy_fqn):
with open(test_config.proxy_fqn, 'w') as f:
f.write('proxy content')
test_subject = mc.define_subject(test_config)
assert test_subject is not None, 'expect a proxy subject'
test_config.proxy_fqn = '/nonexistent'
with pytest.raises(mc.CadcException):
mc.define_subject(test_config)
test_config.proxy_fqn = None
with pytest.raises(mc.CadcException):
mc.define_subject(test_config)
finally:
os.getcwd = getcwd_orig
def __init__(self, config, preview_suffix='jpg'):
super(QueryTimeBoxDataSourceTS, self).__init__(config)
self._preview_suffix = preview_suffix
subject = mc.define_subject(config)
self._client = CadcTapClient(subject, resource_id=self._config.tap_id)
self._logger = logging.getLogger(self.__class__.__name__)
def _set_clients(config):
subject = mc.define_subject(config)
caom_client = CAOM2RepoClient(subject, config.logging_level,
config.resource_id)
cadc_client = mc.declare_client(config)
return cadc_client, caom_client