def run_query():
"""
Run the processing for all the entries returned from a time-boxed ad
query.
:param sys.argv[1] the timestamp for the > comparison in the time-boxed
query
:param sys.argv[2] the timestamp for the <= comparison in the time-boxed
query
:return 0 if successful, -1 if there's any sort of failure. Return status
is used by airflow for task instance management and reporting.
"""
prev_exec_date = sys.argv[1]
exec_date = sys.argv[2]
config = mc.Config()
config.get()
config.stream = 'default'
file_list = mc.read_file_list_from_archive(config, APPLICATION,
prev_exec_date, exec_date)
sys.argv = sys.argv[:1]
result = 0
if len(file_list) > 0:
mc.write_to_file(config.work_fqn, '\n'.join(file_list))
result |= ec.run_by_file(GemName,
APPLICATION,
COLLECTION,
config.proxy_fqn,
meta_visitors,
data_visitors,
archive=ARCHIVE)
sys.exit(result)
def vlass_run_single():
import sys
config = mc.Config()
config.get_executors()
config.collection = COLLECTION
config.working_directory = '/usr/src/app'
config.use_local_files = False
config.logging_level = 'INFO'
config.log_to_file = False
config.task_types = [mc.TaskType.INGEST]
config.resource_id = 'ivo://cadc.nrc.ca/sc2repo'
if config.features.run_in_airflow:
temp = tempfile.NamedTemporaryFile()
mc.write_to_file(temp.name, sys.argv[2])
config.proxy_fqn = temp.name
else:
config.proxy_fqn = sys.argv[2]
config.stream = 'raw'
file_name = sys.argv[1]
if config.features.use_file_names:
vlass_name = VlassName(file_name=file_name)
else:
vlass_name = VlassName(obs_id=sys.argv[1])
ec.run_single(config, vlass_name, APPLICATION, meta_visitors=visitors,
data_visitors=None)
def report(self):
self._reporter.add_timeouts(self._organizer.timeouts)
self._reporter.add_errors(self._config.count_retries())
self._reporter.add_rejections(self._organizer.rejected_count)
msg = self._reporter.report()
self._logger.info(msg)
mc.write_to_file(self._config.report_fqn, msg)
def test_write_to_file():
content = ['a.txt', 'b.jpg', 'c.fits.gz']
test_fqn = '{}/test_out.txt'.format(TESTDATA_DIR)
if os.path.exists(test_fqn):
os.remove(test_fqn)
mc.write_to_file(test_fqn, '\n'.join(content))
assert os.path.exists(test_fqn)
def test_write_to_file():
content = ['a.txt', 'b.jpg', 'c.fits.gz']
test_fqn = f'{tc.TEST_DATA_DIR}/test_out.txt'
if os.path.exists(test_fqn):
os.remove(test_fqn)
mc.write_to_file(test_fqn, '\n'.join(content))
assert os.path.exists(test_fqn)
def cgps_run_single():
import sys
config = mc.Config()
config.get_executors()
if config.features.run_in_airflow:
temp = tempfile.NamedTemporaryFile()
mc.write_to_file(temp.name, sys.argv[2])
config.proxy = temp.name
else:
config.proxy = sys.argv[2]
if config.features.use_file_names:
storage_name = mc.StorageName(file_name=sys.argv[1])
else:
storage_name = mc.StorageName(obs_id=sys.argv[1])
ec.run_single(config, storage_name, APPLICATION, meta_visitors,
data_visitors)
def run_single():
import sys
config = mc.Config()
config.get_executors()
config.resource_id = 'ivo://cadc.nrc.ca/sc2repo'
if config.features.run_in_airflow:
temp = tempfile.NamedTemporaryFile()
mc.write_to_file(temp.name, sys.argv[2])
config.proxy = temp.name
else:
config.proxy = sys.argv[2]
config.stream = 'raw'
if config.features.use_file_names:
storage_name = GMIMSName(file_name=sys.argv[1])
else:
obs_id = GMIMSName.remove_extensions(sys.argv[1])
storage_name = GMIMSName(obs_id=obs_id)
result = ec.run_single(config, storage_name, APPLICATION, meta_visitors,
data_visitors)
sys.exit(result)
def run_single():
"""
Run the processing for a single entry.
:return 0 if successful, -1 if there's any sort of failure. Return status
is used by airflow for task instance management and reporting.
"""
config = mc.Config()
config.get_executors()
config.resource_id = 'ivo://cadc.nrc.ca/sc2repo'
if config.features.run_in_airflow:
temp = tempfile.NamedTemporaryFile()
mc.write_to_file(temp.name, sys.argv[2])
config.proxy = temp.name
else:
config.proxy = sys.argv[2]
config.stream = 'default'
if config.features.use_file_names:
storage_name = GemName(file_name=sys.argv[1])
else:
raise mc.CadcException('No code to handle running GEM by obs id.')
result = ec.run_single(config, storage_name, APPLICATION, meta_visitors,
data_visitors)
sys.exit(result)
def run_single():
"""
Run the processing for a single entry.
:return 0 if successful, -1 if there's any sort of failure. Return status
is used by airflow for task instance management and reporting.
"""
config = mc.Config()
config.get_executors()
config.resource_id = 'ivo://cadc.nrc.ca/sc2repo'
if config.features.run_in_airflow:
temp = tempfile.NamedTemporaryFile()
mc.write_to_file(temp.name, sys.argv[2])
config.proxy = temp.name
else:
config.proxy = sys.argv[2]
config.stream = 'raw'
if config.features.use_file_names:
storage_name = ec.StorageName(file_name=sys.argv[1])
else:
obs_id = ec.StorageName.remove_extensions(sys.argv[1])
storage_name = HLAName(obs_id=obs_id)
result = ec.run_single(config, storage_name, APPLICATION, meta_visitors,
data_visitors)
sys.exit(result)
def omm_run_single():
import sys
config = mc.Config()
config.get_executors()
config.collection = COLLECTION
config.working_directory = '/usr/src/app'
config.task_types = [mc.TaskType.INGEST,
mc.TaskType.MODIFY]
config.resource_id = 'ivo://cadc.nrc.ca/sc2repo'
if config.features.run_in_airflow:
temp = tempfile.NamedTemporaryFile()
mc.write_to_file(temp.name, sys.argv[2])
config.proxy_fqn = temp.name
else:
config.proxy_fqn = sys.argv[2]
config.stream = 'raw'
if config.features.use_file_names:
storage_name = OmmName(file_name=sys.argv[1])
else:
obs_id = OmmName.remove_extensions(sys.argv[1])
storage_name = OmmName(obs_id=obs_id)
result = ec.run_single(config, storage_name, APPLICATION, meta_visitors,
data_visitors, OmmChooser())
sys.exit(result)
