def run_query(): """ Run the processing for all the entries returned from a time-boxed ad query. :param sys.argv[1] the timestamp for the > comparison in the time-boxed query :param sys.argv[2] the timestamp for the <= comparison in the time-boxed query :return 0 if successful, -1 if there's any sort of failure. Return status is used by airflow for task instance management and reporting. """ prev_exec_date = sys.argv[1] exec_date = sys.argv[2] config = mc.Config() config.get() config.stream = 'default' file_list = mc.read_file_list_from_archive(config, APPLICATION, prev_exec_date, exec_date) sys.argv = sys.argv[:1] result = 0 if len(file_list) > 0: mc.write_to_file(config.work_fqn, '\n'.join(file_list)) result |= ec.run_by_file(GemName, APPLICATION, COLLECTION, config.proxy_fqn, meta_visitors, data_visitors, archive=ARCHIVE) sys.exit(result)
def vlass_run_single(): import sys config = mc.Config() config.get_executors() config.collection = COLLECTION config.working_directory = '/usr/src/app' config.use_local_files = False config.logging_level = 'INFO' config.log_to_file = False config.task_types = [mc.TaskType.INGEST] config.resource_id = 'ivo://cadc.nrc.ca/sc2repo' if config.features.run_in_airflow: temp = tempfile.NamedTemporaryFile() mc.write_to_file(temp.name, sys.argv[2]) config.proxy_fqn = temp.name else: config.proxy_fqn = sys.argv[2] config.stream = 'raw' file_name = sys.argv[1] if config.features.use_file_names: vlass_name = VlassName(file_name=file_name) else: vlass_name = VlassName(obs_id=sys.argv[1]) ec.run_single(config, vlass_name, APPLICATION, meta_visitors=visitors, data_visitors=None)
def report(self): self._reporter.add_timeouts(self._organizer.timeouts) self._reporter.add_errors(self._config.count_retries()) self._reporter.add_rejections(self._organizer.rejected_count) msg = self._reporter.report() self._logger.info(msg) mc.write_to_file(self._config.report_fqn, msg)
def test_write_to_file(): content = ['a.txt', 'b.jpg', 'c.fits.gz'] test_fqn = '{}/test_out.txt'.format(TESTDATA_DIR) if os.path.exists(test_fqn): os.remove(test_fqn) mc.write_to_file(test_fqn, '\n'.join(content)) assert os.path.exists(test_fqn)
def test_write_to_file(): content = ['a.txt', 'b.jpg', 'c.fits.gz'] test_fqn = f'{tc.TEST_DATA_DIR}/test_out.txt' if os.path.exists(test_fqn): os.remove(test_fqn) mc.write_to_file(test_fqn, '\n'.join(content)) assert os.path.exists(test_fqn)
def cgps_run_single(): import sys config = mc.Config() config.get_executors() if config.features.run_in_airflow: temp = tempfile.NamedTemporaryFile() mc.write_to_file(temp.name, sys.argv[2]) config.proxy = temp.name else: config.proxy = sys.argv[2] if config.features.use_file_names: storage_name = mc.StorageName(file_name=sys.argv[1]) else: storage_name = mc.StorageName(obs_id=sys.argv[1]) ec.run_single(config, storage_name, APPLICATION, meta_visitors, data_visitors)
def run_single(): import sys config = mc.Config() config.get_executors() config.resource_id = 'ivo://cadc.nrc.ca/sc2repo' if config.features.run_in_airflow: temp = tempfile.NamedTemporaryFile() mc.write_to_file(temp.name, sys.argv[2]) config.proxy = temp.name else: config.proxy = sys.argv[2] config.stream = 'raw' if config.features.use_file_names: storage_name = GMIMSName(file_name=sys.argv[1]) else: obs_id = GMIMSName.remove_extensions(sys.argv[1]) storage_name = GMIMSName(obs_id=obs_id) result = ec.run_single(config, storage_name, APPLICATION, meta_visitors, data_visitors) sys.exit(result)
def run_single(): """ Run the processing for a single entry. :return 0 if successful, -1 if there's any sort of failure. Return status is used by airflow for task instance management and reporting. """ config = mc.Config() config.get_executors() config.resource_id = 'ivo://cadc.nrc.ca/sc2repo' if config.features.run_in_airflow: temp = tempfile.NamedTemporaryFile() mc.write_to_file(temp.name, sys.argv[2]) config.proxy = temp.name else: config.proxy = sys.argv[2] config.stream = 'default' if config.features.use_file_names: storage_name = GemName(file_name=sys.argv[1]) else: raise mc.CadcException('No code to handle running GEM by obs id.') result = ec.run_single(config, storage_name, APPLICATION, meta_visitors, data_visitors) sys.exit(result)
def run_single(): """ Run the processing for a single entry. :return 0 if successful, -1 if there's any sort of failure. Return status is used by airflow for task instance management and reporting. """ config = mc.Config() config.get_executors() config.resource_id = 'ivo://cadc.nrc.ca/sc2repo' if config.features.run_in_airflow: temp = tempfile.NamedTemporaryFile() mc.write_to_file(temp.name, sys.argv[2]) config.proxy = temp.name else: config.proxy = sys.argv[2] config.stream = 'raw' if config.features.use_file_names: storage_name = ec.StorageName(file_name=sys.argv[1]) else: obs_id = ec.StorageName.remove_extensions(sys.argv[1]) storage_name = HLAName(obs_id=obs_id) result = ec.run_single(config, storage_name, APPLICATION, meta_visitors, data_visitors) sys.exit(result)
def omm_run_single(): import sys config = mc.Config() config.get_executors() config.collection = COLLECTION config.working_directory = '/usr/src/app' config.task_types = [mc.TaskType.INGEST, mc.TaskType.MODIFY] config.resource_id = 'ivo://cadc.nrc.ca/sc2repo' if config.features.run_in_airflow: temp = tempfile.NamedTemporaryFile() mc.write_to_file(temp.name, sys.argv[2]) config.proxy_fqn = temp.name else: config.proxy_fqn = sys.argv[2] config.stream = 'raw' if config.features.use_file_names: storage_name = OmmName(file_name=sys.argv[1]) else: obs_id = OmmName.remove_extensions(sys.argv[1]) storage_name = OmmName(obs_id=obs_id) result = ec.run_single(config, storage_name, APPLICATION, meta_visitors, data_visitors, OmmChooser()) sys.exit(result)