def ntfs2mimir(self, instance_name, input, autocomplete_version, job_id=None, dataset_uid=None): """ launch ntfs2mimir """ # We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true if job_id: job = models.Job.query.get(job_id) instance = job.instance logger = get_instance_logger(instance, task_id=job_id) else: logger = get_task_logger(logging.getLogger("autocomplete")) executable = "ntfs2mimir" if autocomplete_version == 2 else "ntfs2mimir7" logger.debug('running {} for Es{}'.format(executable, autocomplete_version)) working_directory = unzip_if_needed(input) cosmogony_file = models.DataSet.get_cosmogony_file_path() argv = get_ntfs2mimir_params(instance_name, working_directory, autocomplete_version, cosmogony_file) try: res = launch_exec(executable, argv, logger) if res != 0: # Do not raise error because it breaks celery tasks chain. # ntfs2mimir have to be non-blocking. # @TODO : Find a way to raise error without breaking celery tasks chain logger.error('{} failed'.format(executable)) if job_id: job.state = 'failed' models.db.session.commit() except: logger.exception('') if job_id: job.state = 'failed' models.db.session.commit() raise
def ntfs2mimir(self, instance_name, input, job_id=None, dataset_uid=None): """ launch ntfs2mimir """ # We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true if job_id: job = models.Job.query.get(job_id) instance = job.instance logger = get_instance_logger(instance, task_id=job_id) else: logger = get_task_logger(logging.getLogger("autocomplete")) cnx_string = current_app.config['MIMIR_URL'] working_directory = unzip_if_needed(input) argv = ['--input', working_directory, '--connection-string', cnx_string, '--dataset', instance_name] try: res = launch_exec('ntfs2mimir', argv, logger) if res != 0: # Do not raise error because it breaks celery tasks chain. # ntfs2mimir have to be non-blocking. # @TODO : Find a way to raise error without breaking celery tasks chain logger.error('ntfs2mimir failed') if job_id: job.state = 'failed' models.db.session.commit() except: logger.exception('') if job_id: job.state = 'failed' models.db.session.commit() raise
def poi2mimir(self, instance_name, input, job_id=None, dataset_uid=None): """ launch poi2mimir """ dataset_name = 'priv.{}'.format( instance_name) # We give the dataset a prefix to prevent # collision with other datasets. job = None # We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true if job_id: job = models.Job.query.get(job_id) instance = job.instance logger = get_instance_logger(instance, task_id=job_id) else: logger = get_task_logger(logging.getLogger("autocomplete")) instance = models.Instance.query_existing().filter_by( name=instance_name).first() cnx_string = current_app.config['MIMIR_URL'] poi_file = input # Note: the dataset is for the moment the instance name, we'll need to change this when we'll aggregate argv = [ '--input', poi_file, '--connection-string', cnx_string, '--dataset', dataset_name, '--private' ] try: if job: with collect_metric('poi2mimir', job, dataset_uid): res = launch_exec('poi2mimir', argv, logger) else: res = launch_exec('poi2mimir', argv, logger) if res != 0: # Do not raise error because it breaks celery tasks chain. logger.error('poi2mimir failed') if job_id: job.state = 'failed' models.db.session.commit() else: instance.poi_dataset = dataset_name models.db.session.commit() except: logger.exception('') if job_id: job.state = 'failed' models.db.session.commit() raise
def stops2mimir(self, instance_name, input, job_id=None, dataset_uid=None): """ launch stops2mimir Note: this is temporary, this will be done by tartare when tartare will be available """ # We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true if job_id: job = models.Job.query.get(job_id) instance = job.instance logger = get_instance_logger(instance, task_id=job_id) else: logger = get_task_logger(logging.getLogger("autocomplete")) cnx_string = current_app.config['MIMIR_URL'] working_directory = os.path.dirname(input) stops_file = os.path.join(working_directory, 'stops.txt') # Note: the dataset is for the moment the instance name, we'll need to change this when we'll aggregate argv = [ '--input', stops_file, '--connection-string', cnx_string, '--dataset', instance_name ] try: res = launch_exec('stops2mimir', argv, logger) if res != 0: # Do not raise error because that it breaks celery tasks chain. # stops2mimir have to be non-blocking. # @TODO : Find a way to raise error without breaking celery tasks chain logger.error('stops2mimir failed') if job_id: job.state = 'failed' models.db.session.commit() except: logger.exception('') if job_id: job.state = 'failed' models.db.session.commit() raise
def poi2mimir(self, instance_name, input, autocomplete_version, job_id=None, dataset_uid=None): """launch poi2mimir""" dataset_name = 'priv.{}'.format(instance_name) # We give the dataset a prefix to prevent # collision with other datasets. job = None # We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true if job_id: job = models.Job.query.get(job_id) instance = job.instance logger = get_instance_logger(instance, task_id=job_id) else: logger = get_task_logger(logging.getLogger("autocomplete")) instance = models.Instance.query_existing().filter_by(name=instance_name).first() executable = "poi2mimir" if autocomplete_version == 2 else "poi2mimir7" logger.debug('running {} version autocomplete {}'.format(executable, autocomplete_version)) cosmogony_file = models.DataSet.get_cosmogony_file_path() argv = get_poi2mimir_params(input, dataset_name, autocomplete_version, cosmogony_file) try: if job: with collect_metric(executable, job, dataset_uid): res = launch_exec(executable, argv, logger) else: res = launch_exec(executable, argv, logger) if res != 0: # Do not raise error because it breaks celery tasks chain. logger.error('{} failed'.format(executable)) if job_id: job.state = 'failed' models.db.session.commit() else: instance.poi_dataset = dataset_name models.db.session.commit() except: logger.exception('') if job_id: job.state = 'failed' models.db.session.commit() raise