def ntfs2mimir(self, instance_name, input, autocomplete_version, job_id=None, dataset_uid=None):
"""
launch ntfs2mimir
"""
# We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true
if job_id:
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
else:
logger = get_task_logger(logging.getLogger("autocomplete"))
executable = "ntfs2mimir" if autocomplete_version == 2 else "ntfs2mimir7"
logger.debug('running {} for Es{}'.format(executable, autocomplete_version))
working_directory = unzip_if_needed(input)
cosmogony_file = models.DataSet.get_cosmogony_file_path()
argv = get_ntfs2mimir_params(instance_name, working_directory, autocomplete_version, cosmogony_file)
try:
res = launch_exec(executable, argv, logger)
if res != 0:
# Do not raise error because it breaks celery tasks chain.
# ntfs2mimir have to be non-blocking.
# @TODO : Find a way to raise error without breaking celery tasks chain
logger.error('{} failed'.format(executable))
if job_id:
job.state = 'failed'
models.db.session.commit()
except:
logger.exception('')
if job_id:
job.state = 'failed'
models.db.session.commit()
raise
def _inner_2s3(self, dataset_type, instance_config, filename, job_id, dataset_uid):
job = models.Job.query.get(job_id)
dataset = _retrieve_dataset_and_set_state(dataset_type, job.id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
try:
filename = zip_if_needed(filename)
config = MinioConfig()
client = Minio(endpoint=config.host, access_key=config.key, secret_key=config.secret, secure=False)
dt_now_str = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S")
tags = {
"coverage": instance_config.name,
"datetime": dt_now_str,
"data_type": dataset_type,
"tyr_data_path": filename,
}
file_key = "{coverage}/{dataset_type}.zip".format(
coverage=instance_config.name, dataset_type=dataset_type
)
with collect_metric("{dataset_type}2s3".format(dataset_type=dataset_type), job, dataset_uid):
client.fput_object(config.bucket, file_key, filename, metadata=tags, content_type="application/zip")
dataset.state = "done"
except:
logger.exception("")
job.state = "failed"
dataset.state = "failed"
raise
finally:
models.db.session.commit()
def import_last_dataset(instance_name,
background=False,
reload_kraken=False,
custom_output_dir=None):
"""
reimport the last dataset of a instance
By default the kraken is not reloaded, the '-r' switch can activate it
the custom_output_dir parameter is a subdirectory for the nav file created.
If not given, the instance default one is taken
"""
instance = models.Instance.query.filter_by(name=instance_name).first()
if not instance:
raise Exception("cannot find instance {}".format(instance_name))
files = [d.name for d in instance.last_datasets(1)]
logger = get_instance_logger(instance)
logger.info('we reimport the last dataset of %s, composed of: %s',
instance.name, files)
tasks.import_data(files,
instance,
backup_file=False,
async=background,
reload=reload_kraken,
custom_output_dir=custom_output_dir)
def reload_data(self, instance_config, job_id):
""" reload data on all kraken of this instance"""
job = models.Job.query.get(job_id)
instance = job.instance
logging.info("Unqueuing job {}, reload data of instance {}".format(
job.id, instance.name))
logger = get_instance_logger(instance)
try:
task = navitiacommon.task_pb2.Task()
task.action = navitiacommon.task_pb2.RELOAD
connection = kombu.Connection(current_app.config['CELERY_BROKER_URL'])
exchange = kombu.Exchange(instance_config.exchange,
'topic',
durable=True)
producer = connection.Producer(exchange=exchange)
logger.info("reload kraken")
producer.publish(task.SerializeToString(),
routing_key=instance.name + '.task.reload')
connection.release()
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def synonym2ed(instance_config, filename, job_id):
""" launch synonym2ed """
job = models.Job.query.get(job_id)
instance = job.instance
lock = redis.lock('tyr.lock|' + instance.name)
if not lock.acquire(blocking=False):
synonym2ed.retry(countdown=300, max_retries=10)
logger = get_instance_logger(instance)
try:
connection_string = make_connection_string(instance_config)
res = launch_exec('synonym2ed',
["-i", filename, "--connection-string", connection_string],
logger)
if res != 0:
#@TODO: exception
raise ValueError('synonym2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
finally:
lock.release()
def ed2nav(self, instance_config, job_id, custom_output_dir):
""" Launch ed2nav"""
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
try:
output_file = instance_config.target_file
if custom_output_dir:
# we change the target_filename to store it in a subdir
target_path = os.path.join(os.path.dirname(output_file),
custom_output_dir)
output_file = os.path.join(target_path,
os.path.basename(output_file))
if not os.path.exists(target_path):
os.makedirs(target_path)
connection_string = make_connection_string(instance_config)
argv = ["-o", output_file, "--connection-string", connection_string]
if 'CITIES_DATABASE_URI' in current_app.config and current_app.config[
'CITIES_DATABASE_URI']:
argv.extend([
"--cities-connection-string",
current_app.config['CITIES_DATABASE_URI']
])
res = launch_exec('ed2nav', argv, logger)
if res != 0:
raise ValueError('ed2nav failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def purge_instance(instance_id, nb_to_keep):
instance = models.Instance.query.get(instance_id)
logger = get_instance_logger(instance)
logger.info('purge of backup directories for %s', instance.name)
instance_config = load_instance_config(instance.name)
backups = set(glob.glob('{}/*'.format(instance_config.backup_directory)))
logger.info('backups are: %s', backups)
# we add the realpath not to have problems with double / or stuff like that
loaded = set(
os.path.realpath(os.path.dirname(dataset.name)) for dataset in instance.last_datasets(nb_to_keep)
)
logger.info('loaded data are: %s', loaded)
running = set(os.path.realpath(os.path.dirname(dataset.name)) for dataset in instance.running_datasets())
logger.info('running bina are: %s', running)
to_remove = [os.path.join(instance_config.backup_directory, f) for f in backups - loaded - running]
missing = [l for l in loaded if l not in backups]
if missing:
logger.error(
"MISSING backup files! impossible to find %s in the backup dir, "
"we skip the purge, repair ASAP to fix the purge",
missing,
)
return
logger.info('we remove: %s', to_remove)
for path in to_remove:
shutil.rmtree(path)
def gtfs2ed(self, instance_config, gtfs_filename, job_id):
""" Unzip gtfs file launch gtfs2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
try:
working_directory = os.path.dirname(gtfs_filename)
zip_file = zipfile.ZipFile(gtfs_filename)
zip_file.extractall(path=working_directory)
params = ["-i", working_directory]
if instance_config.aliases_file:
params.append("-a")
params.append(instance_config.aliases_file)
if instance_config.synonyms_file:
params.append("-s")
params.append(instance_config.synonyms_file)
connection_string = make_connection_string(instance_config)
params.append("--connection-string")
params.append(connection_string)
res = launch_exec("gtfs2ed", params, logger)
if res != 0:
raise ValueError('gtfs2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def fare2ed(self, instance_config, filename, job_id, dataset_uid):
""" launch fare2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
try:
working_directory = unzip_if_needed(filename)
params = ["-f", working_directory]
params.append("--connection-string")
params.append(make_connection_string(instance_config))
params.append("--local_syslog")
params.append("--log_comment")
params.append(instance_config.name)
res = launch_exec("fare2ed", params, logger)
if res != 0:
# @TODO: exception
raise ValueError('fare2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def ntfs2mimir(self, instance_name, input, job_id=None, dataset_uid=None):
"""
launch ntfs2mimir
"""
# We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true
if job_id:
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
else:
logger = get_task_logger(logging.getLogger("autocomplete"))
cnx_string = current_app.config['MIMIR_URL']
working_directory = unzip_if_needed(input)
argv = ['--input', working_directory, '--connection-string', cnx_string, '--dataset', instance_name]
try:
res = launch_exec('ntfs2mimir', argv, logger)
if res != 0:
# Do not raise error because it breaks celery tasks chain.
# ntfs2mimir have to be non-blocking.
# @TODO : Find a way to raise error without breaking celery tasks chain
logger.error('ntfs2mimir failed')
if job_id:
job.state = 'failed'
models.db.session.commit()
except:
logger.exception('')
if job_id:
job.state = 'failed'
models.db.session.commit()
raise
def gtfs2ed(self, instance_config, gtfs_filename, job_id, dataset_uid):
""" Unzip gtfs file launch gtfs2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
try:
working_directory = unzip_if_needed(gtfs_filename)
params = ["-i", working_directory]
if instance_config.aliases_file:
params.append("-a")
params.append(instance_config.aliases_file)
if instance_config.synonyms_file:
params.append("-s")
params.append(instance_config.synonyms_file)
connection_string = make_connection_string(instance_config)
params.append("--connection-string")
params.append(connection_string)
res = None
with collect_metric('gtfs2ed', job, dataset_uid):
res = launch_exec("gtfs2ed", params, logger)
if res != 0:
raise ValueError('gtfs2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def synonym2ed(self, instance_config, filename, job_id, dataset_uid):
""" launch synonym2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
try:
connection_string = make_connection_string(instance_config)
res = None
params = ["-i", filename]
params.append("--connection-string")
params.append(connection_string)
params.append("--local_syslog")
params.append("--log_comment")
params.append(instance_config.name)
with collect_metric('synonym2ed', job, dataset_uid):
res = launch_exec('synonym2ed', params, logger)
if res != 0:
# @TODO: exception
raise ValueError('synonym2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def poi2ed(self, instance_config, filename, job_id, dataset_uid):
""" launch poi2ed """
job = models.Job.query.get(job_id)
dataset = _retrieve_dataset_and_set_state("poi", job.id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
try:
working_directory = unzip_if_needed(filename)
connection_string = make_connection_string(instance_config)
res = None
params = ["-i", working_directory]
params.append("--connection-string")
params.append(connection_string)
params.append("--local_syslog")
params.append("--log_comment")
params.append(instance_config.name)
with collect_metric("poi2ed", job, dataset_uid):
res = launch_exec("poi2ed", params, logger)
if res != 0:
# @TODO: exception
raise ValueError("poi2ed failed")
dataset.state = "done"
except:
logger.exception("")
job.state = "failed"
dataset.state = "failed"
raise
finally:
models.db.session.commit()
def import_last_dataset(
instance_name, background=False, reload_kraken=False, custom_output_dir=None, nowait=False
):
"""
reimport the last dataset of a instance
By default the kraken is not reloaded, the '-r' switch can activate it
the custom_output_dir parameter is a subdirectory for the nav file created.
If not given, the instance default one is taken
By default job is not run on the workers, you need to pass --background for use them, in that case you can
also pass --nowait for not waiting the end of the job
"""
instance = models.Instance.query_existing().filter_by(name=instance_name).first()
if not instance:
raise Exception("cannot find instance {}".format(instance_name))
files = [d.name for d in instance.last_datasets(1)]
logger = get_instance_logger(instance)
logger.info('we reimport the last dataset of %s, composed of: %s', instance.name, files)
future = tasks.import_data(
files,
instance,
backup_file=False,
async=background,
reload=reload_kraken,
custom_output_dir=custom_output_dir,
)
if not nowait and future:
future.wait()
def osm2ed(self, instance_config, osm_filename, job_id, dataset_uid):
""" launch osm2ed """
job = models.Job.query.get(job_id)
instance = job.instance
if os.path.isdir(osm_filename):
osm_filename = glob.glob('{}/*.pbf'.format(osm_filename))[0]
logger = get_instance_logger(instance)
try:
connection_string = make_connection_string(instance_config)
res = None
args = ["-i", osm_filename, "--connection-string", connection_string]
for poi_type in instance.poi_types:
args.append('-p')
if poi_type.name:
args.append(u'{}={}'.format(poi_type.uri, poi_type.name))
else:
args.append(poi_type.uri)
with collect_metric('osm2ed', job, dataset_uid):
res = launch_exec('osm2ed', args, logger)
if res != 0:
#@TODO: exception
raise ValueError('osm2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def fare2ed(instance_config, filename, job_id):
""" launch fare2ed """
job = models.Job.query.get(job_id)
instance = job.instance
lock = redis.lock('tyr.lock|' + instance.name)
if not lock.acquire(blocking=False):
fare2ed.retry(countdown=300, max_retries=10)
logger = get_instance_logger(instance)
try:
working_directory = os.path.dirname(filename)
zip_file = zipfile.ZipFile(filename)
zip_file.extractall(path=working_directory)
res = launch_exec("fare2ed", ['-f', working_directory,
'--connection-string',
make_connection_string(instance_config)],
logger)
if res != 0:
#@TODO: exception
raise ValueError('fare2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
finally:
lock.release()
def nav2rt(instance_config, job_id):
""" Launch nav2rt"""
job = models.Job.query.get(job_id)
instance = job.instance
lock = redis.lock('tyr.lock|' + instance.name)
if not lock.acquire(blocking=False):
nav2rt.retry(countdown=300, max_retries=10)
logger = get_instance_logger(instance)
try:
source_filename = instance_config.tmp_file
target_filename = instance_config.target_file
connection_string = make_connection_string(instance_config)
res = launch_exec('nav2rt',
["-i", source_filename, "-o", target_filename,
"--connection-string", connection_string],
logger)
if res != 0:
raise ValueError('nav2rt failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
finally:
lock.release()
def fare2ed(self, instance_config, filename, job_id):
""" launch fare2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
try:
working_directory = os.path.dirname(filename)
zip_file = zipfile.ZipFile(filename)
zip_file.extractall(path=working_directory)
res = launch_exec("fare2ed", [
'-f', working_directory, '--connection-string',
make_connection_string(instance_config)
], logger)
if res != 0:
#@TODO: exception
raise ValueError('fare2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def poi2ed(self, instance_config, filename, job_id, dataset_uid):
""" launch poi2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
try:
working_directory = os.path.dirname(filename)
zip_file = zipfile.ZipFile(filename)
zip_file.extractall(path=working_directory)
connection_string = make_connection_string(instance_config)
res = None
with collect_metric('poi2ed', job, dataset_uid):
res = launch_exec('poi2ed', [
"-i", working_directory, "--connection-string",
connection_string
], logger)
if res != 0:
#@TODO: exception
raise ValueError('poi2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def wrapper(*args, **kwargs):
job_id = None
if 'job_id' in kwargs:
job_id = kwargs['job_id']
else:
job_id = args[func.func_code.co_varnames.index('job_id')]
logging.debug('args: %s -- kwargs: %s', args, kwargs)
job = models.Job.query.get(job_id)
logger = get_instance_logger(job.instance)
lock = redis.lock('tyr.lock|' + job.instance.name,
timeout=self.timeout)
if not lock.acquire(blocking=False):
logger.info('lock on %s retry %s in 300sec', job.instance.name,
func.__name__)
task = args[func.func_code.co_varnames.index('self')]
task.retry(countdown=60, max_retries=10)
else:
try:
logger.debug('lock acquired on %s for %s',
job.instance.name, func.__name__)
return func(*args, **kwargs)
finally:
logger.debug('release lock on %s for %s',
job.instance.name, func.__name__)
lock.release()
def wrapper(*args, **kwargs):
job_id = get_named_arg('job_id', func, args, kwargs)
logging.debug('args: %s -- kwargs: %s', args, kwargs)
job = models.Job.query.get(job_id)
logger = get_instance_logger(job.instance, task_id=job_id)
task = args[func.func_code.co_varnames.index('self')]
try:
lock = redis.lock('tyr.lock|' + job.instance.name, timeout=self.timeout)
locked = lock.acquire(blocking=False)
except ConnectionError:
logging.exception('Exception with redis while locking. Retrying in 10sec')
task.retry(countdown=10, max_retries=10)
if not locked:
countdown = 300
logger.info('lock on %s retry %s in %s sec', job.instance.name, func.__name__, countdown)
task.retry(countdown=countdown, max_retries=10)
else:
try:
logger.debug('lock acquired on %s for %s', job.instance.name, func.__name__)
return func(*args, **kwargs)
finally:
logger.debug('release lock on %s for %s', job.instance.name, func.__name__)
# sometimes we are disconnected from redis when we want to release the lock,
# so we retry only the release
try:
retrying.Retrying(stop_max_attempt_number=5, wait_fixed=1000).call(
lock_release, lock, logger
)
except ValueError: # LockError(ValueError) since redis 3.0
logger.exception(
"impossible to release lock: continue but following task may be locked :("
)
def stops2mimir(self, instance_config, input, job_id, dataset_uid):
"""
launch stops2mimir
Note: this is temporary, this will be done by tartare when tartare will be available
"""
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
cnx_string = current_app.config['MIMIR_URL']
working_directory = os.path.dirname(input)
stops_file = os.path.join(working_directory, 'stops.txt')
# Note: the dataset is for the moment the instance name, we'll need to change this when we'll aggregate
argv = [
'--input', stops_file, '--connection-string', cnx_string, '--dataset',
instance_config.name
]
try:
res = launch_exec('stops2mimir', argv, logger)
if res != 0:
raise ValueError('stops2mimir failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def poi2mimir(self, instance_name, input, job_id=None, dataset_uid=None):
""" launch poi2mimir """
dataset_name = 'priv.{}'.format(
instance_name) # We give the dataset a prefix to prevent
# collision with other datasets.
job = None
# We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true
if job_id:
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
else:
logger = get_task_logger(logging.getLogger("autocomplete"))
instance = models.Instance.query_existing().filter_by(
name=instance_name).first()
cnx_string = current_app.config['MIMIR_URL']
poi_file = input
# Note: the dataset is for the moment the instance name, we'll need to change this when we'll aggregate
argv = [
'--input', poi_file, '--connection-string', cnx_string, '--dataset',
dataset_name, '--private'
]
try:
if job:
with collect_metric('poi2mimir', job, dataset_uid):
res = launch_exec('poi2mimir', argv, logger)
else:
res = launch_exec('poi2mimir', argv, logger)
if res != 0:
# Do not raise error because it breaks celery tasks chain.
logger.error('poi2mimir failed')
if job_id:
job.state = 'failed'
models.db.session.commit()
else:
instance.poi_dataset = dataset_name
models.db.session.commit()
except:
logger.exception('')
if job_id:
job.state = 'failed'
models.db.session.commit()
raise
def osm2ed(self, instance_config, osm_filename, job_id, dataset_uid):
""" launch osm2ed """
job = models.Job.query.get(job_id)
dataset = _retrieve_dataset_and_set_state("osm", job.id)
instance = job.instance
poi_types_json = None
if instance.poi_type_json:
poi_types_json = instance.poi_type_json.poi_types_json
osm_filename = unzip_if_needed(osm_filename)
if os.path.isdir(osm_filename):
osm_filename = glob.glob('{}/*.pbf'.format(osm_filename))[0]
logger = get_instance_logger(instance, task_id=job_id)
try:
connection_string = make_connection_string(instance_config)
res = None
args = ["-i", osm_filename, "--connection-string", connection_string]
if poi_types_json:
args.append('-p')
args.append(u'{}'.format(poi_types_json))
args.append("--local_syslog")
args.append("--log_comment")
args.append(instance_config.name)
if instance.admins_from_cities_db:
cities_db = current_app.config.get('CITIES_DATABASE_URI')
if not cities_db:
raise ValueError(
"impossible to use osm2ed with cities db since no cities database configuration has been set"
)
args.extend(["--cities-connection-string", cities_db])
with collect_metric("osm2ed", job, dataset_uid):
res = launch_exec("osm2ed", args, logger)
if res != 0:
# @TODO: exception
raise ValueError("osm2ed failed")
dataset.state = "done"
except:
logger.exception("")
job.state = "failed"
dataset.state = "failed"
raise
finally:
models.db.session.commit()
def reload_data(self, instance_config, job_id):
""" reload data on all kraken of this instance"""
job = models.Job.query.get(job_id)
instance = job.instance
logging.info("Unqueuing job {}, reload data of instance {}".format(job.id, instance.name))
logger = get_instance_logger(instance)
try:
task = navitiacommon.task_pb2.Task()
task.action = navitiacommon.task_pb2.RELOAD
rabbit_mq_handler = RabbitMqHandler(current_app.config['CELERY_BROKER_URL'], instance_config.exchange, "topic")
logger.info("reload kraken")
rabbit_mq_handler.publish(task.SerializeToString(), instance.name + '.task.reload')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def ed2nav(self, instance_config, job_id, custom_output_dir):
""" Launch ed2nav"""
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
try:
output_file = instance_config.target_file
if custom_output_dir:
# we change the target_filename to store it in a subdir
target_path = os.path.join(os.path.dirname(output_file),
custom_output_dir)
output_file = os.path.join(target_path,
os.path.basename(output_file))
if not os.path.exists(target_path):
os.makedirs(target_path)
connection_string = make_connection_string(instance_config)
argv = ["-o", output_file, "--connection-string", connection_string]
if 'CITIES_DATABASE_URI' in current_app.config and current_app.config[
'CITIES_DATABASE_URI']:
argv.extend([
"--cities-connection-string",
current_app.config['CITIES_DATABASE_URI']
])
if instance.full_sn_geometries:
argv.extend(['--full_street_network_geometries'])
argv.extend(['--local_syslog'])
argv.extend(["--log_comment", instance_config.name])
res = None
with collect_metric('ed2nav', job, None):
res = launch_exec('ed2nav', argv, logger)
os.system('sync') # we sync to be safe
if res != 0:
raise ValueError('ed2nav failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def stops2mimir(self, instance_name, input, job_id=None, dataset_uid=None):
"""
launch stops2mimir
Note: this is temporary, this will be done by tartare when tartare will be available
"""
# We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true
if job_id:
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
else:
logger = get_task_logger(logging.getLogger("autocomplete"))
cnx_string = current_app.config['MIMIR_URL']
working_directory = os.path.dirname(input)
stops_file = os.path.join(working_directory, 'stops.txt')
# Note: the dataset is for the moment the instance name, we'll need to change this when we'll aggregate
argv = [
'--input', stops_file, '--connection-string', cnx_string, '--dataset',
instance_name
]
try:
res = launch_exec('stops2mimir', argv, logger)
if res != 0:
# Do not raise error because that it breaks celery tasks chain.
# stops2mimir have to be non-blocking.
# @TODO : Find a way to raise error without breaking celery tasks chain
logger.error('stops2mimir failed')
if job_id:
job.state = 'failed'
models.db.session.commit()
except:
logger.exception('')
if job_id:
job.state = 'failed'
models.db.session.commit()
raise
def synonym2ed(self, instance_config, filename, job_id):
""" launch synonym2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
try:
connection_string = make_connection_string(instance_config)
res = launch_exec(
'synonym2ed',
["-i", filename, "--connection-string", connection_string], logger)
if res != 0:
#@TODO: exception
raise ValueError('synonym2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def poi2mimir(self, instance_name, input, autocomplete_version, job_id=None, dataset_uid=None):
"""launch poi2mimir"""
dataset_name = 'priv.{}'.format(instance_name) # We give the dataset a prefix to prevent
# collision with other datasets.
job = None
# We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true
if job_id:
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
else:
logger = get_task_logger(logging.getLogger("autocomplete"))
instance = models.Instance.query_existing().filter_by(name=instance_name).first()
executable = "poi2mimir" if autocomplete_version == 2 else "poi2mimir7"
logger.debug('running {} version autocomplete {}'.format(executable, autocomplete_version))
cosmogony_file = models.DataSet.get_cosmogony_file_path()
argv = get_poi2mimir_params(input, dataset_name, autocomplete_version, cosmogony_file)
try:
if job:
with collect_metric(executable, job, dataset_uid):
res = launch_exec(executable, argv, logger)
else:
res = launch_exec(executable, argv, logger)
if res != 0:
# Do not raise error because it breaks celery tasks chain.
logger.error('{} failed'.format(executable))
if job_id:
job.state = 'failed'
models.db.session.commit()
else:
instance.poi_dataset = dataset_name
models.db.session.commit()
except:
logger.exception('')
if job_id:
job.state = 'failed'
models.db.session.commit()
raise
def fusio2ed(instance_config, filename, job_id):
""" Unzip fusio file and launch fusio2ed """
job = models.Job.query.get(job_id)
instance = job.instance
lock = redis.lock('tyr.lock|' + instance.name)
if not lock.acquire(blocking=False):
fusio2ed.retry(countdown=300, max_retries=10)
logger = get_instance_logger(instance)
try:
working_directory = os.path.dirname(filename)
zip_file = zipfile.ZipFile(filename)
zip_file.extractall(path=working_directory)
params = ["-i", working_directory]
if instance_config.aliases_file:
params.append("-a")
params.append(instance_config.aliases_file)
if instance_config.synonyms_file:
params.append("-s")
params.append(instance_config.synonyms_file)
connection_string = make_connection_string(instance_config)
params.append("--connection-string")
params.append(connection_string)
res = launch_exec("fusio2ed", params, logger)
if res != 0:
raise ValueError('fusio2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
finally:
lock.release()
