def binarize(ed_db_params, output, ed_component_path, cities_db_params):
# type: (DbParams, str, str, DbParams) -> None
"""
Binarize the data from the database to a file.
:param ed_db_params: the parameters of the database
:param output: the name of the output file (usually with extension ".nav.lz4")
:param ed_component_path: the path to the "ed2nav" binary
:param cities_db_params: the parameters for the cities of the database
"""
logger.info('creating data.nav')
ed2nav = 'ed2nav'
if ed_component_path:
ed2nav = os.path.join(ed_component_path, ed2nav)
launch_exec.launch_exec(
ed2nav,
[
"-o",
output,
"--connection-string",
ed_db_params.old_school_cnx_string(),
"--cities-connection-string",
cities_db_params.old_school_cnx_string(),
],
logger,
)
logger.info("data.nav is created successfully: {}".format(output))
def poi2mimir(self, instance_name, input, job_id=None, dataset_uid=None):
""" launch poi2mimir """
dataset_name = 'priv.{}'.format(
instance_name) # We give the dataset a prefix to prevent
# collision with other datasets.
job = None
# We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true
if job_id:
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
else:
logger = get_task_logger(logging.getLogger("autocomplete"))
instance = models.Instance.query_existing().filter_by(
name=instance_name).first()
cnx_string = current_app.config['MIMIR_URL']
poi_file = input
# Note: the dataset is for the moment the instance name, we'll need to change this when we'll aggregate
argv = [
'--input', poi_file, '--connection-string', cnx_string, '--dataset',
dataset_name, '--private'
]
try:
if job:
with collect_metric('poi2mimir', job, dataset_uid):
res = launch_exec('poi2mimir', argv, logger)
else:
res = launch_exec('poi2mimir', argv, logger)
if res != 0:
# Do not raise error because it breaks celery tasks chain.
logger.error('poi2mimir failed')
if job_id:
job.state = 'failed'
models.db.session.commit()
else:
instance.poi_dataset = dataset_name
models.db.session.commit()
except:
logger.exception('')
if job_id:
job.state = 'failed'
models.db.session.commit()
raise
def osm2mimir(self, autocomplete_instance, filename, job_id, dataset_uid, autocomplete_version):
"""launch osm2mimir"""
executable = "osm2mimir" if autocomplete_version == 2 else "osm2mimir7"
autocomplete_instance = models.db.session.merge(autocomplete_instance) # reatache the object
logger = get_autocomplete_instance_logger(autocomplete_instance, task_id=job_id)
logger.debug('running {} for {}'.format(executable, job_id))
job = models.Job.query.get(job_id)
data_filename = unzip_if_needed(filename)
custom_config = "custom_config"
working_directory = os.path.dirname(data_filename)
custom_config_config_toml = '{}/{}.toml'.format(working_directory, custom_config)
data = autocomplete_instance.config_toml.encode("utf-8")
cosmogony_file = models.DataSet.get_cosmogony_file_path()
with open(custom_config_config_toml, 'w') as f:
f.write(data)
params = get_osm2mimir_params(
autocomplete_instance,
data_filename,
working_directory,
custom_config,
autocomplete_version,
cosmogony_file,
)
try:
res = launch_exec(executable, params, logger)
if res != 0:
# @TODO: exception
raise ValueError('{} failed'.format(executable))
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def openaddresses2mimir(self, autocomplete_instance, filename, job_id, dataset_uid, autocomplete_version):
"""launch openaddresses2mimir"""
executable = "openaddresses2mimir" if autocomplete_version == 2 else "openaddresses2mimir7"
autocomplete_instance = models.db.session.merge(autocomplete_instance) # reatache the object
logger = get_autocomplete_instance_logger(autocomplete_instance, task_id=job_id)
if autocomplete_instance.address != 'OA':
logger.warning(
'no open addresses data will be loaded for instance {} because the address are read from {}'.format(
autocomplete_instance.name, autocomplete_instance.address
)
)
return
job = models.Job.query.get(job_id)
working_directory = unzip_if_needed(filename)
cosmogony_file = models.DataSet.get_cosmogony_file_path()
params = get_openaddresses2mimir_params(
autocomplete_instance, working_directory, autocomplete_version, cosmogony_file
)
try:
res = launch_exec(executable, params, logger)
if res != 0:
# @TODO: exception
raise ValueError('{} failed'.format(executable))
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def ntfs2mimir(self, instance_name, input, autocomplete_version, job_id=None, dataset_uid=None):
"""
launch ntfs2mimir
"""
# We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true
if job_id:
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
else:
logger = get_task_logger(logging.getLogger("autocomplete"))
executable = "ntfs2mimir" if autocomplete_version == 2 else "ntfs2mimir7"
logger.debug('running {} for Es{}'.format(executable, autocomplete_version))
working_directory = unzip_if_needed(input)
cosmogony_file = models.DataSet.get_cosmogony_file_path()
argv = get_ntfs2mimir_params(instance_name, working_directory, autocomplete_version, cosmogony_file)
try:
res = launch_exec(executable, argv, logger)
if res != 0:
# Do not raise error because it breaks celery tasks chain.
# ntfs2mimir have to be non-blocking.
# @TODO : Find a way to raise error without breaking celery tasks chain
logger.error('{} failed'.format(executable))
if job_id:
job.state = 'failed'
models.db.session.commit()
except:
logger.exception('')
if job_id:
job.state = 'failed'
models.db.session.commit()
raise
def fare2ed(self, instance_config, filename, job_id):
""" launch fare2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
try:
working_directory = os.path.dirname(filename)
zip_file = zipfile.ZipFile(filename)
zip_file.extractall(path=working_directory)
res = launch_exec("fare2ed", [
'-f', working_directory, '--connection-string',
make_connection_string(instance_config)
], logger)
if res != 0:
#@TODO: exception
raise ValueError('fare2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def bano2mimir(self, autocomplete_instance, filename, job_id, dataset_uid):
""" launch bano2mimir """
autocomplete_instance = models.db.session.merge(
autocomplete_instance) #reatache the object
logger = get_autocomplete_instance_logger(autocomplete_instance,
task_id=job_id)
job = models.Job.query.get(job_id)
cnx_string = current_app.config['MIMIR_URL']
working_directory = unzip_if_needed(filename)
if autocomplete_instance.address != 'BANO':
logger.warn(
'no bano data will be loaded for instance {} because the address are read from {}'
.format(autocomplete_instance.name, autocomplete_instance.address))
return
try:
res = launch_exec("bano2mimir", [
'-i', working_directory, '--connection-string', cnx_string,
'--dataset', autocomplete_instance.name
], logger)
if res != 0:
#@TODO: exception
raise ValueError('bano2mimir failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def fare2ed(self, instance_config, filename, job_id, dataset_uid):
""" launch fare2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
try:
working_directory = unzip_if_needed(filename)
params = ["-f", working_directory]
params.append("--connection-string")
params.append(make_connection_string(instance_config))
params.append("--local_syslog")
params.append("--log_comment")
params.append(instance_config.name)
res = launch_exec("fare2ed", params, logger)
if res != 0:
# @TODO: exception
raise ValueError('fare2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def stops2mimir(self, instance_config, input, job_id, dataset_uid):
"""
launch stops2mimir
Note: this is temporary, this will be done by tartare when tartare will be available
"""
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
cnx_string = current_app.config['MIMIR_URL']
working_directory = os.path.dirname(input)
stops_file = os.path.join(working_directory, 'stops.txt')
# Note: the dataset is for the moment the instance name, we'll need to change this when we'll aggregate
argv = [
'--input', stops_file, '--connection-string', cnx_string, '--dataset',
instance_config.name
]
try:
res = launch_exec('stops2mimir', argv, logger)
if res != 0:
raise ValueError('stops2mimir failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def cosmogony2mimir(self, autocomplete_instance, filename, job_id,
dataset_uid):
""" launch cosmogony2mimir """
autocomplete_instance = models.db.session.merge(
autocomplete_instance) # reattach the object
logger = get_autocomplete_instance_logger(autocomplete_instance,
task_id=job_id)
logger.debug('running cosmogony2mimir for {}'.format(job_id))
job = models.Job.query.get(job_id)
cnx_string = current_app.config['MIMIR_URL']
cosmo_file = unzip_if_needed(filename)
params = [
'--input',
cosmo_file,
'--connection-string',
cnx_string,
'--dataset',
autocomplete_instance.name,
'--french-id-retrocompatibility',
]
try:
res = launch_exec("cosmogony2mimir", params, logger)
if res != 0:
# @TODO: exception
raise ValueError('cosmogony2mimir failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def import_data(data_dir, db_params, ed_component_path):
# type: (str, DbParams, str) -> None
"""
Call the right binary for its data (all the "*2ed") to create data then load it in the database.
:param data_dir: the directory containing the data for "*2ed"
:param db_params: the parameters of the database
:param ed_component_path: the path of the directory containing the binary "*2ed"
"""
files = glob.glob(data_dir + "/*") # type: List[str]
data_type, file_to_load = utils.type_of_data(files) # type: str,str
if not data_type:
logger.info('unknown data type for dir {}, skipping'.format(data_dir))
return
# we consider that we only have to load one kind of data per directory
import_component = data_type + '2ed' # type: str
if ed_component_path:
import_component = os.path.join(ed_component_path, import_component)
if file_to_load.endswith('.zip') or file_to_load.endswith('.geopal'):
# TODO: handle geopal as non zip ; if it's a zip, we unzip it
zip_file = zipfile.ZipFile(file_to_load) # type: zipfile.ZipFile
zip_file.extractall(path=data_dir)
file_to_load = data_dir
if launch_exec.launch_exec(import_component, [
"-i", file_to_load, "--connection-string",
db_params.old_school_cnx_string()
], logger):
raise Exception(
'Error: problem with running {}, stoping'.format(import_component))
def osm2mimir(self, autocomplete_instance, filename, job_id, dataset_uid):
""" launch osm2mimir """
logger = logging.getLogger("autocomplete")
logger.debug('running osm2mimir for {}'.format(job_id))
job = models.Job.query.get(job_id)
cnx_string = current_app.config['MIMIR_URL'] + '/' + MIMIR_INDEX
working_directory = unzip_if_needed(filename)
autocomplete_instance = models.db.session.merge(autocomplete_instance)#reatache the object
try:
params = ['-i', working_directory, '--connection-string', cnx_string]
for lvl in autocomplete_instance.admin_level:
params.append('--level')
params.append(str(lvl))
if autocomplete_instance.admin in utils.admin_source_types:
params.append('--import-admin')
if autocomplete_instance.street in utils.street_source_types:
params.append('--import-way')
res = launch_exec("osm2mimir",
params,
logger)
if res != 0:
#@TODO: exception
raise ValueError('osm2mimir failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def synonym2ed(self, instance_config, filename, job_id, dataset_uid):
""" launch synonym2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
try:
connection_string = make_connection_string(instance_config)
res = None
params = ["-i", filename]
params.append("--connection-string")
params.append(connection_string)
params.append("--local_syslog")
params.append("--log_comment")
params.append(instance_config.name)
with collect_metric('synonym2ed', job, dataset_uid):
res = launch_exec('synonym2ed', params, logger)
if res != 0:
# @TODO: exception
raise ValueError('synonym2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def osm2ed(self, instance_config, osm_filename, job_id, dataset_uid):
""" launch osm2ed """
job = models.Job.query.get(job_id)
instance = job.instance
if os.path.isdir(osm_filename):
osm_filename = glob.glob('{}/*.pbf'.format(osm_filename))[0]
logger = get_instance_logger(instance)
try:
connection_string = make_connection_string(instance_config)
res = None
args = ["-i", osm_filename, "--connection-string", connection_string]
for poi_type in instance.poi_types:
args.append('-p')
if poi_type.name:
args.append(u'{}={}'.format(poi_type.uri, poi_type.name))
else:
args.append(poi_type.uri)
with collect_metric('osm2ed', job, dataset_uid):
res = launch_exec('osm2ed', args, logger)
if res != 0:
#@TODO: exception
raise ValueError('osm2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def ed2nav(self, instance_config, job_id, custom_output_dir):
""" Launch ed2nav"""
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
try:
output_file = instance_config.target_file
if custom_output_dir:
# we change the target_filename to store it in a subdir
target_path = os.path.join(os.path.dirname(output_file),
custom_output_dir)
output_file = os.path.join(target_path,
os.path.basename(output_file))
if not os.path.exists(target_path):
os.makedirs(target_path)
connection_string = make_connection_string(instance_config)
argv = ["-o", output_file, "--connection-string", connection_string]
if 'CITIES_DATABASE_URI' in current_app.config and current_app.config[
'CITIES_DATABASE_URI']:
argv.extend([
"--cities-connection-string",
current_app.config['CITIES_DATABASE_URI']
])
res = launch_exec('ed2nav', argv, logger)
if res != 0:
raise ValueError('ed2nav failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def cities(file_path, job_id, exe):
""" Launch 'cities' or 'cosmogony2cities' """
job = models.Job.query.get(job_id)
res = -1
try:
res = launch_exec(
"{}".format(exe),
[
'-i', file_path, '--connection-string',
current_app.config['CITIES_DATABASE_URI']
],
logging,
)
if res != 0:
job.state = 'failed'
logging.error('{} failed'.format(exe))
else:
job.state = 'done'
except Exception as e:
logging.exception('{} exception : {}'.format(exe, e.message))
job.state = 'failed'
models.db.session.commit()
raise
models.db.session.commit()
logging.info('Import of {} finished'.format(exe))
return res
def gtfs2ed(self, instance_config, gtfs_filename, job_id):
""" Unzip gtfs file launch gtfs2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
try:
working_directory = os.path.dirname(gtfs_filename)
zip_file = zipfile.ZipFile(gtfs_filename)
zip_file.extractall(path=working_directory)
params = ["-i", working_directory]
if instance_config.aliases_file:
params.append("-a")
params.append(instance_config.aliases_file)
if instance_config.synonyms_file:
params.append("-s")
params.append(instance_config.synonyms_file)
connection_string = make_connection_string(instance_config)
params.append("--connection-string")
params.append(connection_string)
res = launch_exec("gtfs2ed", params, logger)
if res != 0:
raise ValueError('gtfs2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def ntfs2mimir(self, instance_name, input, job_id=None, dataset_uid=None):
"""
launch ntfs2mimir
"""
# We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true
if job_id:
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
else:
logger = get_task_logger(logging.getLogger("autocomplete"))
cnx_string = current_app.config['MIMIR_URL']
working_directory = unzip_if_needed(input)
argv = ['--input', working_directory, '--connection-string', cnx_string, '--dataset', instance_name]
try:
res = launch_exec('ntfs2mimir', argv, logger)
if res != 0:
# Do not raise error because it breaks celery tasks chain.
# ntfs2mimir have to be non-blocking.
# @TODO : Find a way to raise error without breaking celery tasks chain
logger.error('ntfs2mimir failed')
if job_id:
job.state = 'failed'
models.db.session.commit()
except:
logger.exception('')
if job_id:
job.state = 'failed'
models.db.session.commit()
raise
def poi2ed(self, instance_config, filename, job_id, dataset_uid):
""" launch poi2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
try:
working_directory = os.path.dirname(filename)
zip_file = zipfile.ZipFile(filename)
zip_file.extractall(path=working_directory)
connection_string = make_connection_string(instance_config)
res = None
with collect_metric('poi2ed', job, dataset_uid):
res = launch_exec('poi2ed', [
"-i", working_directory, "--connection-string",
connection_string
], logger)
if res != 0:
#@TODO: exception
raise ValueError('poi2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def gtfs2ed(self, instance_config, gtfs_filename, job_id, dataset_uid):
""" Unzip gtfs file launch gtfs2ed """
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance)
try:
working_directory = unzip_if_needed(gtfs_filename)
params = ["-i", working_directory]
if instance_config.aliases_file:
params.append("-a")
params.append(instance_config.aliases_file)
if instance_config.synonyms_file:
params.append("-s")
params.append(instance_config.synonyms_file)
connection_string = make_connection_string(instance_config)
params.append("--connection-string")
params.append(connection_string)
res = None
with collect_metric('gtfs2ed', job, dataset_uid):
res = launch_exec("gtfs2ed", params, logger)
if res != 0:
raise ValueError('gtfs2ed failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def poi2ed(self, instance_config, filename, job_id, dataset_uid):
""" launch poi2ed """
job = models.Job.query.get(job_id)
dataset = _retrieve_dataset_and_set_state("poi", job.id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
try:
working_directory = unzip_if_needed(filename)
connection_string = make_connection_string(instance_config)
res = None
params = ["-i", working_directory]
params.append("--connection-string")
params.append(connection_string)
params.append("--local_syslog")
params.append("--log_comment")
params.append(instance_config.name)
with collect_metric("poi2ed", job, dataset_uid):
res = launch_exec("poi2ed", params, logger)
if res != 0:
# @TODO: exception
raise ValueError("poi2ed failed")
dataset.state = "done"
except:
logger.exception("")
job.state = "failed"
dataset.state = "failed"
raise
finally:
models.db.session.commit()
def poi2mimir(self, instance_name, input, autocomplete_version, job_id=None, dataset_uid=None):
"""launch poi2mimir"""
dataset_name = 'priv.{}'.format(instance_name) # We give the dataset a prefix to prevent
# collision with other datasets.
job = None
# We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true
if job_id:
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
else:
logger = get_task_logger(logging.getLogger("autocomplete"))
instance = models.Instance.query_existing().filter_by(name=instance_name).first()
executable = "poi2mimir" if autocomplete_version == 2 else "poi2mimir7"
logger.debug('running {} version autocomplete {}'.format(executable, autocomplete_version))
cosmogony_file = models.DataSet.get_cosmogony_file_path()
argv = get_poi2mimir_params(input, dataset_name, autocomplete_version, cosmogony_file)
try:
if job:
with collect_metric(executable, job, dataset_uid):
res = launch_exec(executable, argv, logger)
else:
res = launch_exec(executable, argv, logger)
if res != 0:
# Do not raise error because it breaks celery tasks chain.
logger.error('{} failed'.format(executable))
if job_id:
job.state = 'failed'
models.db.session.commit()
else:
instance.poi_dataset = dataset_name
models.db.session.commit()
except:
logger.exception('')
if job_id:
job.state = 'failed'
models.db.session.commit()
raise
def cities(osm_path):
""" launch cities """
res = -1
try:
res = launch_exec("cities", ['-i', osm_path,
'--connection-string',
current_app.config['CITIES_DATABASE_URI']],
logging)
if res!=0:
logging.error('cities failed')
except:
logging.exception('')
logging.info('Import of cities finished')
return res
def cosmogony2cities(cosmogony_path):
""" launch cosmogony2cities """
res = -1
try:
res = launch_exec(
"cosmogony2cities",
['--input', cosmogony_path, '--connection-string', current_app.config['CITIES_DATABASE_URI']],
logging,
)
if res != 0:
logging.error('cosmogony2cities failed')
except Exception as e:
logging.exception('cosmogony2cities exception : {}'.format(e.message))
logging.info('Import of cosmogony2cities finished')
return res
def osm2ed(self, instance_config, osm_filename, job_id, dataset_uid):
""" launch osm2ed """
job = models.Job.query.get(job_id)
dataset = _retrieve_dataset_and_set_state("osm", job.id)
instance = job.instance
poi_types_json = None
if instance.poi_type_json:
poi_types_json = instance.poi_type_json.poi_types_json
osm_filename = unzip_if_needed(osm_filename)
if os.path.isdir(osm_filename):
osm_filename = glob.glob('{}/*.pbf'.format(osm_filename))[0]
logger = get_instance_logger(instance, task_id=job_id)
try:
connection_string = make_connection_string(instance_config)
res = None
args = ["-i", osm_filename, "--connection-string", connection_string]
if poi_types_json:
args.append('-p')
args.append(u'{}'.format(poi_types_json))
args.append("--local_syslog")
args.append("--log_comment")
args.append(instance_config.name)
if instance.admins_from_cities_db:
cities_db = current_app.config.get('CITIES_DATABASE_URI')
if not cities_db:
raise ValueError(
"impossible to use osm2ed with cities db since no cities database configuration has been set"
)
args.extend(["--cities-connection-string", cities_db])
with collect_metric("osm2ed", job, dataset_uid):
res = launch_exec("osm2ed", args, logger)
if res != 0:
# @TODO: exception
raise ValueError("osm2ed failed")
dataset.state = "done"
except:
logger.exception("")
job.state = "failed"
dataset.state = "failed"
raise
finally:
models.db.session.commit()
def load_cities(cities_file, cities_db_params, cities_exec_path):
# type: (str, DbParams, str) -> None
"""
Load cities in the database.
:param cities_file: the path to the directory containing the data for the "ed2nav" binary
:param cities_db_params: the parameters of the database
:param cities_exec_path: the path of the directory containing the "cities" binary
"""
cities_exec = os.path.join(cities_exec_path, 'cities') # type: str
if launch_exec.launch_exec(cities_exec, [
"-i", cities_file, "--connection-string",
cities_db_params.old_school_cnx_string()
], logger):
raise Exception(
'Error: problem with running {}, stoping'.format(cities_exec))
def cosmogony2mimir(self, autocomplete_instance, filename, job_id, dataset_uid, autocomplete_version):
executable = "cosmogony2mimir" if autocomplete_version == 2 else "cosmogony2mimir7"
autocomplete_instance = models.db.session.merge(autocomplete_instance) # reattach the object
logger = get_autocomplete_instance_logger(autocomplete_instance, task_id=job_id)
logger.debug('running {} for {}, version autocomplete {}'.format(executable, job_id, autocomplete_version))
job = models.Job.query.get(job_id)
cosmo_file = unzip_if_needed(filename)
params = get_cosmogony2mimir_params(cosmo_file, autocomplete_instance, autocomplete_version)
try:
res = launch_exec(executable, params, logger)
if res != 0:
# @TODO: exception
raise ValueError('{} failed'.format(executable))
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def bano2mimir(self, autocomplete_instance, filename, job_id, dataset_uid):
""" launch bano2mimir """
logger = logging.getLogger("autocomplete")
job = models.Job.query.get(job_id)
cnx_string = current_app.config['MIMIR_URL'] + '/' + MIMIR_INDEX
working_directory = unzip_if_needed(filename)
try:
res = launch_exec("bano2mimir",
['-i', working_directory, '--connection-string', cnx_string],
logger)
if res != 0:
#@TODO: exception
raise ValueError('bano2mimir failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def ed2nav(self, instance_config, job_id, custom_output_dir):
""" Launch ed2nav"""
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
try:
output_file = instance_config.target_file
if custom_output_dir:
# we change the target_filename to store it in a subdir
target_path = os.path.join(os.path.dirname(output_file),
custom_output_dir)
output_file = os.path.join(target_path,
os.path.basename(output_file))
if not os.path.exists(target_path):
os.makedirs(target_path)
connection_string = make_connection_string(instance_config)
argv = ["-o", output_file, "--connection-string", connection_string]
if 'CITIES_DATABASE_URI' in current_app.config and current_app.config[
'CITIES_DATABASE_URI']:
argv.extend([
"--cities-connection-string",
current_app.config['CITIES_DATABASE_URI']
])
if instance.full_sn_geometries:
argv.extend(['--full_street_network_geometries'])
argv.extend(['--local_syslog'])
argv.extend(["--log_comment", instance_config.name])
res = None
with collect_metric('ed2nav', job, None):
res = launch_exec('ed2nav', argv, logger)
os.system('sync') # we sync to be safe
if res != 0:
raise ValueError('ed2nav failed')
except:
logger.exception('')
job.state = 'failed'
models.db.session.commit()
raise
def stops2mimir(self, instance_name, input, job_id=None, dataset_uid=None):
"""
launch stops2mimir
Note: this is temporary, this will be done by tartare when tartare will be available
"""
# We don't have job_id while doing a reimport of all instances with import_stops_in_mimir = true
if job_id:
job = models.Job.query.get(job_id)
instance = job.instance
logger = get_instance_logger(instance, task_id=job_id)
else:
logger = get_task_logger(logging.getLogger("autocomplete"))
cnx_string = current_app.config['MIMIR_URL']
working_directory = os.path.dirname(input)
stops_file = os.path.join(working_directory, 'stops.txt')
# Note: the dataset is for the moment the instance name, we'll need to change this when we'll aggregate
argv = [
'--input', stops_file, '--connection-string', cnx_string, '--dataset',
instance_name
]
try:
res = launch_exec('stops2mimir', argv, logger)
if res != 0:
# Do not raise error because that it breaks celery tasks chain.
# stops2mimir have to be non-blocking.
# @TODO : Find a way to raise error without breaking celery tasks chain
logger.error('stops2mimir failed')
if job_id:
job.state = 'failed'
models.db.session.commit()
except:
logger.exception('')
if job_id:
job.state = 'failed'
models.db.session.commit()
raise
