def import_in_mimir(_file, instance, asynchronous=True):
"""
Import pt data stops to autocomplete
"""
datatype, _ = utils.type_of_data(_file)
family_type = utils.family_of_data(datatype)
current_app.logger.debug("Import {} data to mimir".format(family_type))
action = None
if family_type == 'pt':
if instance.import_ntfs_in_mimir:
action = ntfs2mimir.si(instance.name, _file)
if instance.import_stops_in_mimir and not instance.import_ntfs_in_mimir:
action = stops2mimir.si(instance.name, _file)
elif family_type == 'poi':
action = poi2mimir.si(instance.name, _file)
else:
current_app.logger.warning("Unsupported family_type {}".format(family_type))
if asynchronous:
return action.delay()
else:
# all job are run in sequence and import_in_mimir will only return when all the jobs are finish
return action.apply()
def import_in_mimir(_file, instance, asynchronous=True):
"""
Import pt data stops to autocomplete
"""
datatype, _ = utils.type_of_data(_file)
family_type = utils.family_of_data(datatype)
current_app.logger.debug("Import {} data to mimir".format(family_type))
action = None
if family_type == 'pt':
if instance.import_ntfs_in_mimir:
action = ntfs2mimir.si(instance.name, _file)
# Deprecated: https://github.com/CanalTP/mimirsbrunn/blob/4430eed1d81247fffa7cf32ba675a9c5ad8b1cbe/documentation/components.md#stops2mimir
if instance.import_stops_in_mimir and not instance.import_ntfs_in_mimir:
action = stops2mimir.si(instance.name, _file)
elif family_type == 'poi':
action = poi2mimir.si(instance.name, _file)
else:
current_app.logger.warning("Unsupported family_type {}".format(family_type))
if asynchronous:
return action.delay()
else:
# all job are run in sequence and import_in_mimir will only return when all the jobs are finish
return action.apply()
def import_in_mimir(_file, instance, asynchronous=True):
"""
Import pt data stops to autocomplete
"""
datatype, _ = utils.type_of_data(_file)
family_type = utils.family_of_data(datatype)
current_app.logger.debug("Import {} data to mimir".format(family_type))
actions = []
for version in (2, 7):
if not is_activate_autocomplete_version(version):
logging.getLogger(__name__).info(
"Disable import mimir version {}".format(version))
continue
if family_type == 'pt':
if instance.import_ntfs_in_mimir:
actions.append(ntfs2mimir.si(instance.name, _file, version))
# Deprecated: https://github.com/hove-io/mimirsbrunn/blob/4430eed1d81247fffa7cf32ba675a9c5ad8b1cbe/documentation/components.md#stops2mimir
if instance.import_stops_in_mimir and not instance.import_ntfs_in_mimir:
actions.append(stops2mimir.si(instance.name, _file, version))
elif family_type == 'poi':
actions.append(poi2mimir.si(instance.name, _file, version))
else:
current_app.logger.warning(
"Unsupported family_type {}".format(family_type))
if asynchronous:
return chain(*actions).delay()
else:
# all job are run in sequence and import_in_mimir will only return when all the jobs are finish
return chain(*actions).apply()
def send_to_mimir(instance, filename, family_type):
"""
:param instance: instance to receive the data
:param filename: file to inject towards mimir
:param family_type: dataset's family type
- create a job with a data_set
- data injection towards mimir(stops2mimir, ntfs2mimir, poi2mimir)
returns action list
"""
# if mimir isn't setup do not try to import data for the autocompletion
if not current_app.config.get('MIMIR_URL'):
return []
# Bail out if the family type is not one that mimir deals with.
if family_type not in ['pt', 'poi']:
return []
# This test is to avoid creating a new job if there is no action on mimir.
if not (instance.import_ntfs_in_mimir or instance.import_stops_in_mimir):
return []
actions = []
job = models.Job()
job.instance = instance
job.state = 'running'
dataset = models.DataSet()
dataset.family_type = 'mimir'
dataset.type = 'fusio'
# currently the name of a dataset is the path to it
dataset.name = filename
models.db.session.add(dataset)
job.data_sets.append(dataset)
models.db.session.add(job)
models.db.session.commit()
if family_type == 'pt':
# Import ntfs in Mimir
if instance.import_ntfs_in_mimir:
actions.append(ntfs2mimir.si(instance.name, filename, job.id, dataset_uid=dataset.uid))
# Import stops in Mimir.
# if we are loading pt data we might want to load the stops to autocomplete
# This action is deprecated: https://github.com/CanalTP/mimirsbrunn/blob/4430eed1d81247fffa7cf32ba675a9c5ad8b1cbe/documentation/components.md#stops2mimir
if instance.import_stops_in_mimir and not instance.import_ntfs_in_mimir:
actions.append(stops2mimir.si(instance.name, filename, job.id, dataset_uid=dataset.uid))
else: # assume family_type == 'poi':
actions.append(poi2mimir.si(instance.name, filename, job.id, dataset_uid=dataset.uid))
actions.append(finish_job.si(job.id))
return actions
Import pt data stops to autocomplete
"""
datatype, _ = utils.type_of_data(_file)
family_type = utils.family_of_data(datatype)
current_app.logger.debug("Import {} data to mimir".format(family_type))
action = None
if family_type == 'pt':
if instance.import_ntfs_in_mimir:
action = ntfs2mimir.si(instance.name, _file)
if instance.import_stops_in_mimir and not instance.import_ntfs_in_mimir:
action = stops2mimir.si(instance.name, _file)
elif family_type == 'poi':
action = poi2mimir.si(instance.name, _file)
else:
current_app.logger.warning(
"Unsupported family_type {}".format(family_type))
if async:
return action.delay()
else:
# all job are run in sequence and import_in_mimir will only return when all the jobs are finish
return action.apply()
@celery.task()
def update_autocomplete():
current_app.logger.debug("Update autocomplete data")
autocomplete_dir = current_app.config['TYR_AUTOCOMPLETE_DIR']
def send_to_mimir(instance, filename, family_type):
"""
:param instance: instance to receive the data
:param filename: file to inject towards mimir
:param family_type: dataset's family type
- create a job with a data_set
- data injection towards mimir(stops2mimir, ntfs2mimir, poi2mimir)
returns action list
"""
# if mimir isn't setup do not try to import data for the autocompletion
if not any([
is_activate_autocomplete_version(2)
or is_activate_autocomplete_version(7)
]):
return []
# Bail out if the family type is not one that mimir deals with.
if family_type not in ['pt', 'poi']:
return []
# This test is to avoid creating a new job if there is no action on mimir.
if not (instance.import_ntfs_in_mimir or instance.import_stops_in_mimir):
return []
actions = []
job = models.Job()
job.instance = instance
job.state = 'running'
if is_activate_autocomplete_version(7):
dataset_es7 = create_and_get_dataset(ds_type="fusio",
family_type="mimir7",
filename=filename)
models.db.session.add(dataset_es7)
job.data_sets.append(dataset_es7)
if is_activate_autocomplete_version(2):
dataset_es2 = create_and_get_dataset(ds_type="fusio",
family_type="mimir",
filename=filename)
models.db.session.add(dataset_es2)
job.data_sets.append(dataset_es2)
models.db.session.add(job)
models.db.session.commit()
for version in (2, 7):
if not is_activate_autocomplete_version(version):
logging.getLogger(__name__).info(
"Disable import mimir version {}".format(version))
continue
ds = dataset_es7 if version == 7 else dataset_es2
if family_type == 'pt':
# Import ntfs in Mimir
if instance.import_ntfs_in_mimir:
actions.append(
ntfs2mimir.si(instance.name,
filename,
version,
job.id,
dataset_uid=ds.uid))
# Import stops in Mimir.
# if we are loading pt data we might want to load the stops to autocomplete
# This action is deprecated: https://github.com/hove-io/mimirsbrunn/blob/4430eed1d81247fffa7cf32ba675a9c5ad8b1cbe/documentation/components.md#stops2mimir
if instance.import_stops_in_mimir and not instance.import_ntfs_in_mimir:
actions.append(
stops2mimir.si(instance.name,
filename,
version,
job.id,
dataset_uid=ds.uid))
else: # assume family_type == 'poi':
actions.append(
poi2mimir.si(instance.name,
filename,
version,
job.id,
dataset_uid=ds.uid))
actions.append(finish_job.si(job.id))
return actions