def patch(self, coverage_id):
coverage = models.Coverage.get(coverage_id)
if coverage is None:
raise ObjectNotFound(
"Coverage '{}' not found.".format(coverage_id))
if 'id' in request.json and coverage.id != request.json['id']:
raise InvalidArguments(
'The modification of the id is not possible')
coverage_schema = schema.CoverageSchema(partial=True)
errors = coverage_schema.validate(request.json, partial=True)
if errors:
raise InvalidArguments(errors)
logging.debug(request.json)
try:
coverage = models.Coverage.update(coverage_id, request.json)
except PyMongoError:
raise InternalServerError(
'Impossible to update coverage with dataset {}'.format(
request.json))
return {
'coverages': schema.CoverageSchema().dump([coverage],
many=True).data
}, 200
def post(self, coverage_id):
coverage = models.Coverage.get(coverage_id)
if coverage is None:
raise ObjectNotFound("Coverage {} not found.".format(coverage_id))
if not request.files:
raise InvalidArguments('The archive is missing.')
content = request.files['file']
logger = logging.getLogger(__name__)
logger.info('content received: {}'.format(content))
if not zipfile.is_zipfile(content):
raise InvalidArguments('Invalid ZIP.')
zip_file = zipfile.ZipFile(content)
valid_file, missing_files = is_valid_file(zip_file)
if not valid_file:
raise InvalidArguments('File(s) missing : {}.'.format(''.join(missing_files)))
# check files header
valid_header, invalid_files = check_files_header(zip_file)
if not valid_header:
raise InvalidArguments('Non-compliant file(s) : {}.'.format(''.join(invalid_files)))
content.stream.seek(0)
coverage.save_grid_calendars(content)
zip_file.close()
#run the update of navitia in background
for k, env in coverage.environments.items():
if env.current_ntfs_id:
#TODO: use a chain later
tasks.send_ntfs_to_tyr.delay(coverage.id, k)
return {'message': 'OK'}, 200
def add_coverage_data(coverage_id, coverage, environment_type):
if coverage is None:
raise ObjectNotFound("Coverage {} not found.".format(coverage_id))
if environment_type not in coverage.environments:
raise ObjectNotFound("Environment{}' not found.".format(environment_type))
if not request.files:
raise InvalidArguments('No file provided.')
if request.files and 'file' not in request.files:
raise InvalidArguments('File provided with bad param ("file" param expected).')
content = request.files['file']
logger = logging.getLogger(__name__)
logger.info('content received: %s', content)
with tempfile.TemporaryDirectory() as tmpdirname:
tmp_file = os.path.join(tmpdirname, content.filename)
content.save(tmp_file)
#TODO: improve this function so we don't have to write the file localy first
file_type, file_name = data_handler.type_of_data(tmp_file)
if file_type in [None, "tmp"]:
logger.warning('invalid file provided: %s', content.filename)
raise InvalidArguments('Invalid file provided: {}.'.format(content.filename))
with open(tmp_file, 'rb') as file:
if file_type == 'fusio': #ntfs is called fusio in type_of_data
coverage.save_ntfs(environment_type, file)
tasks.send_ntfs_to_tyr.delay(coverage_id, environment_type)
else:
#we need to temporary save the file before sending it
file_id = models.save_file_in_gridfs(file, filename=content.filename)
tasks.send_file_to_tyr_and_discard.delay(coverage_id, environment_type, file_id)
return {'message': 'Valid {} file provided : {}'.format(file_type, file_name)}, 200
def patch(self, contributor_id, data_source_id=None):
ds = models.DataSource.get(contributor_id, data_source_id)
if len(ds) != 1:
abort(404)
schema_data_source = schema.DataSourceSchema(partial=True)
errors = schema_data_source.validate(request.json, partial=True)
if errors:
raise ObjectNotFound(
"Data source '{}' not found.".format(contributor_id))
if 'id' in request.json and ds[0].id != request.json['id']:
raise InvalidArguments(
'The modification of the id is not possible')
try:
data_sources = models.DataSource.update(contributor_id,
data_source_id,
request.json)
except ValueError as e:
raise InvalidArguments(str(e))
except PyMongoError as e:
raise InternalServerError(
'impossible to update contributor with dataset {}'.format(
request.json))
return {
'data_sources':
schema.DataSourceSchema(many=True).dump(data_sources).data
}, 200
def post(self, coverage_id):
coverage = models.Coverage.get(coverage_id)
if coverage is None:
raise ObjectNotFound("Coverage {} not found.".format(coverage_id))
if 'id' not in request.json:
raise InvalidArguments(
'Missing data_source_id attribute in request body.')
data_source_id = request.json['id']
data_sources = models.DataSource.get(data_source_id=data_source_id)
if data_sources is None:
raise ObjectNotFound(
"Data source {} not found.".format(data_source_id))
if coverage.has_data_source(data_sources[0]):
raise DuplicateEntry(
'Data source id {} already exists in coverage {}.'.format(
data_source_id, coverage_id))
coverage.add_data_source(data_sources[0])
try:
coverage = models.Coverage.update(
coverage_id, {"data_sources": coverage.data_sources})
except (PyMongoError, ValueError) as e:
raise InternalServerError(
'Impossible to update coverage {} with data_source {}.'.format(
coverage_id, data_source_id))
return {
'coverages': schema.CoverageSchema().dump([coverage],
many=True).data
}, 200
def patch(self, contributor_id, preprocess_id):
ds = models.PreProcess.get(contributor_id, preprocess_id)
if len(ds) != 1:
abort(404)
schema_preprocess = schema.PreProcessSchema(partial=True)
errors = schema_preprocess.validate(request.json, partial=True)
if errors:
raise ObjectNotFound(
"Preprocess '{}' not found.".format(contributor_id))
try:
p = request.json
validate_preprocesses_or_raise([p])
preprocesses = models.PreProcess.update(contributor_id,
preprocess_id, p)
except ValueError as e:
raise InvalidArguments(str(e))
except PyMongoError as e:
raise InternalServerError(
'impossible to update contributor with preprocess {}'.format(
p))
return {
'preprocesses':
schema.PreProcessSchema(many=True).dump(preprocesses).data
}, 200
def validate_preprocesses_or_raise(preprocesses):
for p in preprocesses:
p_type = p.get('type')
kls = getattr(tartare.processes, p_type, None)
if kls is None:
msg = 'Invalid process type {}'.format(p_type)
logging.getLogger(__name__).error(msg)
raise InvalidArguments(msg)
def delete(self, contributor_id, data_source_id=None):
try:
nb_deleted = models.DataSource.delete(contributor_id,
data_source_id)
if nb_deleted == 0:
raise ObjectNotFound(
"Data source '{}' not found.".format(contributor_id))
except ValueError as e:
raise InvalidArguments(str(e))
return {'data_sources': []}, 204
def delete(self, contributor_id, preprocess_id):
try:
nb_deleted = models.PreProcess.delete(contributor_id,
preprocess_id)
if nb_deleted == 0:
raise ObjectNotFound(
"Preprocess '{}' not found.".format(contributor_id))
except ValueError as e:
raise InvalidArguments(str(e))
return {'preprocesses': []}, 204
def get(self, contributor_id, data_source_id=None):
try:
ds = models.DataSource.get(contributor_id, data_source_id)
if ds is None:
raise ObjectNotFound(
"Data source '{}' not found.".format(data_source_id))
except ValueError as e:
raise InvalidArguments(str(e))
return {
'data_sources': schema.DataSourceSchema(many=True).dump(ds).data
}, 200
def get(self, contributor_id, preprocess_id=None):
try:
ps = models.PreProcess.get(contributor_id, preprocess_id)
if not ps and preprocess_id:
raise ObjectNotFound(
"Preprocess '{}' not found.".format(preprocess_id))
except ValueError as e:
raise InvalidArguments(str(e))
return {
'preprocesses': schema.PreProcessSchema(many=True).dump(ps).data
}, 200
def get(self, coverage_id, environment_type, data_type):
available_data_types = ['ntfs']
if data_type.lower() not in available_data_types:
raise InvalidArguments('Bad data type {} (expected formats: {}).'
.format(data_type, ','.join(available_data_types)))
coverage = models.Coverage.get(coverage_id)
if coverage is None:
raise ObjectNotFound("Coverage {} not found.".format(coverage_id))
if environment_type not in coverage.environments:
raise ObjectNotFound("Environment{}' not found.".format(environment_type))
ntfs_file_id = coverage.environments[environment_type].current_ntfs_id
ntfs_file = models.get_file_from_gridfs(ntfs_file_id)
return flask.send_file(ntfs_file, mimetype='application/zip')
def post(self, data_source_id):
datasource = models.DataSource.get(data_source_id=data_source_id)
if datasource is None:
raise ObjectNotFound(
"Data source '{}' not found.".format(data_source_id))
if not request.files:
raise InvalidArguments('No file provided.')
if request.files and 'file' not in request.files:
raise InvalidArguments(
'File provided with bad param ("file" param expected).')
content = request.files['file']
logger = logging.getLogger(__name__)
logger.info('content received: %s', content)
with tempfile.TemporaryDirectory() as tmpdirname:
tmp_file = os.path.join(tmpdirname, content.filename)
content.save(tmp_file)
file_type, file_name = data_handler.type_of_data(tmp_file)
if file_type in [None, "tmp"]:
logger.warning('invalid file provided: %s', content.filename)
raise InvalidArguments('Invalid file provided: {}.'.format(
content.filename))
# backup content
input_dir = coverage.technical_conf.input_dir
if not os.path.exists(input_dir):
os.makedirs(input_dir)
full_file_name = os.path.join(os.path.realpath(input_dir),
content.filename)
shutil.move(tmp_file, full_file_name + ".tmp")
shutil.move(full_file_name + ".tmp", full_file_name)
return {
'message':
'Valid {} file provided : {}'.format(file_type, file_name)
}, 200
def post(self, contributor_id):
data_source_schema = schema.DataSourceSchema(strict=True)
try:
d = request.json
data_source = data_source_schema.load(d).data
except ValidationError as err:
raise InvalidArguments(err.messages)
try:
data_source.save(contributor_id)
return {
'data_sources':
schema.DataSourceSchema(many=True).dump([data_source]).data
}, 201
except PyMongoError:
raise InternalServerError('Impossible to add data source.')
except ValueError as e:
raise DuplicateEntry(str(e))
def post(self, contributor_id):
preprocess_schema = schema.PreProcessSchema(strict=True)
try:
d = request.json
validate_preprocesses_or_raise([d])
preprocess = preprocess_schema.load(d).data
except ValidationError as err:
raise InvalidArguments(err.messages)
try:
preprocess.save(contributor_id)
except (PyMongoError, ValueError) as e:
raise InternalServerError('Impossible to add data source.')
return {
'preprocesses':
schema.PreProcessSchema(many=True).dump([preprocess]).data
}, 201
def post(self):
coverage_schema = schema.CoverageSchema(strict=True)
try:
coverage = coverage_schema.load(request.json).data
except ValidationError as err:
raise InvalidArguments(err.messages)
try:
coverage.save()
except DuplicateKeyError:
raise DuplicateEntry("Coverage {} already exists.".format(
request.json['id']))
except PyMongoError as e:
raise InternalServerError('Impossible to add coverage.')
return {
'coverages': coverage_schema.dump([coverage], many=True).data
}, 201
def post(self):
post_data = request.json
#first a check on the data_sources id and providing a uuid if not provided
for ds in post_data.get('data_sources', []):
# set id if not existent
ds.setdefault('id', str(uuid.uuid4()))
preprocesses = post_data.get('preprocesses', [])
validate_preprocesses_or_raise(preprocesses)
for ps in preprocesses:
# set id if not existent
ps.setdefault('id', str(uuid.uuid4()))
contributor_schema = schema.ContributorSchema(strict=True)
try:
contributor = contributor_schema.load(post_data).data
except ValidationError as err:
raise InvalidArguments(err.messages)
contributor_id = post_data["id"]
try:
contributor.save()
except DuplicateKeyError as e:
raise DuplicateEntry(
"Impossible to add contributor, id {} or data_prefix {} already used."
.format(request.json['id'], request.json['data_prefix']))
except PyMongoError:
raise InternalServerError(
'Impossible to add contributor {}'.format(contributor))
return {
'contributors': [
contributor_schema.dump(
models.Contributor.get(contributor_id)).data
]
}, 201
def patch(self, contributor_id):
# "data_prefix" field is not modifiable, impacts of the modification
# need to be checked. The previous value needs to be checked for an error
contributor = models.Contributor.get(contributor_id)
if contributor is None:
raise ObjectNotFound(
"Contributor '{}' not found.".format(contributor_id))
request_data = request.json
#checking errors before updating PATCH data
for ds in request_data.get('data_sources', []):
if not ds.get('id', None):
ds['id'] = str(uuid.uuid4())
schema_contributor = schema.ContributorSchema(partial=True)
errors = schema_contributor.validate(request_data, partial=True)
if errors:
raise InvalidArguments(errors)
if 'data_prefix' in request_data and contributor.data_prefix != request_data[
'data_prefix']:
raise InvalidArguments(
'The modification of the data_prefix is not possible ({} => {})'
.format(contributor.data_prefix, request_data['data_prefix']))
if 'id' in request_data and contributor.id != request_data['id']:
raise InvalidArguments(
'The modification of the id is not possible')
existing_ds_id = [d.id for d in contributor.data_sources]
logging.getLogger(__name__).debug(
"PATCH : list of existing data_sources ids %s",
str(existing_ds_id))
#constructing PATCH data
patched_data_sources = None
if "data_sources" in request_data:
patched_data_sources = schema.DataSourceSchema(many=True).dump(
contributor.data_sources).data
for ds in request_data["data_sources"]:
if ds['id'] in existing_ds_id:
pds = next(
(p
for p in patched_data_sources if p['id'] == ds['id']),
None)
if pds:
pds.update(ds)
else:
#adding a new data_source
patched_data_sources.append(ds)
if patched_data_sources:
request_data['data_sources'] = patched_data_sources
try:
contributor = models.Contributor.update(contributor_id,
request_data)
except PyMongoError:
raise InternalServerError(
'impossible to update contributor with dataset {}'.format(
request_data))
return {
'contributors':
[schema.ContributorSchema().dump(contributor).data]
}, 200