def delete_files():
"""Delete files
Returns
-------
json
files: list of all files of current user
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
data = request.get_json()
if not (data and data.get('filesIdToDelete')):
return jsonify({
'files': [],
'error': True,
'errorMessage': "Missing filesIdToDelete parameter"
}), 400
try:
files = FilesHandler(current_app, session)
remaining_files = files.delete_files(data.get('filesIdToDelete', []))
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'files': [],
'error': True,
'errorMessage': str(e)
}), 500
return jsonify({
'files': remaining_files,
'error': False,
'errorMessage': ''
})
def main(self):
"""Upload the files"""
for file_name in os.listdir(self.args.files_directory):
file_path = os.path.join(self.args.files_directory, file_name)
file_size = os.path.getsize(file_path)
file_path_on_askomics = None
file_ext = os.path.splitext(file_name)[1]
files = FilesHandler(self.application, self.session)
fp = open(file_path)
first = True
for (last, chunk) in self.generator_is_last(
self.read_in_chunks(fp, self.chunk_size)):
data = {
"first": first,
"last": last,
"chunk": chunk,
"path": file_path_on_askomics,
"name": file_name,
"size": file_size,
"type": files.get_type(file_ext)
}
first = False
file_path_on_askomics = files.persist_chunk(data)
def get_files():
"""Get files info of the logged user
Returns
-------
json
files: list of all files of current user
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
files_id = None
if request.method == 'POST':
data = request.get_json()
if data:
files_id = data.get('filesId')
try:
files_handler = FilesHandler(current_app, session)
files = files_handler.get_files_infos(files_id=files_id)
disk_space = files_handler.get_size_occupied_by_user()
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'files': [],
'diskSpace': 0,
'error': True,
'errorMessage': str(e)
}), 500
return jsonify({
'files': files,
'diskSpace': disk_space,
'error': False,
'errorMessage': ''
})
def get_preview():
"""Get files preview
Returns
-------
json
previewFiles: preview of selected files
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
data = request.get_json()
try:
files_handler = FilesHandler(current_app, session)
files_handler.handle_files(data['filesId'])
results = []
for file in files_handler.files:
file.set_preview()
res = file.get_preview()
results.append(res)
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'previewFiles': [],
'error': True,
'errorMessage': str(e)
}), 500
return jsonify({
'previewFiles': results,
'error': False,
'errorMessage': ''
})
def upload_url():
"""Upload a distant file with an URL
Returns
-------
json
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
files_utils = FilesUtils(current_app, session)
disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None
if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:
return jsonify({
'errorMessage': "Exceeded quota",
"error": True
}), 500
data = request.get_json()
try:
files = FilesHandler(current_app, session)
files.download_url(data["url"])
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
"error": True,
"errorMessage": str(e)
}), 500
return jsonify({
"error": False,
"errorMessage": ""
})
def integrate():
"""Integrate a file
Returns
-------
json
task_id: celery task id
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
data = request.get_json()
session_dict = {'user': session['user']}
task = None
try:
files_handler = FilesHandler(current_app, session, host_url=request.host_url)
files_handler.handle_files([data["fileId"], ])
for file in files_handler.files:
data["externalEndpoint"] = data["externalEndpoint"] if data["externalEndpoint"] else None
data["customUri"] = data["customUri"] if data["customUri"] else None
dataset_info = {
"celery_id": None,
"file_id": file.id,
"name": file.human_name,
"graph_name": file.file_graph,
"public": data["public"] if session["user"]["admin"] else False
}
dataset = Dataset(current_app, session, dataset_info)
dataset.save_in_db()
data["dataset_id"] = dataset.id
task = current_app.celery.send_task('integrate', (session_dict, data, request.host_url))
dataset.update_celery(task.id)
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'error': True,
'errorMessage': str(e),
'task_id': ''
}), 500
return jsonify({
'error': False,
'errorMessage': '',
'task_id': task.id if task else ''
})
def download_file(self, session, url):
"""Send a mail to the new user
Parameters
----------
session : dict
AskOmics session
user : dict
New user
"""
files = FilesHandler(app, session)
files.download_url(url, download_file.request.id)
def upload_chunk():
"""Upload a file chunk
Returns
-------
json
path: name of the local file. To append the next chunk into it
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
files_utils = FilesUtils(current_app, session)
disk_space = files_utils.get_size_occupied_by_user(
) if "user" in session else None
if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]:
return jsonify({
'errorMessage': "Exceeded quota",
"path": '',
"error": True
}), 400
data = request.get_json()
if not (data and all([
key in data for key in
["first", "last", "size", "name", "type", "size", "chunk"]
])):
return jsonify({
"path": '',
"error": True,
"errorMessage": "Missing parameters"
}), 400
if not (data["first"] or data.get("path")):
return jsonify({
"path": '',
"error": True,
"errorMessage": "Missing path parameter"
}), 400
try:
files = FilesHandler(current_app, session)
path = files.persist_chunk(data)
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
"path": '',
"error": True,
"errorMessage": str(e)
}), 500
return jsonify({"path": path, "error": False, "errorMessage": ""})
def integrate_file(self, info, public=False):
"""Summary
Parameters
----------
info : TYPE
Description
Returns
-------
TYPE
Description
"""
files_handler = FilesHandler(self.app, self.session)
files_handler.handle_files([
info["id"],
])
for file in files_handler.files:
dataset_info = {
"celery_id": "000000000",
"file_id": file.id,
"name": file.name,
"graph_name": file.file_graph,
"public": public
}
dataset = Dataset(self.app, self.session, dataset_info)
dataset.save_in_db()
if file.type == "csv/tsv":
file.integrate(dataset.id, info["columns_type"], public=public)
elif file.type == "gff/gff3":
file.integrate(dataset.id, info["entities"], public=public)
elif file.type == "bed":
file.integrate(dataset.id, info["entity_name"], public=public)
# done
dataset.update_in_db("success")
dataset.set_info_from_db()
return {
"timestamp": file.timestamp,
"start": dataset.start,
"end": dataset.end
}
def upload_file_url(self, file_url):
"""Summary
Parameters
----------
file_path : TYPE
Description
Returns
-------
TYPE
Description
"""
files = FilesHandler(self.app, self.session)
files.download_url(file_url, "1")
return files.date
def upload_file(self, file_path):
"""Summary
Parameters
----------
file_path : TYPE
Description
Returns
-------
TYPE
Description
"""
file_name = file_path.split("/")[-1]
file_extension = os.path.splitext(file_path)[1]
file_type = {
".tsv": "text/tab-separated-values",
".csv": "text/tab-separated-values",
".gff3": "null",
".bed": "null"
}
with open(file_path, 'r') as file_content:
content = file_content.read()
file_data = {
"first": True,
"last": True,
"chunk": content,
"name": file_name,
"type": file_type[file_extension],
"size": os.path.getsize(file_path)
}
files = FilesHandler(self.app, self.session)
filepath = files.persist_chunk(file_data)
filedate = files.date
return {
"file_path": filepath,
"file_date": filedate
}
def download_datasets(self, datasets_id):
"""Download galaxy datasets into AskOmics
Parameters
----------
datasets_id : list
List of Galaxy datasets id
"""
galaxy_instance = galaxy.GalaxyInstance(self.url, self.apikey)
files_handler = FilesHandler(self.app, self.session)
for dataset_id in datasets_id:
dataset = galaxy_instance.datasets.show_dataset(dataset_id)
file_name = Utils.get_random_string(10)
path = "{}/{}".format(self.upload_path, file_name)
filetype = dataset["file_ext"]
size = dataset["file_size"]
galaxy_instance.datasets.download_dataset(
dataset_id, file_path=path, use_default_filename=False)
files_handler.store_file_info_in_db(dataset["name"], filetype,
file_name, size)
def get_files():
"""Get all files info
Returns
-------
json
files: list of all files
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
try:
files_handler = FilesHandler(current_app, session)
files = files_handler.get_all_files_infos()
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'files': [],
'error': True,
'errorMessage': str(e)
}), 500
return jsonify({'files': files, 'error': False, 'errorMessage': ''})
def edit_file():
"""Edit file name
Returns
-------
json
files: list of all files of current user
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
data = request.get_json()
current_app.logger.debug(data)
if not (data and data.get("id") and data.get("newName")):
return jsonify({
'files': [],
'diskSpace': 0,
'error': True,
'errorMessage': "Missing parameters"
}), 400
files_id = [data["id"]]
new_name = data["newName"]
try:
files_handler = FilesHandler(current_app, session)
files_handler.handle_files(files_id)
for file in files_handler.files:
file.edit_name_in_db(new_name)
files = files_handler.get_files_infos()
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'files': [],
'diskSpace': 0,
'error': True,
'errorMessage': str(e)
}), 500
return jsonify({
'files': files,
'diskSpace': files_handler.get_size_occupied_by_user(),
'error': False,
'errorMessage': ''
})
def integrate():
"""Integrate a file
Returns
-------
json
datasets_id: dataset ids
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
data = request.get_json()
if not (data and data.get("fileId")):
return jsonify({
'error': True,
'errorMessage': "Missing fileId parameter",
'dataset_ids': None
}), 400
session_dict = {'user': session['user']}
task = None
dataset_ids = []
try:
files_handler = FilesHandler(current_app,
session,
host_url=request.host_url)
files_handler.handle_files([
data["fileId"],
])
for file in files_handler.files:
data["externalEndpoint"] = data["externalEndpoint"] if (
data.get("externalEndpoint")
and isinstance(file, RdfFile)) else None
data["customUri"] = data["customUri"] if (
data.get("customUri")
and not isinstance(file, RdfFile)) else None
dataset_info = {
"celery_id": None,
"file_id": file.id,
"name": file.human_name,
"graph_name": file.file_graph,
"public":
data.get("public") if session["user"]["admin"] else False
}
dataset = Dataset(current_app, session, dataset_info)
dataset.save_in_db()
data["dataset_id"] = dataset.id
dataset_ids.append(dataset.id)
task = current_app.celery.send_task(
'integrate', (session_dict, data, request.host_url))
dataset.update_celery(task.id)
except Exception as e:
traceback.print_exc(file=sys.stdout)
return jsonify({
'error': True,
'errorMessage': str(e),
'dataset_ids': None
}), 500
return jsonify({
'error': False,
'errorMessage': '',
'dataset_ids': dataset_ids
})
def integrate(self, session, data, host_url):
"""Integrate a file into the triplestore
Parameters
----------
session : dict
AskOmics session
data : dict
fileId: file to integrate
public: integrate as public or private data
host_url : string
AskOmics host url
Returns
-------
dict
error: True if error, else False
errorMessage: the error message of error, else an empty string
"""
files_handler = FilesHandler(app,
session,
host_url=host_url,
external_endpoint=data["externalEndpoint"],
custom_uri=data["customUri"])
files_handler.handle_files([
data["fileId"],
])
public = data["public"] if session["user"]["admin"] else False
for file in files_handler.files:
try:
dataset_info = {
"celery_id": self.request.id,
"id": data["dataset_id"],
"file_id": file.id,
"name": file.human_name,
"graph_name": file.file_graph,
"public": public
}
dataset = Dataset(app, session, dataset_info)
dataset.update_in_db("started",
update_date=True,
update_graph=True)
if file.type == "csv/tsv":
file.integrate(data["dataset_id"],
data['columns_type'],
data['header_names'],
public=public)
elif file.type == "gff/gff3":
file.integrate(data["dataset_id"],
data["entities"],
public=public)
elif file.type == "turtle":
file.integrate(public=public)
elif file.type == "bed":
file.integrate(data["dataset_id"],
data["entity_name"],
public=public)
# done
dataset.update_in_db("success", ntriples=file.ntriples)
except Exception as e:
traceback.print_exc(file=sys.stdout)
trace = traceback.format_exc()
dataset.update_in_db("failure",
error=True,
error_message=str(e),
traceback=trace)
# Rollback
file.rollback()
raise e
return {'error': True, 'errorMessage': str(e)}
return {'error': False, 'errorMessage': ''}
