def delete_files(): """Delete files Returns ------- json files: list of all files of current user error: True if error, else False errorMessage: the error message of error, else an empty string """ data = request.get_json() if not (data and data.get('filesIdToDelete')): return jsonify({ 'files': [], 'error': True, 'errorMessage': "Missing filesIdToDelete parameter" }), 400 try: files = FilesHandler(current_app, session) remaining_files = files.delete_files(data.get('filesIdToDelete', [])) except Exception as e: traceback.print_exc(file=sys.stdout) return jsonify({ 'files': [], 'error': True, 'errorMessage': str(e) }), 500 return jsonify({ 'files': remaining_files, 'error': False, 'errorMessage': '' })
def main(self): """Upload the files""" for file_name in os.listdir(self.args.files_directory): file_path = os.path.join(self.args.files_directory, file_name) file_size = os.path.getsize(file_path) file_path_on_askomics = None file_ext = os.path.splitext(file_name)[1] files = FilesHandler(self.application, self.session) fp = open(file_path) first = True for (last, chunk) in self.generator_is_last( self.read_in_chunks(fp, self.chunk_size)): data = { "first": first, "last": last, "chunk": chunk, "path": file_path_on_askomics, "name": file_name, "size": file_size, "type": files.get_type(file_ext) } first = False file_path_on_askomics = files.persist_chunk(data)
def get_files(): """Get files info of the logged user Returns ------- json files: list of all files of current user error: True if error, else False errorMessage: the error message of error, else an empty string """ files_id = None if request.method == 'POST': data = request.get_json() if data: files_id = data.get('filesId') try: files_handler = FilesHandler(current_app, session) files = files_handler.get_files_infos(files_id=files_id) disk_space = files_handler.get_size_occupied_by_user() except Exception as e: traceback.print_exc(file=sys.stdout) return jsonify({ 'files': [], 'diskSpace': 0, 'error': True, 'errorMessage': str(e) }), 500 return jsonify({ 'files': files, 'diskSpace': disk_space, 'error': False, 'errorMessage': '' })
def get_preview(): """Get files preview Returns ------- json previewFiles: preview of selected files error: True if error, else False errorMessage: the error message of error, else an empty string """ data = request.get_json() try: files_handler = FilesHandler(current_app, session) files_handler.handle_files(data['filesId']) results = [] for file in files_handler.files: file.set_preview() res = file.get_preview() results.append(res) except Exception as e: traceback.print_exc(file=sys.stdout) return jsonify({ 'previewFiles': [], 'error': True, 'errorMessage': str(e) }), 500 return jsonify({ 'previewFiles': results, 'error': False, 'errorMessage': '' })
def upload_url(): """Upload a distant file with an URL Returns ------- json error: True if error, else False errorMessage: the error message of error, else an empty string """ files_utils = FilesUtils(current_app, session) disk_space = files_utils.get_size_occupied_by_user() if "user" in session else None if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]: return jsonify({ 'errorMessage': "Exceeded quota", "error": True }), 500 data = request.get_json() try: files = FilesHandler(current_app, session) files.download_url(data["url"]) except Exception as e: traceback.print_exc(file=sys.stdout) return jsonify({ "error": True, "errorMessage": str(e) }), 500 return jsonify({ "error": False, "errorMessage": "" })
def integrate(): """Integrate a file Returns ------- json task_id: celery task id error: True if error, else False errorMessage: the error message of error, else an empty string """ data = request.get_json() session_dict = {'user': session['user']} task = None try: files_handler = FilesHandler(current_app, session, host_url=request.host_url) files_handler.handle_files([data["fileId"], ]) for file in files_handler.files: data["externalEndpoint"] = data["externalEndpoint"] if data["externalEndpoint"] else None data["customUri"] = data["customUri"] if data["customUri"] else None dataset_info = { "celery_id": None, "file_id": file.id, "name": file.human_name, "graph_name": file.file_graph, "public": data["public"] if session["user"]["admin"] else False } dataset = Dataset(current_app, session, dataset_info) dataset.save_in_db() data["dataset_id"] = dataset.id task = current_app.celery.send_task('integrate', (session_dict, data, request.host_url)) dataset.update_celery(task.id) except Exception as e: traceback.print_exc(file=sys.stdout) return jsonify({ 'error': True, 'errorMessage': str(e), 'task_id': '' }), 500 return jsonify({ 'error': False, 'errorMessage': '', 'task_id': task.id if task else '' })
def download_file(self, session, url): """Send a mail to the new user Parameters ---------- session : dict AskOmics session user : dict New user """ files = FilesHandler(app, session) files.download_url(url, download_file.request.id)
def upload_chunk(): """Upload a file chunk Returns ------- json path: name of the local file. To append the next chunk into it error: True if error, else False errorMessage: the error message of error, else an empty string """ files_utils = FilesUtils(current_app, session) disk_space = files_utils.get_size_occupied_by_user( ) if "user" in session else None if session["user"]["quota"] > 0 and disk_space >= session["user"]["quota"]: return jsonify({ 'errorMessage': "Exceeded quota", "path": '', "error": True }), 400 data = request.get_json() if not (data and all([ key in data for key in ["first", "last", "size", "name", "type", "size", "chunk"] ])): return jsonify({ "path": '', "error": True, "errorMessage": "Missing parameters" }), 400 if not (data["first"] or data.get("path")): return jsonify({ "path": '', "error": True, "errorMessage": "Missing path parameter" }), 400 try: files = FilesHandler(current_app, session) path = files.persist_chunk(data) except Exception as e: traceback.print_exc(file=sys.stdout) return jsonify({ "path": '', "error": True, "errorMessage": str(e) }), 500 return jsonify({"path": path, "error": False, "errorMessage": ""})
def integrate_file(self, info, public=False): """Summary Parameters ---------- info : TYPE Description Returns ------- TYPE Description """ files_handler = FilesHandler(self.app, self.session) files_handler.handle_files([ info["id"], ]) for file in files_handler.files: dataset_info = { "celery_id": "000000000", "file_id": file.id, "name": file.name, "graph_name": file.file_graph, "public": public } dataset = Dataset(self.app, self.session, dataset_info) dataset.save_in_db() if file.type == "csv/tsv": file.integrate(dataset.id, info["columns_type"], public=public) elif file.type == "gff/gff3": file.integrate(dataset.id, info["entities"], public=public) elif file.type == "bed": file.integrate(dataset.id, info["entity_name"], public=public) # done dataset.update_in_db("success") dataset.set_info_from_db() return { "timestamp": file.timestamp, "start": dataset.start, "end": dataset.end }
def edit_file(): """Edit file name Returns ------- json files: list of all files of current user error: True if error, else False errorMessage: the error message of error, else an empty string """ data = request.get_json() current_app.logger.debug(data) if not (data and data.get("id") and data.get("newName")): return jsonify({ 'files': [], 'diskSpace': 0, 'error': True, 'errorMessage': "Missing parameters" }), 400 files_id = [data["id"]] new_name = data["newName"] try: files_handler = FilesHandler(current_app, session) files_handler.handle_files(files_id) for file in files_handler.files: file.edit_name_in_db(new_name) files = files_handler.get_files_infos() except Exception as e: traceback.print_exc(file=sys.stdout) return jsonify({ 'files': [], 'diskSpace': 0, 'error': True, 'errorMessage': str(e) }), 500 return jsonify({ 'files': files, 'diskSpace': files_handler.get_size_occupied_by_user(), 'error': False, 'errorMessage': '' })
def upload_file_url(self, file_url): """Summary Parameters ---------- file_path : TYPE Description Returns ------- TYPE Description """ files = FilesHandler(self.app, self.session) files.download_url(file_url, "1") return files.date
def upload_file(self, file_path): """Summary Parameters ---------- file_path : TYPE Description Returns ------- TYPE Description """ file_name = file_path.split("/")[-1] file_extension = os.path.splitext(file_path)[1] file_type = { ".tsv": "text/tab-separated-values", ".csv": "text/tab-separated-values", ".gff3": "null", ".bed": "null" } with open(file_path, 'r') as file_content: content = file_content.read() file_data = { "first": True, "last": True, "chunk": content, "name": file_name, "type": file_type[file_extension], "size": os.path.getsize(file_path) } files = FilesHandler(self.app, self.session) filepath = files.persist_chunk(file_data) filedate = files.date return { "file_path": filepath, "file_date": filedate }
def download_datasets(self, datasets_id): """Download galaxy datasets into AskOmics Parameters ---------- datasets_id : list List of Galaxy datasets id """ galaxy_instance = galaxy.GalaxyInstance(self.url, self.apikey) files_handler = FilesHandler(self.app, self.session) for dataset_id in datasets_id: dataset = galaxy_instance.datasets.show_dataset(dataset_id) file_name = Utils.get_random_string(10) path = "{}/{}".format(self.upload_path, file_name) filetype = dataset["file_ext"] size = dataset["file_size"] galaxy_instance.datasets.download_dataset( dataset_id, file_path=path, use_default_filename=False) files_handler.store_file_info_in_db(dataset["name"], filetype, file_name, size)
def get_files(): """Get all files info Returns ------- json files: list of all files error: True if error, else False errorMessage: the error message of error, else an empty string """ try: files_handler = FilesHandler(current_app, session) files = files_handler.get_all_files_infos() except Exception as e: traceback.print_exc(file=sys.stdout) return jsonify({ 'files': [], 'error': True, 'errorMessage': str(e) }), 500 return jsonify({'files': files, 'error': False, 'errorMessage': ''})
def integrate(): """Integrate a file Returns ------- json datasets_id: dataset ids error: True if error, else False errorMessage: the error message of error, else an empty string """ data = request.get_json() if not (data and data.get("fileId")): return jsonify({ 'error': True, 'errorMessage': "Missing fileId parameter", 'dataset_ids': None }), 400 session_dict = {'user': session['user']} task = None dataset_ids = [] try: files_handler = FilesHandler(current_app, session, host_url=request.host_url) files_handler.handle_files([ data["fileId"], ]) for file in files_handler.files: data["externalEndpoint"] = data["externalEndpoint"] if ( data.get("externalEndpoint") and isinstance(file, RdfFile)) else None data["customUri"] = data["customUri"] if ( data.get("customUri") and not isinstance(file, RdfFile)) else None dataset_info = { "celery_id": None, "file_id": file.id, "name": file.human_name, "graph_name": file.file_graph, "public": data.get("public") if session["user"]["admin"] else False } dataset = Dataset(current_app, session, dataset_info) dataset.save_in_db() data["dataset_id"] = dataset.id dataset_ids.append(dataset.id) task = current_app.celery.send_task( 'integrate', (session_dict, data, request.host_url)) dataset.update_celery(task.id) except Exception as e: traceback.print_exc(file=sys.stdout) return jsonify({ 'error': True, 'errorMessage': str(e), 'dataset_ids': None }), 500 return jsonify({ 'error': False, 'errorMessage': '', 'dataset_ids': dataset_ids })
def integrate(self, session, data, host_url): """Integrate a file into the triplestore Parameters ---------- session : dict AskOmics session data : dict fileId: file to integrate public: integrate as public or private data host_url : string AskOmics host url Returns ------- dict error: True if error, else False errorMessage: the error message of error, else an empty string """ files_handler = FilesHandler(app, session, host_url=host_url, external_endpoint=data["externalEndpoint"], custom_uri=data["customUri"]) files_handler.handle_files([ data["fileId"], ]) public = data["public"] if session["user"]["admin"] else False for file in files_handler.files: try: dataset_info = { "celery_id": self.request.id, "id": data["dataset_id"], "file_id": file.id, "name": file.human_name, "graph_name": file.file_graph, "public": public } dataset = Dataset(app, session, dataset_info) dataset.update_in_db("started", update_date=True, update_graph=True) if file.type == "csv/tsv": file.integrate(data["dataset_id"], data['columns_type'], data['header_names'], public=public) elif file.type == "gff/gff3": file.integrate(data["dataset_id"], data["entities"], public=public) elif file.type == "turtle": file.integrate(public=public) elif file.type == "bed": file.integrate(data["dataset_id"], data["entity_name"], public=public) # done dataset.update_in_db("success", ntriples=file.ntriples) except Exception as e: traceback.print_exc(file=sys.stdout) trace = traceback.format_exc() dataset.update_in_db("failure", error=True, error_message=str(e), traceback=trace) # Rollback file.rollback() raise e return {'error': True, 'errorMessage': str(e)} return {'error': False, 'errorMessage': ''}