def api_get_project_data(project_id): # noqa: F401 """Get info on the article""" if not is_project(project_id): response = jsonify(message="Project not found.") return response, 404 try: filename = get_data_file_path(project_id).stem # get statistics of the dataset statistics = get_data_statistics(project_id) statistics["filename"] = filename except FileNotFoundError as err: print(err) statistics = {"filename": None} except Exception as err: print(err) message = f"Failed to get file. {err}" return jsonify(message=message), 400 response = jsonify(statistics) response.headers.add('Access-Control-Allow-Origin', '*') return response
def api_upload_data_project(project_id): # noqa: F401 """Get info on the article""" if not is_project(project_id): response = jsonify(message="project-not-found") return response, 404 if request.form.get('demo_data', None): # download file and save to folder demo_data = get_dataset(request.form['demo_data']) if demo_data.dataset_id in ["hall", "ace", "ptsd"]: download_url = demo_data.url_demo else: download_url = demo_data.url url_parts = urllib.parse.urlparse(download_url) filename = url_parts.path.rsplit('/', 1)[-1] urlretrieve(download_url, get_data_path(project_id) / filename) add_dataset_to_project(project_id, filename) elif 'file' in request.files: data_file = request.files['file'] # check the file is file is in a correct format check_dataset(data_file) # TODO{qubixes}: implement val strategy try: filename = secure_filename(data_file.filename) fp_data = get_data_path(project_id) / filename # save the file data_file.save(str(fp_data)) add_dataset_to_project(project_id, filename) except Exception as err: logging.error(err) response = jsonify(message="project-upload-failure") return response, 500 else: response = jsonify(message="no-file-found") return response, 500 return jsonify({"success": True})
def api_import_project(): """Import uploaded project""" import_project_id = None if 'file' in request.files: project_file = request.files['file'] filename = secure_filename(project_file.filename) try: with zipfile.ZipFile(project_file, "r") as zipObj: FileNames = zipObj.namelist() # check if the zip file contains a ASReview project if sum([fn.endswith("project.json") for fn in FileNames]) == 1: # extract all files to a temporary folder tmpdir = tempfile.TemporaryDirectory() zipObj.extractall(path=tmpdir.name) for fn in FileNames: if fn.endswith("project.json"): fp = Path(tmpdir.name, fn) with open(fp, "r+") as f: project = json.load(f) # if the uploaded project already exists, # then make a copy if is_project(project["id"]): project["id"] += " copy" project["name"] += " copy" f.seek(0) json.dump(project, f) f.truncate() else: response = jsonify( message="No project found within the chosen file.") return response, 404 try: # check if a copy of a project already exists os.rename(tmpdir.name, asreview_path() / f"{project['id']}") import_project_id = project['id'] except Exception as err: logging.error(err) response = jsonify( message=f"A copy of {project['id'][:-5]} already exists.") return response, 400 except Exception as err: logging.error(err) response = jsonify(message=f"Failed to upload file '{filename}'.") return response, 400 else: response = jsonify(message="No file found to upload.") return response, 400 # return the project info in the same format as project_info return api_get_project_info(import_project_id)
def api_upload_data_to_project(project_id): # noqa: F401 """Get info on the article""" if not is_project(project_id): response = jsonify(message="Project not found.") return response, 404 if request.form.get('demo_data', None): # download file and save to folder demo_data = DatasetManager().find(request.form['demo_data']) if demo_data.dataset_id in ["hall", "ace", "ptsd"]: download_url = demo_data.url_demo else: download_url = demo_data.url url_parts = urllib.parse.urlparse(download_url) filename = secure_filename(url_parts.path.rsplit('/', 1)[-1]) urlretrieve(download_url, get_data_path(project_id) / filename) elif request.form.get('url', None): # download file and save to folder download_url = request.form['url'] try: url_parts = urllib.parse.urlparse(download_url) filename = secure_filename(url_parts.path.rsplit('/', 1)[-1]) urlretrieve(download_url, get_data_path(project_id) / filename) except ValueError as err: logging.error(err) message = f"Invalid URL '{download_url}'." if isinstance(download_url, str) \ and not download_url.startswith("http"): message += " Usually, the URL starts with 'http' or 'https'." return jsonify(message=message), 400 except Exception as err: logging.error(err) message = f"Can't retrieve data from URL {download_url}." return jsonify(message=message), 400 elif 'file' in request.files: data_file = request.files['file'] # check the file is file is in a correct format check_dataset(data_file) # TODO{qubixes}: implement val strategy try: filename = secure_filename(data_file.filename) fp_data = get_data_path(project_id) / filename # save the file data_file.save(str(fp_data)) except Exception as err: logging.error(err) response = jsonify( message=f"Failed to upload file '{filename}'. {err}") return response, 400 else: response = jsonify(message="No file or dataset found to upload.") return response, 400 try: # add the file to the project add_dataset_to_project(project_id, filename) # Bad format. TODO{Jonathan} Return informative message with link. except BadFileFormatError as err: message = f"Failed to upload file '{filename}'. {err}" return jsonify(message=message), 400 response = jsonify({'success': True}) response.headers.add('Access-Control-Allow-Origin', '*') return response
def api_import_project(): """Import uploaded project""" # raise error if file not given if 'file' not in request.files: response = jsonify(message="No file found to upload.") return response, 400 # import project id import_project = None # set the project file project_file = request.files['file'] try: with zipfile.ZipFile(project_file, "r") as zip_obj: zip_filenames = zip_obj.namelist() # raise error if no ASReview project file if "project.json" not in zip_filenames: response = jsonify( message="File doesn't contain valid project format.") return response, 404 # extract all files to a temporary folder tmpdir = tempfile.TemporaryDirectory() zip_obj.extractall(path=tmpdir.name) # Open the project file and check the id. The id needs to be # unique, otherwise it is exended with -copy. fp = Path(tmpdir.name, "project.json") with open(fp, "r+") as f: # load the project info in scope of function import_project = json.load(f) # If the uploaded project already exists, # then overwrite project.json with a copy suffix. while is_project(import_project["id"]): # project update import_project["id"] = f"{import_project['id']}-copy" import_project["name"] = f"{import_project['name']} copy" else: # write to file f.seek(0) json.dump(import_project, f) f.truncate() except Exception as err: # Unknown error. logging.error(err) response = jsonify(message="Unknown error when uploading project " f"'{project_file.filename}'.") return response, 400 # location to copy file to fp_copy = get_project_path(import_project["id"]) try: # Move the project from the temp folder to the projects folder. os.rename(tmpdir.name, fp_copy) except Exception as err: logging.error(err) response = jsonify(message=f"Failed to copy project to {fp_copy}.") return response, 400 # return the project info in the same format as project_info return api_get_project_info(import_project["id"])
def api_upload_data_to_project(project_id): # noqa: F401 """Get info on the article""" if not is_project(project_id): response = jsonify(message="Project not found.") return response, 404 if request.form.get('plugin', None): plugin_data = DatasetManager().find(request.form['plugin']) url_parts = urllib.parse.urlparse(plugin_data.url) filename = secure_filename(url_parts.path.rsplit('/', 1)[-1]) urlretrieve(plugin_data.url, get_data_path(project_id) / filename) elif request.form.get('benchmark', None): benchmark_dataset_id = DatasetManager().find(request.form['benchmark']) # read dataset df = pd.read_csv(benchmark_dataset_id.url) # rename label column df.rename({"label_included": "debug_label"}, axis=1, inplace=True) # define export filepath url_parts = urllib.parse.urlparse(benchmark_dataset_id.url) filename = secure_filename(url_parts.path.rsplit('/', 1)[-1]) export_fp = get_data_path(project_id) / filename # export file df.to_csv(export_fp, index=False) elif request.form.get('url', None): # download file and save to folder download_url = request.form['url'] try: url_parts = urllib.parse.urlparse(download_url) filename = secure_filename(url_parts.path.rsplit('/', 1)[-1]) urlretrieve(download_url, get_data_path(project_id) / filename) except ValueError as err: logging.error(err) message = f"Invalid URL '{download_url}'." if isinstance(download_url, str) \ and not download_url.startswith("http"): message += " Usually, the URL starts with 'http' or 'https'." return jsonify(message=message), 400 except Exception as err: logging.error(err) message = f"Can't retrieve data from URL {download_url}." return jsonify(message=message), 400 elif 'file' in request.files: data_file = request.files['file'] # check the file is file is in a correct format check_dataset(data_file) # TODO{qubixes}: implement val strategy try: filename = secure_filename(data_file.filename) fp_data = get_data_path(project_id) / filename # save the file data_file.save(str(fp_data)) except Exception as err: logging.error(err) response = jsonify( message=f"Failed to upload file '{filename}'. {err}") return response, 400 else: response = jsonify(message="No file or dataset found to upload.") return response, 400 try: # add the file to the project add_dataset_to_project(project_id, filename) # Bad format. TODO{Jonathan} Return informative message with link. except BadFileFormatError as err: message = f"Failed to upload file '{filename}'. {err}" return jsonify(message=message), 400 response = jsonify({'success': True}) response.headers.add('Access-Control-Allow-Origin', '*') return response
def api_import_project(): """Import uploaded project""" if 'file' in request.files: project_file = request.files['file'] filename = secure_filename(project_file.filename) # check the file format if not os.path.splitext(filename)[1] == ".asreview": response = jsonify(message="Incorrect file format.") return response, 400 try: with zipfile.ZipFile(project_file, "r") as zipObj: FileNames = zipObj.namelist() # check if the zip file contains a ASReview project if sum([fn.endswith("project.json") for fn in FileNames]) == 1: # extract all files to a temporary folder tmpdir = tempfile.TemporaryDirectory() zipObj.extractall(path=tmpdir.name) for fn in FileNames: if fn.endswith("project.json"): fp = Path(tmpdir.name, fn) with open(fp, "r+") as f: project = json.load(f) # if the uploaded project already exists, then make a copy if is_project(project["id"]): project["id"] += " copy" project["name"] += " copy" f.seek(0) json.dump(project, f) f.truncate() else: response = jsonify(message="No project found within the chosen file.") return response, 404 try: # check if a copy of a project already exists os.rename(tmpdir.name, asreview_path() / f"{project['id']}") except Exception as err: logging.error(err) response = jsonify(message=f"A copy of {project['id'][:-5]} already exists.") return response, 400 except Exception as err: logging.error(err) response = jsonify(message=f"Failed to upload file '{filename}'. {err}") return response, 400 else: response = jsonify(message="No file found to upload.") return response, 400 response = jsonify({'success': True}) response.headers.add('Access-Control-Allow-Origin', '*') return response