def upload_yaml(): """ This function process the yaml :return: """ user_id = session['uid'] project_id = request.form['pid'] yaml_data = request.form["yaml"] project_meta = dict() project_config_path = get_project_config_path(user_id, project_id) project = Project(project_config_path) data_file_name, sheet_name = project.get_current_file_and_sheet() yaml_configuration = YAMLFile() data_file_path = str(Path.cwd() / "config" / "uploads" / user_id / project_id / "df" / data_file_name) response = {'error': None} if check_if_string_is_invalid(yaml_data): response['error'] = "YAML file is either empty or not valid" else: yaml_file_id = project.get_yaml_file_id(data_file_name, sheet_name) if not yaml_file_id: yaml_file_id = generate_id() yaml_file_name = yaml_file_id + ".yaml" yaml_config_file_name = yaml_file_id + ".pickle" yaml_config_file_path = str(Path.cwd() / "config" / "uploads" / user_id / project_id / "yf" / yaml_config_file_name) yaml_file_path = str(Path.cwd() / "config" / "uploads" / user_id / project_id / "yf" / yaml_file_name) with open(yaml_file_path, "w", newline='') as f: f.write(yaml_data) yaml_configuration.set_file_location(yaml_file_path) project.add_yaml_file(data_file_name, sheet_name, yaml_file_id) if data_file_name: wikifier_config_file_name = project.get_or_create_wikifier_region_filename( data_file_name, sheet_name) wikifier_config = deserialize_wikifier_config( user_id, project_id, wikifier_config_file_name) item_table = ItemTable(wikifier_config) region, template, created_by = load_yaml_data( yaml_file_path, item_table, data_file_path, sheet_name) yaml_configuration.set_region(region) yaml_configuration.set_template(template) yaml_configuration.set_created_by(created_by) save_yaml_config(yaml_config_file_path, yaml_configuration) template = yaml_configuration.get_template() response['yamlRegions'] = highlight_region(item_table, data_file_path, sheet_name, region, template) project_meta["yamlMapping"] = dict() project_meta["yamlMapping"][data_file_name] = dict() project_meta["yamlMapping"][data_file_name][ sheet_name] = yaml_file_id project.update_project_config(project_meta) else: response['yamlRegions'] = None response['error'] = "Upload data file before applying YAML." return json.dumps(response, indent=3)
def upload_yaml(): """ This function process the yaml :return: """ user_id = request.form["id"] yaml_data = request.values["yaml"] os.makedirs("uploads", exist_ok=True) user = app.config['users'].get_user(user_id) user.reset('yaml') yaml_configuration = user.get_yaml_data() excel_data_filepath = user.get_excel_data().get_file_location() response = dict() if check_if_empty(yaml_data): response['error'] = "YAML file is either empty or not valid" else: sheet_name = user.get_excel_data().get_sheet_name() filename = str(Path(app.config['UPLOAD_FOLDER']) / user_id) + ".yaml" with open(filename, "w") as f: f.write(yaml_data) yaml_configuration.set_file_location(filename) region, template = load_yaml_data(filename) yaml_configuration.set_region(region) yaml_configuration.set_template(template) item_table = user.get_wikifier_output_data().get_item_table() template = yaml_configuration.get_template() response['region'] = highlight_region(item_table, excel_data_filepath, sheet_name, region, template) return json.dumps(response, indent=3)
def run_t2wml( data_file_path: str, wikified_output_path: str, t2wml_spec: str, output_directory: str, sheet_name: str = None, sparql_endpoint: str = "http://dsbox02.isi.edu:8888/bigdata/namespace/wdq/sparql"): try: yaml_configuration = YAMLFile() yaml_configuration.set_file_location(t2wml_spec) region, template = load_yaml_data(t2wml_spec) yaml_configuration.set_region(region) yaml_configuration.set_template(template) except: logging.error("Invalid YAML File") return try: item_table = ItemTable() build_item_table(item_table, wikified_output_path, data_file_path, sheet_name) except: logging.error("Invalid Wikfied Output File") return filetype = "ttl" response = generate_download_file(None, item_table, data_file_path, sheet_name, region, template, filetype, sparql_endpoint) file_name = Path(data_file_path).name result_directory = '.'.join(file_name.split(".")[:-1]) try: file_extension = file_name.split(".")[-1] except: logging.error("Data file has no extension") return output_path = Path() if file_extension == "csv": output_path = Path(output_directory) / result_directory elif file_extension == "xls" or file_extension == "xlsx": if not sheet_name: sheet_name = get_first_sheet_name(data_file_path) output_path = Path(output_directory) / result_directory / sheet_name Path.mkdir(output_path, parents=True, exist_ok=True) with open(str(output_path / "results.ttl"), "w") as fp: fp.write(response["data"]) with open(str(output_path / "changes.tsv"), "w") as fp: serialize_change_record(fp)