def upload_yaml():
"""
This function process the yaml
:return:
"""
user_id = session['uid']
project_id = request.form['pid']
yaml_data = request.form["yaml"]
project_meta = dict()
project_config_path = get_project_config_path(user_id, project_id)
project = Project(project_config_path)
data_file_name, sheet_name = project.get_current_file_and_sheet()
yaml_configuration = YAMLFile()
data_file_path = str(Path.cwd() / "config" / "uploads" / user_id /
project_id / "df" / data_file_name)
response = {'error': None}
if check_if_string_is_invalid(yaml_data):
response['error'] = "YAML file is either empty or not valid"
else:
yaml_file_id = project.get_yaml_file_id(data_file_name, sheet_name)
if not yaml_file_id:
yaml_file_id = generate_id()
yaml_file_name = yaml_file_id + ".yaml"
yaml_config_file_name = yaml_file_id + ".pickle"
yaml_config_file_path = str(Path.cwd() / "config" / "uploads" /
user_id / project_id / "yf" /
yaml_config_file_name)
yaml_file_path = str(Path.cwd() / "config" / "uploads" / user_id /
project_id / "yf" / yaml_file_name)
with open(yaml_file_path, "w", newline='') as f:
f.write(yaml_data)
yaml_configuration.set_file_location(yaml_file_path)
project.add_yaml_file(data_file_name, sheet_name, yaml_file_id)
if data_file_name:
wikifier_config_file_name = project.get_or_create_wikifier_region_filename(
data_file_name, sheet_name)
wikifier_config = deserialize_wikifier_config(
user_id, project_id, wikifier_config_file_name)
item_table = ItemTable(wikifier_config)
region, template, created_by = load_yaml_data(
yaml_file_path, item_table, data_file_path, sheet_name)
yaml_configuration.set_region(region)
yaml_configuration.set_template(template)
yaml_configuration.set_created_by(created_by)
save_yaml_config(yaml_config_file_path, yaml_configuration)
template = yaml_configuration.get_template()
response['yamlRegions'] = highlight_region(item_table,
data_file_path,
sheet_name, region,
template)
project_meta["yamlMapping"] = dict()
project_meta["yamlMapping"][data_file_name] = dict()
project_meta["yamlMapping"][data_file_name][
sheet_name] = yaml_file_id
project.update_project_config(project_meta)
else:
response['yamlRegions'] = None
response['error'] = "Upload data file before applying YAML."
return json.dumps(response, indent=3)
def upload_yaml():
"""
This function process the yaml
:return:
"""
user_id = request.form["id"]
yaml_data = request.values["yaml"]
os.makedirs("uploads", exist_ok=True)
user = app.config['users'].get_user(user_id)
user.reset('yaml')
yaml_configuration = user.get_yaml_data()
excel_data_filepath = user.get_excel_data().get_file_location()
response = dict()
if check_if_empty(yaml_data):
response['error'] = "YAML file is either empty or not valid"
else:
sheet_name = user.get_excel_data().get_sheet_name()
filename = str(Path(app.config['UPLOAD_FOLDER']) / user_id) + ".yaml"
with open(filename, "w") as f:
f.write(yaml_data)
yaml_configuration.set_file_location(filename)
region, template = load_yaml_data(filename)
yaml_configuration.set_region(region)
yaml_configuration.set_template(template)
item_table = user.get_wikifier_output_data().get_item_table()
template = yaml_configuration.get_template()
response['region'] = highlight_region(item_table, excel_data_filepath, sheet_name, region, template)
return json.dumps(response, indent=3)
def run_t2wml(
data_file_path: str,
wikified_output_path: str,
t2wml_spec: str,
output_directory: str,
sheet_name: str = None,
sparql_endpoint:
str = "http://dsbox02.isi.edu:8888/bigdata/namespace/wdq/sparql"):
try:
yaml_configuration = YAMLFile()
yaml_configuration.set_file_location(t2wml_spec)
region, template = load_yaml_data(t2wml_spec)
yaml_configuration.set_region(region)
yaml_configuration.set_template(template)
except:
logging.error("Invalid YAML File")
return
try:
item_table = ItemTable()
build_item_table(item_table, wikified_output_path, data_file_path,
sheet_name)
except:
logging.error("Invalid Wikfied Output File")
return
filetype = "ttl"
response = generate_download_file(None, item_table, data_file_path,
sheet_name, region, template, filetype,
sparql_endpoint)
file_name = Path(data_file_path).name
result_directory = '.'.join(file_name.split(".")[:-1])
try:
file_extension = file_name.split(".")[-1]
except:
logging.error("Data file has no extension")
return
output_path = Path()
if file_extension == "csv":
output_path = Path(output_directory) / result_directory
elif file_extension == "xls" or file_extension == "xlsx":
if not sheet_name:
sheet_name = get_first_sheet_name(data_file_path)
output_path = Path(output_directory) / result_directory / sheet_name
Path.mkdir(output_path, parents=True, exist_ok=True)
with open(str(output_path / "results.ttl"), "w") as fp:
fp.write(response["data"])
with open(str(output_path / "changes.tsv"), "w") as fp:
serialize_change_record(fp)