def create_catalogue_section():
config.log_message("Converting the catalogue to JSON...")
pd = read_sheet('catalogue',{'key':str,'value':str})
pd=pd.dropna()
result = pd.to_dict(orient="records")
supported_values=['title','description','creator','contactPoint','license','versionInfo','keyword','identifier','rights','publisher_name','publisher_url']
catalogue_dict={}
publisher={}
for row in result:
if row['key'] in supported_values:
# Those that need more work to get into required structure
if row['key']=='publisher_name':
publisher['name']=row['value']
elif row['key']=='publisher_url':
publisher['url']=row['value']
elif row['key']=='keyword':
catalogue_dict['keyword']=row['value'].split(",")
else:
catalogue_dict[row['key']]=row['value']
catalogue_dict["publisher"]=publisher
config.log_message("Done!")
return catalogue_dict
def rearrange_docs():
#term_dict=create_vocabulary()
if os.path.isfile('all_docs_vecs') == False: all_docs_vecs = {}
else: all_docs_vecs = pd.to_dict('all_docs_vecs')
res_file_1000 = run_query()
map1 = ret_eval(res_file_1000)
all_q_relevant_docs = pd.read_csv(
res_file_1000,
sep=' ',
index_col=False,
header=None,
names=['query', 'b1', 'docno', 'b2', 'b3', 'b4'])
new_relevant_docs = cluster_docs(all_q_relevant_docs, term_dict,
all_docs_vecs)
pd.DataFrame(all_docs_vecs).to_csv('all_docs_vecs')
map2 = ret_eval(new_relevant_docs)
#print (map1,map2)
if map1 < map2: create_ret_file(new_relevant_docs)
def create_configuration_section():
config.log_message("Converting the configuration to JSON...")
pd = read_sheet('configuration',{'key':str,'value':str})
pd=pd.dropna()
result = pd.to_dict(orient="records")
supported_values=['visibility','workflow_key','code']
configuration_dict={}
for row in result:
if row['key'] in supported_values:
configuration_dict[row['key']]=row['value']
config.log_message("Done!")
return configuration_dict
def create_lookups_json(lookup_name, lookup_type):
config.log_message("-- Converting lookup: '" + lookup_name + "' to JSON...")
pd = read_sheet('lookups',{'lookup':str,'name':str,'description':str})
result = pd.to_dict(orient="records")
lookups_dict={}
options_arr=[]
lookups_dict['type']=lookup_type
lookups_dict['options']=options_arr
for row in result:
if row['lookup']==lookup_name:
options_dict={}
options_dict['name']=row['name']
options_dict['description']=str(row['description'])
options_arr.append(options_dict)
return lookups_dict
def add_rows(self, pd, formats=None):
rows = pd.to_dict(orient='records')
if formats is None:
formats = [{}] * len(rows[0].keys())
for i, row in enumerate(rows):
rows[i] = [
Latex.format_row_value(v, f)
for v, f in zip(row.values(), formats)
]
t = jinja2.Template(r"""
{% for row in data %}{% for v in row %} {{ v }} {% if not loop.last %}&{% endif %}{% endfor %} \\
{% if not loop.last %}\hdashline{% endif %}{% endfor %}
""")
self._f.write(t.render(data=rows))
def create_dictionary_section():
pd = read_sheet('dictionaries',{'code':str,'name':str,'description':str})
dictionaries = pd.to_dict(orient="records")
dicts_arr=[]
for row in dictionaries:
config.log_message("Converting dictionary '" + row['name'] + "' to JSON...")
fields_dict={}
fields_dict['code']=row['code']
fields_dict['name']=row['name']
fields_dict['description']=row['description']
fields,lookups=create_fields_json(row['code'])
fields_dict['fields']=fields
fields_dict['lookups']=lookups
dicts_arr.append(fields_dict)
config.log_message("Done!")
return dicts_arr
def create_fields_json(dictionary_code):
config.log_message("-- Converting fields to JSON...")
pd = read_sheet('fields',{'dictionary_code':str,'name':str,'label':str,'type':str,'constraints':str,'description':str})
pd=pd.replace(np.nan,"null")
result = pd.to_dict(orient="records")
fields_arr=[]
constraints_dict={}
for row in result:
if row['dictionary_code']==dictionary_code:
fields_dict={}
fields_dict['name']=row['name']
fields_dict['label']=row['label']
fields_dict['type']=row['type']
fields_dict['constraints']=row['constraints']
if row['constraints']!='null':
constraints_dict[row['constraints']]=row['type']
fields_dict['description']=row['description']
fields_arr.append(fields_dict)
lookups_dict={}
for constraint, type in constraints_dict.items():
lookups_dict[constraint]=create_lookups_json(constraint, type)
return fields_arr, lookups_dict
def data_to_json(self, pd, file_name):
response = pd.to_dict('records')
with open(os.path.join(self.data_path, file_name), 'w') as f:
f.write(json.dumps(response, indent=4))