def main():
###################################################################
# load config file
with open("config.json") as f:
global_config = json.load( f)
""" load three csv files,
aggregate them to form a join
"""
json_person = {}
filename = os.path.join(global_config["home"], "data/work/iswc2013/raw/payments.csv")
json_payment = UtilCsv.csv2json(filename)
for entry in json_payment:
key = entry["name"].lower()
if key in json_person:
data = json_person[key]
else:
data = {"name":entry["name"],
"paid":False,
"attend":False,
"paper":[]}
json_person[key]=data
data["email_payment"]= entry["email"]
data["id_payment"]= entry["id"]
data["paid"]= True
filename = os.path.join(global_config["home"], "data/work/iswc2013/raw/attendees.csv")
json_attendees = UtilCsv.csv2json(filename)
for entry in json_attendees:
key = entry["name"].lower()
if key in json_person:
data = json_person[key]
else:
data = {"name":entry["name"],
"paid":False,
"attend":False,
"paper":[]}
json_person[key]=data
data["email_attendees"]= entry["email"]
data["id_attendees"]= entry["id"]
data["attend"]= True
json_output=[]
filename = os.path.join(global_config["home"], "data/source/iswc-2013-paper.csv")
json_paper = UtilCsv.csv2json(filename)
#split authors
for entry in json_paper:
title = entry["title"]
entry["author_list"]= [x.strip() for x in entry["author"].split(',')]
#print len(json_paper), entry
data_paper = { "paid":[], "attend":[]}
for key in ["title","category","author"]:
data_paper[key] =entry[key]
json_output.append(data_paper)
for name in entry["author_list"]:
key =name.lower()
if key in json_person:
json_person[key]["paper"].append(title)
if json_person[key]["paid"]:
data_paper["paid"].append(name)
if json_person[key]["attend"]:
data_paper["attend"].append(name)
filename_output = os.path.join(global_config["home"], "data/work/iswc2013/raw/stat_paper.csv")
with open(filename_output,"w") as f:
csvwriter = UnicodeWriter(f)
headers = ["category","author","title","paid","attend"]
csvwriter.writerow(headers)
for entry in json_output:
#print entry
row = UtilString.json2list(entry, headers)
csvwriter.writerow(row)
filename_output = os.path.join(global_config["home"], "data/work/iswc2013/raw/stat_person.csv")
with open(filename_output,"w") as f:
csvwriter = UnicodeWriter(f)
headers = ["name","paid","attend","paper"]
csvwriter.writerow(headers)
for entry in sorted(json_person.values(), key=lambda x:x["name"]):
#print entry
row = UtilString.json2list(entry, headers)
csvwriter.writerow(row)
def gen_open_access_index(global_config, local_config):
dir_home = global_config["home"]
id_data = local_config["id"]
id_html = "gen_open_access_index"
data_all = []
list_filename_input = []
# list_filename_input.append( "{0}/data/source/iswc-all-papers.csv".format(
# dir_home))
list_filename_input.append("{0}/data/source/iswc-2013-paper.csv".format(
dir_home))
for filename_input in list_filename_input:
#load data
data_json = UtilCsv.csv2json(filename_input)
IswcDirect.debug("load {} entries from [{}]".format(len(data_json), filename_input))
#prepare json for templating
p_group = "tracks"
p_item = "papers"
data_for_jsont = {p_group: []}
prev = None
for entry in data_json:
if len(entry['title'])==0:
continue
if None == prev or entry["proceedings_uri"] != prev["proceedings_uri"]:
prev = {"label": entry["category"],
"proceedings_uri": entry["proceedings_uri"],
p_item: []}
data_for_jsont = {p_group: []}
data_all_entry = {"label": "-".join([entry["year"], entry["proceedings_uri"].split("/")[-1]]),
"data": data_for_jsont
}
data_all.append(data_all_entry)
data_for_jsont[p_group].append(prev)
if None == prev or entry["category"] != prev["label"]:
prev = {"label": entry["category"],
"proceedings_uri": entry["proceedings_uri"],
p_item: []}
data_for_jsont[p_group].append(prev)
#copy entry_new metadata
entry_new = {}
for p in ["author", "title", "pages", "link_open_access", "abstract"]:
entry_new[p] = entry[p]
# link local
entry_new["link_local"] = entry_new["link_open_access"].split("/")[-1]
#print '---->', entry_new['title']
#if len(entry_new["link_open_access"])>0:
# id_data = entry_new["link_open_access"].split("/")[-2]
# dir_paper = os.path.join(dir_home, "data/paper/{}".format(id_data))
#
# temp_filename = entry_new["link_open_access"].split("/")[-1]
# #print temp_filename
# temp_id, temp_ext = temp_filename.split('.')
# pretty_name = IswcDirect.create_pretty_filename(entry_new['title'])
# entry_new["link_local"] = '{}-{}.{}'.format(temp_id, pretty_name, temp_ext)
# print "{}".format(entry_new["link_local"])
#
# filename_old = os.path.join(dir_paper, temp_filename)
# #print os.path.exists(filename_old)
# filename_new = os.path.join(dir_paper, entry_new["link_local"])
# #print os.path.exists(filename_new)
# os.rename(filename_old, filename_new)
# link local
if len(entry_new["abstract"]) == 0:
entry_new["abstract"] = "TBA"
if entry["pages"]:
entry_new['page_start'] = entry["pages"].split("-")[0]
#add uri
entry_new["uri"] = "#{}".format(create_ascii_localname(entry_new["title"], escape=True))
entry_new['author_latex'] = unicode2latex(entry["author"])
entry_new['authors'] = []
for x in entry["author"].split(","):
person = {"name": x.strip()}
person["uri"] = "#{}".format(create_ascii_localname(person["name"], escape=True))
entry_new['authors'].append(person)
prev[p_item].append(entry_new)
filename_input = list_filename_input[0]
head, tail = os.path.split(filename_input)
filename_output = "{}/data/open_access/{}".format(
dir_home,
tail)
shutil.copyfile(filename_input, filename_output)
for data_all_entry in data_all:
data_for_jsont = data_all_entry["data"]
path = data_all_entry["label"]
#print json.dumps(data_for_jsont, indent=4)
#write index-usb
filename_html = "%s/data/open_access/%s/index.html" % (
dir_home,
path)
json_template = resource_string('resources.files', '{}.jsont'.format(id_html))
content = jsontemplate.expand(json_template, data_for_jsont)
if os.path.exists(os.path.dirname(filename_html)):
with codecs.open(filename_html, "w", "utf-8") as f:
f.write(u'\ufeff')
f.write(content)
IswcDirect.debug("write to file [{}]".format(filename_html))
def gen_conf_organizer(global_config, local_config):
dir_home = global_config["home"]
id_data = local_config["id"]
id_html = "gen_conf_organizer"
#load data
filename_input = "{0}/data/source/{1}-person.csv".format(
dir_home,
id_data)
data_json = UtilCsv.csv2json(filename_input)
IswcDirect.debug("load {} entries from [{}]".format(len(data_json), filename_input))
#only keep organizer
data_json_new = []
for entry in data_json:
if entry["role_event"] == "[ME]" and entry["role_type"] == "swc:Chair":
data_json_new.append(entry)
data_json = data_json_new
IswcDirect.debug("keep {} persons for conf organization".format(len(data_json)))
#prepare json for templating
data_for_jsont = {"roles": []}
role_prev = None
for entry in data_json:
if None == role_prev or entry["role_label"] != role_prev["label"]:
role_prev = {"label": entry["role_label"],
"persons": []}
data_for_jsont["roles"].append(role_prev)
#copy person metadata
person = {}
for p in ["name", "homepage", "organization", "country"]:
person[p] = entry[p]
#add person uri
person["uri"] = "#{}".format(create_ascii_localname(person["name"], escape=True))
role_prev["persons"].append(person)
#print json.dumps(data_for_jsont, indent=4)
filename_json = "%s/data/www/%s-%s.json" % (
dir_home,
id_data,
id_html)
with codecs.open(filename_json, "w", "utf-8") as f:
json.dump(data_for_jsont, f, indent=4)
#write html
filename_html = "%s/data/www/%s-%s.html" % (
dir_home,
id_data,
id_html)
json_template = resource_string('resources.files', '{}.jsont'.format(id_html))
content = jsontemplate.expand(json_template, data_for_jsont)
with codecs.open(filename_html, "w", "utf-8") as f:
f.write(u'\ufeff')
f.write(content)
IswcDirect.debug("write to file [{}]".format(filename_html))
def gen_conf_paper(global_config, local_config):
dir_home = global_config["home"]
id_data = local_config["id"]
id_html = "gen_conf_paper"
#load data
filename_input = "{0}/data/source/{1}-paper.csv".format(
dir_home,
id_data)
data_json = UtilCsv.csv2json(filename_input)
IswcDirect.debug("load {} entries from [{}]".format(len(data_json), filename_input))
#prepare json for templating
p_group = "tracks"
p_item = "papers"
data_for_jsont = {p_group: []}
prev = None
for entry in data_json:
x_proceedings = entry["proceedings_uri"].split("/")[-1]
if x_proceedings not in ["proceedings", "proceedings-1", "proceedings-2"]:
#skip non conference proceedings paper
continue
if None == prev or entry["category"] != prev["label"]:
prev = {"label": entry["category"],
p_item: []}
data_for_jsont[p_group].append(prev)
#copy entry_new metadata
entry_new = {}
for p in ["author", "title", "pages", "link_open_access", "abstract"]:
entry_new[p] = entry[p]
# link local
entry_new["link_local"] = "{}".format(entry_new["link_open_access"].split("/")[-1])
# link local
if len(entry_new["abstract"]) == 0:
entry_new["abstract"] = "TBA"
if entry["pages"]:
entry_new['page_start'] = entry["pages"].split("-")[0]
#add uri
entry_new["uri"] = "#{}".format(create_ascii_localname(entry_new["title"], escape=True))
entry_new['author_latex'] = unicode2latex(entry["author"])
entry_new['authors'] = []
for x in entry["author"].split(","):
person = {"name": x.strip()}
person["uri"] = "#{}".format(create_ascii_localname(person["name"], escape=True))
entry_new['authors'].append(person)
prev[p_item].append(entry_new)
filename_json = "%s/data/www/%s-%s.json" % (
dir_home,
id_data,
id_html)
with codecs.open(filename_json, "w", "utf-8") as f:
json.dump(data_for_jsont, f, indent=4)
#write html
filename_html = "%s/data/www/%s-%s.html" % (
dir_home,
id_data,
id_html)
json_template = resource_string('resources.files', '{}.jsont'.format(id_html))
content = jsontemplate.expand(json_template, data_for_jsont)
with codecs.open(filename_html, "w", "utf-8") as f:
f.write(u'\ufeff')
f.write(content)
IswcDirect.debug("write to file [{}]".format(filename_html))
#write index-usb
filename_html = "%s/data/paper/iswc-2013/index.html" % (
dir_home)
json_template = resource_string('resources.files', '{}.jsont'.format("gen_open_access_index"))
content = jsontemplate.expand(json_template, data_for_jsont)
with codecs.open(filename_html, "w", "utf-8") as f:
f.write(u'\ufeff')
f.write(content)
IswcDirect.debug("write to file [{}]".format(filename_html))
