def main(): ################################################################### # load config file with open("config.json") as f: global_config = json.load( f) """ load three csv files, aggregate them to form a join """ json_person = {} filename = os.path.join(global_config["home"], "data/work/iswc2013/raw/payments.csv") json_payment = UtilCsv.csv2json(filename) for entry in json_payment: key = entry["name"].lower() if key in json_person: data = json_person[key] else: data = {"name":entry["name"], "paid":False, "attend":False, "paper":[]} json_person[key]=data data["email_payment"]= entry["email"] data["id_payment"]= entry["id"] data["paid"]= True filename = os.path.join(global_config["home"], "data/work/iswc2013/raw/attendees.csv") json_attendees = UtilCsv.csv2json(filename) for entry in json_attendees: key = entry["name"].lower() if key in json_person: data = json_person[key] else: data = {"name":entry["name"], "paid":False, "attend":False, "paper":[]} json_person[key]=data data["email_attendees"]= entry["email"] data["id_attendees"]= entry["id"] data["attend"]= True json_output=[] filename = os.path.join(global_config["home"], "data/source/iswc-2013-paper.csv") json_paper = UtilCsv.csv2json(filename) #split authors for entry in json_paper: title = entry["title"] entry["author_list"]= [x.strip() for x in entry["author"].split(',')] #print len(json_paper), entry data_paper = { "paid":[], "attend":[]} for key in ["title","category","author"]: data_paper[key] =entry[key] json_output.append(data_paper) for name in entry["author_list"]: key =name.lower() if key in json_person: json_person[key]["paper"].append(title) if json_person[key]["paid"]: data_paper["paid"].append(name) if json_person[key]["attend"]: data_paper["attend"].append(name) filename_output = os.path.join(global_config["home"], "data/work/iswc2013/raw/stat_paper.csv") with open(filename_output,"w") as f: csvwriter = UnicodeWriter(f) headers = ["category","author","title","paid","attend"] csvwriter.writerow(headers) for entry in json_output: #print entry row = UtilString.json2list(entry, headers) csvwriter.writerow(row) filename_output = os.path.join(global_config["home"], "data/work/iswc2013/raw/stat_person.csv") with open(filename_output,"w") as f: csvwriter = UnicodeWriter(f) headers = ["name","paid","attend","paper"] csvwriter.writerow(headers) for entry in sorted(json_person.values(), key=lambda x:x["name"]): #print entry row = UtilString.json2list(entry, headers) csvwriter.writerow(row)
def gen_open_access_index(global_config, local_config): dir_home = global_config["home"] id_data = local_config["id"] id_html = "gen_open_access_index" data_all = [] list_filename_input = [] # list_filename_input.append( "{0}/data/source/iswc-all-papers.csv".format( # dir_home)) list_filename_input.append("{0}/data/source/iswc-2013-paper.csv".format( dir_home)) for filename_input in list_filename_input: #load data data_json = UtilCsv.csv2json(filename_input) IswcDirect.debug("load {} entries from [{}]".format(len(data_json), filename_input)) #prepare json for templating p_group = "tracks" p_item = "papers" data_for_jsont = {p_group: []} prev = None for entry in data_json: if len(entry['title'])==0: continue if None == prev or entry["proceedings_uri"] != prev["proceedings_uri"]: prev = {"label": entry["category"], "proceedings_uri": entry["proceedings_uri"], p_item: []} data_for_jsont = {p_group: []} data_all_entry = {"label": "-".join([entry["year"], entry["proceedings_uri"].split("/")[-1]]), "data": data_for_jsont } data_all.append(data_all_entry) data_for_jsont[p_group].append(prev) if None == prev or entry["category"] != prev["label"]: prev = {"label": entry["category"], "proceedings_uri": entry["proceedings_uri"], p_item: []} data_for_jsont[p_group].append(prev) #copy entry_new metadata entry_new = {} for p in ["author", "title", "pages", "link_open_access", "abstract"]: entry_new[p] = entry[p] # link local entry_new["link_local"] = entry_new["link_open_access"].split("/")[-1] #print '---->', entry_new['title'] #if len(entry_new["link_open_access"])>0: # id_data = entry_new["link_open_access"].split("/")[-2] # dir_paper = os.path.join(dir_home, "data/paper/{}".format(id_data)) # # temp_filename = entry_new["link_open_access"].split("/")[-1] # #print temp_filename # temp_id, temp_ext = temp_filename.split('.') # pretty_name = IswcDirect.create_pretty_filename(entry_new['title']) # entry_new["link_local"] = '{}-{}.{}'.format(temp_id, pretty_name, temp_ext) # print "{}".format(entry_new["link_local"]) # # filename_old = os.path.join(dir_paper, temp_filename) # #print os.path.exists(filename_old) # filename_new = os.path.join(dir_paper, entry_new["link_local"]) # #print os.path.exists(filename_new) # os.rename(filename_old, filename_new) # link local if len(entry_new["abstract"]) == 0: entry_new["abstract"] = "TBA" if entry["pages"]: entry_new['page_start'] = entry["pages"].split("-")[0] #add uri entry_new["uri"] = "#{}".format(create_ascii_localname(entry_new["title"], escape=True)) entry_new['author_latex'] = unicode2latex(entry["author"]) entry_new['authors'] = [] for x in entry["author"].split(","): person = {"name": x.strip()} person["uri"] = "#{}".format(create_ascii_localname(person["name"], escape=True)) entry_new['authors'].append(person) prev[p_item].append(entry_new) filename_input = list_filename_input[0] head, tail = os.path.split(filename_input) filename_output = "{}/data/open_access/{}".format( dir_home, tail) shutil.copyfile(filename_input, filename_output) for data_all_entry in data_all: data_for_jsont = data_all_entry["data"] path = data_all_entry["label"] #print json.dumps(data_for_jsont, indent=4) #write index-usb filename_html = "%s/data/open_access/%s/index.html" % ( dir_home, path) json_template = resource_string('resources.files', '{}.jsont'.format(id_html)) content = jsontemplate.expand(json_template, data_for_jsont) if os.path.exists(os.path.dirname(filename_html)): with codecs.open(filename_html, "w", "utf-8") as f: f.write(u'\ufeff') f.write(content) IswcDirect.debug("write to file [{}]".format(filename_html))
def gen_conf_organizer(global_config, local_config): dir_home = global_config["home"] id_data = local_config["id"] id_html = "gen_conf_organizer" #load data filename_input = "{0}/data/source/{1}-person.csv".format( dir_home, id_data) data_json = UtilCsv.csv2json(filename_input) IswcDirect.debug("load {} entries from [{}]".format(len(data_json), filename_input)) #only keep organizer data_json_new = [] for entry in data_json: if entry["role_event"] == "[ME]" and entry["role_type"] == "swc:Chair": data_json_new.append(entry) data_json = data_json_new IswcDirect.debug("keep {} persons for conf organization".format(len(data_json))) #prepare json for templating data_for_jsont = {"roles": []} role_prev = None for entry in data_json: if None == role_prev or entry["role_label"] != role_prev["label"]: role_prev = {"label": entry["role_label"], "persons": []} data_for_jsont["roles"].append(role_prev) #copy person metadata person = {} for p in ["name", "homepage", "organization", "country"]: person[p] = entry[p] #add person uri person["uri"] = "#{}".format(create_ascii_localname(person["name"], escape=True)) role_prev["persons"].append(person) #print json.dumps(data_for_jsont, indent=4) filename_json = "%s/data/www/%s-%s.json" % ( dir_home, id_data, id_html) with codecs.open(filename_json, "w", "utf-8") as f: json.dump(data_for_jsont, f, indent=4) #write html filename_html = "%s/data/www/%s-%s.html" % ( dir_home, id_data, id_html) json_template = resource_string('resources.files', '{}.jsont'.format(id_html)) content = jsontemplate.expand(json_template, data_for_jsont) with codecs.open(filename_html, "w", "utf-8") as f: f.write(u'\ufeff') f.write(content) IswcDirect.debug("write to file [{}]".format(filename_html))
def gen_conf_paper(global_config, local_config): dir_home = global_config["home"] id_data = local_config["id"] id_html = "gen_conf_paper" #load data filename_input = "{0}/data/source/{1}-paper.csv".format( dir_home, id_data) data_json = UtilCsv.csv2json(filename_input) IswcDirect.debug("load {} entries from [{}]".format(len(data_json), filename_input)) #prepare json for templating p_group = "tracks" p_item = "papers" data_for_jsont = {p_group: []} prev = None for entry in data_json: x_proceedings = entry["proceedings_uri"].split("/")[-1] if x_proceedings not in ["proceedings", "proceedings-1", "proceedings-2"]: #skip non conference proceedings paper continue if None == prev or entry["category"] != prev["label"]: prev = {"label": entry["category"], p_item: []} data_for_jsont[p_group].append(prev) #copy entry_new metadata entry_new = {} for p in ["author", "title", "pages", "link_open_access", "abstract"]: entry_new[p] = entry[p] # link local entry_new["link_local"] = "{}".format(entry_new["link_open_access"].split("/")[-1]) # link local if len(entry_new["abstract"]) == 0: entry_new["abstract"] = "TBA" if entry["pages"]: entry_new['page_start'] = entry["pages"].split("-")[0] #add uri entry_new["uri"] = "#{}".format(create_ascii_localname(entry_new["title"], escape=True)) entry_new['author_latex'] = unicode2latex(entry["author"]) entry_new['authors'] = [] for x in entry["author"].split(","): person = {"name": x.strip()} person["uri"] = "#{}".format(create_ascii_localname(person["name"], escape=True)) entry_new['authors'].append(person) prev[p_item].append(entry_new) filename_json = "%s/data/www/%s-%s.json" % ( dir_home, id_data, id_html) with codecs.open(filename_json, "w", "utf-8") as f: json.dump(data_for_jsont, f, indent=4) #write html filename_html = "%s/data/www/%s-%s.html" % ( dir_home, id_data, id_html) json_template = resource_string('resources.files', '{}.jsont'.format(id_html)) content = jsontemplate.expand(json_template, data_for_jsont) with codecs.open(filename_html, "w", "utf-8") as f: f.write(u'\ufeff') f.write(content) IswcDirect.debug("write to file [{}]".format(filename_html)) #write index-usb filename_html = "%s/data/paper/iswc-2013/index.html" % ( dir_home) json_template = resource_string('resources.files', '{}.jsont'.format("gen_open_access_index")) content = jsontemplate.expand(json_template, data_for_jsont) with codecs.open(filename_html, "w", "utf-8") as f: f.write(u'\ufeff') f.write(content) IswcDirect.debug("write to file [{}]".format(filename_html))