def toDB(file_path, server_name, db_name, table_name, user, password, csv_delimiter): folder, file = os.path.split(file_path) with Util.cd(folder): reader = UnicodeReader(open(file,'rb')) db = Database(db_name, user, server_name, 5432, password) csv_delimiter = csv_delimiter if csv_delimiter != "" else ";" fields, values = getInsertCommand(reader.next()[0].split(csv_delimiter), table_name) print 'running...' v = [] while True: try: row = reader.next()[0].split(',') except Exception: break; for i in range(len(row)): if row[i] == 'None': row[i] = '' v.append(tuple(row)) args_str = ','.join(db.cur.mogrify(values, x) for x in tuple(v)) db.cur.execute("INSERT INTO " + table_name + " " + fields + " VALUES " + args_str) # db.cur.executemany(insert_command, tuple(v)) db.conn.commit() db.conn.close() print 'done!'
def loadGenderList(gender, country, dataPath, hasHeader): fd = open(os.path.join(dataPath, '%s%sUTF8.csv' % (country, gender)), 'rb') reader = UnicodeReader(fd) names = {} if hasHeader: unused_header = reader.next() '''Load names as-is, but lower cased''' for row in reader: name = row[0].lower() try: '''The second column should be the count (number of babies in some year with this name)''' count = row[1] except: '''If second column does not exist, default to count=1''' count = 1 if names.has_key(name): '''If here then I've seen this name before, modulo case. Only count once (there is no frequency information anyway)''' count = 0 if names.has_key(name): names[name] += count else: names[name] = count fd.close() '''Add versions without diacritics''' for name in names.keys(): dname = unidecode(name) if not names.has_key(dname): names[dname] = names[name] return names
def loadGenderList(gender, country, dataPath, hasHeader): fd = open(os.path.join(dataPath, '%s%sUTF8.csv' % (country, gender)), 'rb') reader = UnicodeReader(fd) names = {} if hasHeader: unused_header = reader.next() '''Load names as-is, but lower cased''' for row in reader: name = row[0].lower() try: '''The second column should be the count (number of babies in some year with this name)''' count = row[1] except: '''If second column does not exist, default to count=1''' count = 1 if names.has_key(name): '''If here then I've seen this name before, modulo case. Only count once (there is no frequency information anyway)''' count = 0 if names.has_key(name): names[name] += count else: names[name] = count fd.close() '''Add versions without diacritics''' for name in names.keys(): dname = unidecode(name) if not names.has_key(dname): names[dname] = names[name] return names
def load_acceptance_ratio(): session = SessionFactory.get_session() print 'Loading acceptance ratios:' f = open(os.path.join(DATA_PATH, 'numSubmissions.csv'), "rb") reader = UnicodeReader(f) header = reader.next() subm = {} for row in reader: year = int(row[0]) for idx,val in enumerate(row[1:]): conf = header[idx+1] try: count = int(val) if conf not in subm.keys(): subm[conf] = {} subm[conf][year] = count except: pass for acronym, name, impact in CONFERENCES: print acronym.upper() conference = session.query(Venue).\ filter_by(acronym=acronym.upper()).\ one() for (year,count) in subm[acronym.upper()].items(): numSubm = SubmissionsCount(year, count) numSubm.venue = conference session.add(numSubm)
author = session.query(Person).\ filter_by(name=author_name).\ one() except: # New author; add to database author = Person(author_name) session.add(author) paper.authors.append(author) # --- Update 8/12/2013 --- # Record also the role of PC members (PC Chair or General Chair) f = open(os.path.join(dataPath, 'SE-conf-roles.csv'), 'rb') reader = UnicodeReader(f) header = reader.next() roles = {} def confName(conf): if conf == 'ESEC/FSE': return 'FSE' elif conf == 'CSMR-WCRE': return 'CSMR' else: return conf for row in reader: conf = confName(row[0].strip()).lower() year = int(row[1]) name = '%s %s' % (row[2].strip(), row[3].strip()) try:
# Choose dataset #dataset = 'aliases2' dataset = 'gnome' dataset = 'icsm' # Choose dataset size datasetsize = 'full' #datasetsize = 'sample' nameEmailData = MyDict() if dataset == 'aliases2': f = open(os.path.join(dataPath, "aliases2.csv"), "rb") # f = open(os.path.join(dataPath, "testData2.csv"), "rb") reader = UnicodeReader(f) header = reader.next() for row in reader: try: idx = int(row[0]) name = row[1] email = unspam(row[2]) nameEmailData[idx] = (name, email) except: print row f.close() print 'Using the aliases2.csv data set...' elif dataset == 'gnome': import email.utils import email.header
def load_pc(): session = SessionFactory.get_session() # --- Update 8/12/2013 --- # Record also the role of PC members (PC Chair or General Chair) f = open(os.path.join(DATA_PATH, 'SE-conf-roles.csv'), 'rb') reader = UnicodeReader(f) header = reader.next() roles = {} def confName(conf): if conf == 'ESEC/FSE': return 'FSE' elif conf == 'CSMR-WCRE': return 'CSMR' else: return conf for row in reader: conf = confName(row[0].strip()).lower() year = int(row[1]) name = '%s %s' % (row[2].strip(), row[3].strip()) try: name = nameMap[name] except: pass role = row[5] if role == 'Organiser': role = 'PC member main track' if '?' not in name and role != 'Challenge Chair' and role != 'Data Chair': roles[(name, conf, year)] = role #Conference;Year;First Name;Last Name;Sex;Role #CSMR;2013;Anthony ;Cleve;Male;Program Chair #CSMR;2013;Filippo;Ricca;Male;Program Chair #CSMR;2013;Maura;Cerioli;Female;General Chair # ----------------------- print 'Loading PC members:' for acronym, name, impact in CONFERENCES: print acronym.upper() # Get the conference object try: # I already have this PC conference in the database conference = session.query(Venue).\ filter_by(acronym=acronym).\ one() except: # New conference; add to database conference = Venue(acronym.upper(), impact, name, is_conference=True) session.add(conference) # Load the data into a csv reader f = open(os.path.join(DATA_PATH, 'normalised-pc', '%s.csv' % acronym.lower()), 'rb') reader = UnicodeReader(f) # --- Update 8/12/2013 --- withRole = set([(name, year) for (name, conf, year) in roles.keys() if conf==acronym]) # ----------------------- for row in reader: # Deconstruct each row year = int(row[0]) role = row[1] pcMemberName = row[2].strip() # --- Update 8/12/2013 --- if roles.has_key((pcMemberName, acronym, year)): role = roles[(pcMemberName, acronym, year)] try: withRole.remove((pcMemberName, year)) except: pass else: role = 'PC member main track' # ----------------------- if len(pcMemberName): # Get the PC member object try: # I already have this PC member in the database pcMember = session.query(Person).\ filter_by(name=pcMemberName).\ one() except: # New person; add to database pcMember = Person(pcMemberName) session.add(pcMember) try: membership = session.query(PCMembership).\ filter_by(year=year).\ filter_by(role=role).\ filter_by(pcmember=pcMember).\ filter_by(venue=conference).\ one() except: # New, add to database membership = PCMembership(year, role) membership.pcmember = pcMember membership.venue = conference session.add(membership) # --- Update 8/12/2013 --- print sorted(withRole) # ----------------------- session.commit()