def process_multiple(prefs):
print prefs['infile']
infile = os.path.join(args.indir, prefs['infile'])
with open(infile, 'rb') as f:
reader = csv.DictReader(f)
progress = 1
done = float(check_output(['wc', '-l', infile]).split()[0])
aggregated_data_dict = defaultdict(list)
# go through aggregation keys appending to appropriate data structre
for line in reader:
progress += 1
sys.stdout.write("\r{:.2%}".format(progress/done))
for agg_region in prefs['agg_region']:
k = agg_col[agg_region]
if line[k]:
line = process_line(agg_region, k, line)
outputline = [line.get(x, '') for x in header[agg_region]]
aggregated_data_dict[agg_region].append(outputline)
print ' -- DONE PROCESSING FILE'
for agg_region in prefs['agg_region']:
outdata = aggregated_data_dict[agg_region]
print agg_region + ': ' + str(len(outdata))
ensureDir(os.path.join(args.outdir, agg_region))
outfile = os.path.join(args.outdir, agg_region, prefs['outfile'] + '.csv')
with open(outfile, 'wb') as f_out:
writer = csv.writer(f_out)
writer.writerow(header[agg_region])
writer.writerows(outdata)
reader = csv.DictReader(f)
progress = 1
done = float(check_output(['wc', '-l', infile]).split()[0])
# go through aggregation keys appending to appropriate data structre
output = []
for line in reader:
progress += 1
sys.stdout.write("\r{:.2%}".format(progress/done))
sub = line['CHEM_E']
if sub not in subdict: continue
emissions = float(line['Total_kg'])
teq = convertTF(emissions, subdict[sub])
output.append({agg[key]: line[agg[key]], subdict[sub]: emissions, subdict[sub].split('_')[0] + '_TQ': teq})
print '\n' + key + ': ' + str(len(output))
ensureDir(os.path.join(args.outdir, key))
outfile = os.path.join(args.outdir, key, 'Industrial Emitters.csv')
with open(outfile, 'wb') as f_out:
writer = csv.DictWriter(f_out, fieldnames=header[key])
writer.writeheader()
writer.writerows(output)
print ' -- DONE PROCESSING FILE'