def build_panel(config, switch, handles, datafilter):
(geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, '')
(origdata, maindata, metadata) = request_datasets(config, switch, modern, historical, handles, geolist)
(subsets, panel) = ({}, [])
logscale = ''
for handle in handles:
(datasubset, ctrlist) = datasetfilter(maindata[handle], datafilter)
if not datasubset.empty:
datasubset = datasubset.dropna(how='all')
try:
if np.nan in datasubset.index:
datasubset = datasubset.drop(np.nan, axis=0)
except:
skip = 'yes'
datasubset['handle'] = handle
metadata['url'] = 0
panel.append(datasubset)
subsets[handle] = datasubset
(panelcells, originalvalues) = dataset2panel(config, subsets[handle], historical, logscale)
totalpanel = pd.concat(panel)
try:
if np.nan in totalpanel.index:
totalpanel = totalpanel.drop(np.nan, axis=0)
except:
skip = 'yes'
if switch == 'historical':
geocoder = historical
else:
geocoder = modern
# Remove columns with empty years
for colyear in totalpanel.columns:
if totalpanel[colyear].count() == 0:
totalpanel = totalpanel.drop(colyear, axis=1)
(allyears, nyears) = selectint(totalpanel.columns)
print totalpanel.index
panels = []
known = {}
matrix = {}
#return (str(totalpanel.to_html()), '', '', '')
for code in totalpanel.index:
try:
country = geocoder.ix[int(code)][config['webmappercountry']]
except:
country = ''
for thisyear in allyears:
thiskey = str(int(code)) + str(thisyear)
if thiskey not in known:
dataitem = [country]
dataitem.append(thisyear)
known[thiskey] = thisyear
matrix[thiskey] = ''
for handle in handles:
tmpframe = totalpanel.loc[totalpanel['handle'] == handle]
try:
thisval = tmpframe.ix[int(code)][thisyear]
matrix[thiskey] = thisval
except:
thisval = ''
dataitem.append(thisval)
# Filter out np.NaN
if str(thisval) != 'nan':
if country:
if matrix[thiskey]:
panels.append(dataitem)
# Build header
header = ['Country', 'Year']
for handle in handles:
header.append(metadata[handle]['title'])
return (header, panels, metadata, totalpanel)
csvio = StringIO(content)
dataframe = pd.read_csv(csvio, sep='\t', dtype='unicode')
return dataframe
fullpath = "/home/tikhonov/completedatasets14Sept2015/GDPperCapita.xlsx"
allfiles = ["/home/tikhonov/completedatasets14Sept2015/GDPperCapita.xlsx", "/home/tikhonov/completedatasets14Sept2015/TotalCO2Emissions.xlsx"]
outdir = "/home/tikhonov/data/historical.all"
path = "/home/tikhonov/tmp"
mypath = "/home/tikhonov/data/clio2/completedatainclKeesJonat1Nov2015"
onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]
fileID = "test"
config = configuration()
config['remote'] = 'yes'
(geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, 'geocoder')
def compiledataset(csvfile):
handles = []
remote = 'on'
# Geocoder
config = configuration()
config['remote'] = 'yes'
dataframe = loaddataset_fromfile(config, csvfile)
dataset = dataframe
title = dataframe.columns[1]
units = dataframe.ix[0][1]
metadata = {}
switch = 'modern'
switch = 'historical'
#dataset = dataset.convert_objects(convert_numeric=True)
def build_panel(config, switch, handles, datafilter):
(geocoder, geolist, oecd2webmapper, modern,
historical) = request_geocoder(config, '')
(origdata, maindata, metadata) = request_datasets(config, switch, modern,
historical, handles,
geolist)
(subsets, panel) = ({}, [])
logscale = ''
for handle in handles:
(datasubset, ctrlist) = datasetfilter(maindata[handle], datafilter)
if not datasubset.empty:
datasubset = datasubset.dropna(how='all')
try:
if np.nan in datasubset.index:
datasubset = datasubset.drop(np.nan, axis=0)
except:
skip = 'yes'
datasubset['handle'] = handle
metadata['url'] = 0
panel.append(datasubset)
subsets[handle] = datasubset
(panelcells,
originalvalues) = dataset2panel(config, subsets[handle],
historical, logscale)
totalpanel = pd.concat(panel)
try:
if np.nan in totalpanel.index:
totalpanel = totalpanel.drop(np.nan, axis=0)
except:
skip = 'yes'
if switch == 'historical':
geocoder = historical
else:
geocoder = modern
# Remove columns with empty years
for colyear in totalpanel.columns:
if totalpanel[colyear].count() == 0:
totalpanel = totalpanel.drop(colyear, axis=1)
(allyears, nyears) = selectint(totalpanel.columns)
print totalpanel.index
panels = []
known = {}
matrix = {}
#return (str(totalpanel.to_html()), '', '', '')
for code in totalpanel.index:
try:
country = geocoder.ix[int(code)][config['webmappercountry']]
except:
country = ''
for thisyear in allyears:
thiskey = str(int(code)) + str(thisyear)
if thiskey not in known:
dataitem = [country]
dataitem.append(thisyear)
known[thiskey] = thisyear
matrix[thiskey] = ''
for handle in handles:
tmpframe = totalpanel.loc[totalpanel['handle'] == handle]
try:
thisval = tmpframe.ix[int(code)][thisyear]
matrix[thiskey] = thisval
except:
thisval = ''
dataitem.append(thisval)
# Filter out np.NaN
if str(thisval) != 'nan':
if country:
if matrix[thiskey]:
panels.append(dataitem)
# Build header
header = ['Country', 'Year']
for handle in handles:
header.append(metadata[handle]['title'])
return (header, panels, metadata, totalpanel)
fullpath = "/home/tikhonov/completedatasets14Sept2015/GDPperCapita.xlsx"
allfiles = [
"/home/tikhonov/completedatasets14Sept2015/GDPperCapita.xlsx",
"/home/tikhonov/completedatasets14Sept2015/TotalCO2Emissions.xlsx"
]
outdir = "/home/tikhonov/data/historical.all"
path = "/home/tikhonov/tmp"
mypath = "/home/tikhonov/data/clio2/completedatainclKeesJonat1Nov2015"
onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]
fileID = "test"
config = configuration()
config['remote'] = 'yes'
(geocoder, geolist, oecd2webmapper, modern,
historical) = request_geocoder(config, 'geocoder')
def compiledataset(csvfile):
handles = []
remote = 'on'
# Geocoder
config = configuration()
config['remote'] = 'yes'
dataframe = loaddataset_fromfile(config, csvfile)
dataset = dataframe
title = dataframe.columns[1]
units = dataframe.ix[0][1]
metadata = {}
switch = 'modern'
