def dataframe_compiler(config, fullpath, handle, switch, datafilter):
handles = []
ctrlist = []
metadata = {}
#switch = 'historical'
# Load Geocoder
(classification, dataset, title,
units) = content2dataframe(config, config['geocoderhandle'])
(geocoder, geolist, oecd2webmapper) = buildgeocoder(dataset, config, '')
(modern, historical) = loadgeocoder(config, dataset, 'geocoder')
coderyears = []
# Default years selection
for i in range(1500, 2016):
coderyears.append(i)
# Years selected
if datafilter['startyear']:
coderyears = []
for i in range(int(datafilter['startyear']),
int(datafilter['endyear'])):
coderyears.append(i)
# Reading dataset
(class1, dataset, title, units) = content2dataframe(config, handle)
filetitle = title
filetitle = re.sub(' ', '_', filetitle)
filetitle = re.sub(r'[\(\)]', '_', filetitle)
if filetitle:
fullpath = "%s/%s_%s.xlsx" % (fullpath, filetitle, switch)
#return ('test', 'test')
if switch == 'modern':
activeindex = modern.index
coder = modern
class1 = switch
else:
activeindex = historical.index
coder = historical
(moderndata, historicaldata) = loadgeocoder(config, dataset, '')
if switch == 'modern':
maindata = moderndata
else:
# Do conversion to webmapper system if there are no historical data
isdata = ''
try:
if not historicaldata.empty:
isdata = 'yes'
except:
isdata = 'no'
if isdata == 'no':
maindata = moderndata
webmapperindex = []
for code in maindata.index:
try:
webmappercode = oecd2webmapper[int(code)]
except:
webmappercode = -1
webmapperindex.append(webmappercode)
maindata.index = webmapperindex
# Drop not recognized locations
maindata = maindata[maindata.index > 0]
else:
maindata = historicaldata
if title:
metadata['title'] = title
else:
metadata['title'] = ''
if units:
metadata['units'] = units
else:
metadata['units'] = ''
(cfilter, notint) = selectint(maindata.columns)
codes = selectint(maindata.index)
geo = load_geocodes(config, switch, codes, maindata, geolist)
for colname in notint:
maindata = maindata.drop(colname, axis=1)
# Apply filters
if datafilter:
(datasubset, ctrlist) = datasetfilter(maindata, datafilter)
else:
datasetset = maindata
# Create Excel file out from dataframe
(yearscolumns, notyears) = selectint(maindata.columns)
(countryinfo, notcountry) = selectint(maindata.index)
(finalsubset, icoder, isyear, ctrfilter,
nodata) = dataset_analyzer(datasubset, coder, yearscolumns)
#return (finalsubset, fullpath, finalsubset)
# Apply filter to countries
if datafilter['ctrlist']:
tmpcoder = icoder.ix[ctrlist]
icoder = pd.DataFrame(tmpcoder)
#return (finalsubset.to_html(), 'test')
if fullpath:
config['emptyvalues'] = 'no'
if config['emptyvalues'] == 'no':
(coderyears, notyears) = selectint(finalsubset.columns)
datafile = create_excel_dataset(fullpath, icoder, metadata,
icoder.columns, coderyears,
finalsubset, isyear, ctrfilter)
return (filetitle, fullpath, finalsubset)
def dataframe_compiler(config, fullpath, handle, switch, datafilter):
handles = []
ctrlist = []
metadata = {}
# switch = 'historical'
# Load Geocoder
(classification, dataset, title, units) = content2dataframe(config, config["geocoderhandle"])
(geocoder, geolist, oecd2webmapper) = buildgeocoder(dataset, config, "")
(modern, historical) = loadgeocoder(config, dataset, "geocoder")
coderyears = []
# Default years selection
for i in range(1500, 2016):
coderyears.append(i)
# Years selected
if datafilter["startyear"]:
coderyears = []
for i in range(int(datafilter["startyear"]), int(datafilter["endyear"])):
coderyears.append(i)
# Reading dataset
(class1, dataset, title, units) = content2dataframe(config, handle)
filetitle = title
filetitle = re.sub(" ", "_", filetitle)
filetitle = re.sub(r"[\(\)]", "_", filetitle)
if filetitle:
fullpath = "%s/%s_%s.xlsx" % (fullpath, filetitle, switch)
# return ('test', 'test')
if switch == "modern":
activeindex = modern.index
coder = modern
class1 = switch
else:
activeindex = historical.index
coder = historical
(moderndata, historicaldata) = loadgeocoder(config, dataset, "")
if switch == "modern":
maindata = moderndata
else:
# Do conversion to webmapper system if there are no historical data
isdata = ""
try:
if not historicaldata.empty:
isdata = "yes"
except:
isdata = "no"
if isdata == "no":
maindata = moderndata
webmapperindex = []
for code in maindata.index:
try:
webmappercode = oecd2webmapper[int(code)]
except:
webmappercode = -1
webmapperindex.append(webmappercode)
maindata.index = webmapperindex
# Drop not recognized locations
maindata = maindata[maindata.index > 0]
else:
maindata = historicaldata
if title:
metadata["title"] = title
else:
metadata["title"] = ""
if units:
metadata["units"] = units
else:
metadata["units"] = ""
(cfilter, notint) = selectint(maindata.columns)
codes = selectint(maindata.index)
geo = load_geocodes(config, switch, codes, maindata, geolist)
for colname in notint:
maindata = maindata.drop(colname, axis=1)
# Apply filters
if datafilter:
(datasubset, ctrlist) = datasetfilter(maindata, datafilter)
else:
datasetset = maindata
# Create Excel file out from dataframe
(yearscolumns, notyears) = selectint(maindata.columns)
(countryinfo, notcountry) = selectint(maindata.index)
(finalsubset, icoder, isyear, ctrfilter, nodata) = dataset_analyzer(datasubset, coder, yearscolumns)
# return (finalsubset, fullpath, finalsubset)
# Apply filter to countries
if datafilter["ctrlist"]:
tmpcoder = icoder.ix[ctrlist]
icoder = pd.DataFrame(tmpcoder)
# return (finalsubset.to_html(), 'test')
if fullpath:
config["emptyvalues"] = "no"
if config["emptyvalues"] == "no":
(coderyears, notyears) = selectint(finalsubset.columns)
datafile = create_excel_dataset(
fullpath, icoder, metadata, icoder.columns, coderyears, finalsubset, isyear, ctrfilter
)
return (filetitle, fullpath, finalsubset)
from sys import argv
handles = []
config = configuration()
config['remote'] = ''
# Geocoder
handle = config['geocoderhandle']
(classification, dataset, title,
units) = content2dataframe(config, config['geocoderhandle'])
#if remote:
# (classification, dataset) = loaddataset_fromurl(config, handle)
#else:
# dataset = loaddataset(handles)
(geocoder, geolist, oecdlist) = buildgeocoder(dataset, config, '')
#print geocoder
(modern, historical) = loadgeocoder(config, dataset, 'geocoder')
switch = 'historical'
switch = 'modern'
if switch == 'modern':
activeindex = modern.index
coder = modern
class1 = switch
else:
activeindex = historical.index
coder = historical
handle = "hdl:10622/DIUBXI"
handle = "hdl:10622/WNGZ4A"
import openpyxl
from openpyxl.cell import get_column_letter
import numpy as np
handles = []
remote = 'on'
config = configuration()
# Geocoder
handle = config['geocoderhandle']
if remote:
(classification, dataset, title, units) = loaddataset_fromurl(config, handle)
else:
dataset = loaddataset(handles)
(geocoder, geolist, oecd) = buildgeocoder(dataset, config, '')
(modern, historical) = loadgeocoder(config, dataset, 'geocoder')
coderyears = []
for i in range(1500, 2015):
coderyears.append(i)
# Reading dataset
handle = "hdl:10622/WNGZ4A"
#handle = "hdl:10622/DIUBXI"
handles = []
handles.append(handle)
switch = 'historical'
#switch = 'modern'
if remote:
(class1, dataset, title, units) = loaddataset_fromurl(config, handle)
