def dataframe_compiler(config, fullpath, handle, switch, datafilter): handles = [] ctrlist = [] metadata = {} #switch = 'historical' # Load Geocoder (classification, dataset, title, units) = content2dataframe(config, config['geocoderhandle']) (geocoder, geolist, oecd2webmapper) = buildgeocoder(dataset, config, '') (modern, historical) = loadgeocoder(config, dataset, 'geocoder') coderyears = [] # Default years selection for i in range(1500, 2016): coderyears.append(i) # Years selected if datafilter['startyear']: coderyears = [] for i in range(int(datafilter['startyear']), int(datafilter['endyear'])): coderyears.append(i) # Reading dataset (class1, dataset, title, units) = content2dataframe(config, handle) filetitle = title filetitle = re.sub(' ', '_', filetitle) filetitle = re.sub(r'[\(\)]', '_', filetitle) if filetitle: fullpath = "%s/%s_%s.xlsx" % (fullpath, filetitle, switch) #return ('test', 'test') if switch == 'modern': activeindex = modern.index coder = modern class1 = switch else: activeindex = historical.index coder = historical (moderndata, historicaldata) = loadgeocoder(config, dataset, '') if switch == 'modern': maindata = moderndata else: # Do conversion to webmapper system if there are no historical data isdata = '' try: if not historicaldata.empty: isdata = 'yes' except: isdata = 'no' if isdata == 'no': maindata = moderndata webmapperindex = [] for code in maindata.index: try: webmappercode = oecd2webmapper[int(code)] except: webmappercode = -1 webmapperindex.append(webmappercode) maindata.index = webmapperindex # Drop not recognized locations maindata = maindata[maindata.index > 0] else: maindata = historicaldata if title: metadata['title'] = title else: metadata['title'] = '' if units: metadata['units'] = units else: metadata['units'] = '' (cfilter, notint) = selectint(maindata.columns) codes = selectint(maindata.index) geo = load_geocodes(config, switch, codes, maindata, geolist) for colname in notint: maindata = maindata.drop(colname, axis=1) # Apply filters if datafilter: (datasubset, ctrlist) = datasetfilter(maindata, datafilter) else: datasetset = maindata # Create Excel file out from dataframe (yearscolumns, notyears) = selectint(maindata.columns) (countryinfo, notcountry) = selectint(maindata.index) (finalsubset, icoder, isyear, ctrfilter, nodata) = dataset_analyzer(datasubset, coder, yearscolumns) #return (finalsubset, fullpath, finalsubset) # Apply filter to countries if datafilter['ctrlist']: tmpcoder = icoder.ix[ctrlist] icoder = pd.DataFrame(tmpcoder) #return (finalsubset.to_html(), 'test') if fullpath: config['emptyvalues'] = 'no' if config['emptyvalues'] == 'no': (coderyears, notyears) = selectint(finalsubset.columns) datafile = create_excel_dataset(fullpath, icoder, metadata, icoder.columns, coderyears, finalsubset, isyear, ctrfilter) return (filetitle, fullpath, finalsubset)
import simplejson sys.path.append( os.path.abspath(os.path.join(os.path.dirname("__file__"), '../modules'))) from config import configuration, dataverse2indicators, load_dataverse, findpid, load_metadata from storage import data2store, readdata, readdataset, readdatasets, datasetadd, formdatasetquery from datasets import loaddataset, loaddataset_fromurl, loadgeocoder, treemap, selectint, buildgeocoder, content2dataframe from sys import argv handles = [] config = configuration() config['remote'] = '' # Geocoder handle = config['geocoderhandle'] (classification, dataset, title, units) = content2dataframe(config, config['geocoderhandle']) #if remote: # (classification, dataset) = loaddataset_fromurl(config, handle) #else: # dataset = loaddataset(handles) (geocoder, geolist, oecdlist) = buildgeocoder(dataset, config, '') #print geocoder (modern, historical) = loadgeocoder(config, dataset, 'geocoder') switch = 'historical' switch = 'modern' if switch == 'modern': activeindex = modern.index coder = modern class1 = switch
def __init__(self): Configuration.__init__(self) self.geocoderhandle = self.config['geocoderhandle'] (self.classification, self.geodataset, self.title, self.units) = content2dataframe(self.config, self.geocoderhandle)
def dataframe_compiler(config, fullpath, handle, switch, datafilter): handles = [] ctrlist = [] metadata = {} # switch = 'historical' # Load Geocoder (classification, dataset, title, units) = content2dataframe(config, config["geocoderhandle"]) (geocoder, geolist, oecd2webmapper) = buildgeocoder(dataset, config, "") (modern, historical) = loadgeocoder(config, dataset, "geocoder") coderyears = [] # Default years selection for i in range(1500, 2016): coderyears.append(i) # Years selected if datafilter["startyear"]: coderyears = [] for i in range(int(datafilter["startyear"]), int(datafilter["endyear"])): coderyears.append(i) # Reading dataset (class1, dataset, title, units) = content2dataframe(config, handle) filetitle = title filetitle = re.sub(" ", "_", filetitle) filetitle = re.sub(r"[\(\)]", "_", filetitle) if filetitle: fullpath = "%s/%s_%s.xlsx" % (fullpath, filetitle, switch) # return ('test', 'test') if switch == "modern": activeindex = modern.index coder = modern class1 = switch else: activeindex = historical.index coder = historical (moderndata, historicaldata) = loadgeocoder(config, dataset, "") if switch == "modern": maindata = moderndata else: # Do conversion to webmapper system if there are no historical data isdata = "" try: if not historicaldata.empty: isdata = "yes" except: isdata = "no" if isdata == "no": maindata = moderndata webmapperindex = [] for code in maindata.index: try: webmappercode = oecd2webmapper[int(code)] except: webmappercode = -1 webmapperindex.append(webmappercode) maindata.index = webmapperindex # Drop not recognized locations maindata = maindata[maindata.index > 0] else: maindata = historicaldata if title: metadata["title"] = title else: metadata["title"] = "" if units: metadata["units"] = units else: metadata["units"] = "" (cfilter, notint) = selectint(maindata.columns) codes = selectint(maindata.index) geo = load_geocodes(config, switch, codes, maindata, geolist) for colname in notint: maindata = maindata.drop(colname, axis=1) # Apply filters if datafilter: (datasubset, ctrlist) = datasetfilter(maindata, datafilter) else: datasetset = maindata # Create Excel file out from dataframe (yearscolumns, notyears) = selectint(maindata.columns) (countryinfo, notcountry) = selectint(maindata.index) (finalsubset, icoder, isyear, ctrfilter, nodata) = dataset_analyzer(datasubset, coder, yearscolumns) # return (finalsubset, fullpath, finalsubset) # Apply filter to countries if datafilter["ctrlist"]: tmpcoder = icoder.ix[ctrlist] icoder = pd.DataFrame(tmpcoder) # return (finalsubset.to_html(), 'test') if fullpath: config["emptyvalues"] = "no" if config["emptyvalues"] == "no": (coderyears, notyears) = selectint(finalsubset.columns) datafile = create_excel_dataset( fullpath, icoder, metadata, icoder.columns, coderyears, finalsubset, isyear, ctrfilter ) return (filetitle, fullpath, finalsubset)