def build_panel(config, switch, handles, datafilter): (geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, '') (origdata, maindata, metadata) = request_datasets(config, switch, modern, historical, handles, geolist) (subsets, panel) = ({}, []) logscale = '' for handle in handles: (datasubset, ctrlist) = datasetfilter(maindata[handle], datafilter) if not datasubset.empty: datasubset = datasubset.dropna(how='all') try: if np.nan in datasubset.index: datasubset = datasubset.drop(np.nan, axis=0) except: skip = 'yes' datasubset['handle'] = handle metadata['url'] = 0 panel.append(datasubset) subsets[handle] = datasubset (panelcells, originalvalues) = dataset2panel(config, subsets[handle], historical, logscale) totalpanel = pd.concat(panel) try: if np.nan in totalpanel.index: totalpanel = totalpanel.drop(np.nan, axis=0) except: skip = 'yes' if switch == 'historical': geocoder = historical else: geocoder = modern # Remove columns with empty years for colyear in totalpanel.columns: if totalpanel[colyear].count() == 0: totalpanel = totalpanel.drop(colyear, axis=1) (allyears, nyears) = selectint(totalpanel.columns) print totalpanel.index panels = [] known = {} matrix = {} #return (str(totalpanel.to_html()), '', '', '') for code in totalpanel.index: try: country = geocoder.ix[int(code)][config['webmappercountry']] except: country = '' for thisyear in allyears: thiskey = str(int(code)) + str(thisyear) if thiskey not in known: dataitem = [country] dataitem.append(thisyear) known[thiskey] = thisyear matrix[thiskey] = '' for handle in handles: tmpframe = totalpanel.loc[totalpanel['handle'] == handle] try: thisval = tmpframe.ix[int(code)][thisyear] matrix[thiskey] = thisval except: thisval = '' dataitem.append(thisval) # Filter out np.NaN if str(thisval) != 'nan': if country: if matrix[thiskey]: panels.append(dataitem) # Build header header = ['Country', 'Year'] for handle in handles: header.append(metadata[handle]['title']) return (header, panels, metadata, totalpanel)
csvio = StringIO(content) dataframe = pd.read_csv(csvio, sep='\t', dtype='unicode') return dataframe fullpath = "/home/tikhonov/completedatasets14Sept2015/GDPperCapita.xlsx" allfiles = ["/home/tikhonov/completedatasets14Sept2015/GDPperCapita.xlsx", "/home/tikhonov/completedatasets14Sept2015/TotalCO2Emissions.xlsx"] outdir = "/home/tikhonov/data/historical.all" path = "/home/tikhonov/tmp" mypath = "/home/tikhonov/data/clio2/completedatainclKeesJonat1Nov2015" onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))] fileID = "test" config = configuration() config['remote'] = 'yes' (geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, 'geocoder') def compiledataset(csvfile): handles = [] remote = 'on' # Geocoder config = configuration() config['remote'] = 'yes' dataframe = loaddataset_fromfile(config, csvfile) dataset = dataframe title = dataframe.columns[1] units = dataframe.ix[0][1] metadata = {} switch = 'modern' switch = 'historical' #dataset = dataset.convert_objects(convert_numeric=True)
fullpath = "/home/tikhonov/completedatasets14Sept2015/GDPperCapita.xlsx" allfiles = [ "/home/tikhonov/completedatasets14Sept2015/GDPperCapita.xlsx", "/home/tikhonov/completedatasets14Sept2015/TotalCO2Emissions.xlsx" ] outdir = "/home/tikhonov/data/historical.all" path = "/home/tikhonov/tmp" mypath = "/home/tikhonov/data/clio2/completedatainclKeesJonat1Nov2015" onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))] fileID = "test" config = configuration() config['remote'] = 'yes' (geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, 'geocoder') def compiledataset(csvfile): handles = [] remote = 'on' # Geocoder config = configuration() config['remote'] = 'yes' dataframe = loaddataset_fromfile(config, csvfile) dataset = dataframe title = dataframe.columns[1] units = dataframe.ix[0][1] metadata = {} switch = 'modern'