def printme(): config = configuration() if config['error']: return config['error'] year = request.args.get("year") handle = request.args.get("handle") handles = [] handles.append(handle) hquery = formdatasetquery(handles,'') datainfo = readdatasets('datasets', json.loads(hquery)) try: for item in datainfo: datasetID = item['datasetID'] except: datasetID = 228 root = config['dataverseroot'] + "/api/datasets/" + str(datasetID) + "/versions/?key=" + config['key'] + "&show_entity_ids=true&q=authorName:*" data = load_api_data(root, 1) (title, citation) = get_citation(data['data']) uhandle = handle uhandle = uhandle.replace('hdl:', '') mapcopyright = config['cshapes_copyright'] if int(year) < 1946: mapcopyright = config['geacron_copyright'] # "Note: Map polygons provided by Geacron <a href=\"geacron.com\">http://www.geacron.com</a>" resp = make_response(render_template('printall.html', title=title, citation=citation, mapcopyright=mapcopyright, year=year, handle=handle, uhandle=uhandle)) return resp
def loaddataset(config, handle): handles = [] handles.append(handle) hquery = formdatasetquery(handles, '') d = readdatasets('datasets', json.loads(hquery)) alldata = '' for dataset in d: alldata = str(dataset['csvframe']) thishandle = dataset['handle'] units = dataset['units'] title = dataset['title'] csvio = StringIO(str(alldata)) dataframe = pd.read_csv(csvio, sep='\t', dtype='unicode') maincode = 'Code' #df = adjustdataframe(dataframe) #df.index = df[maincode] #print df.index return (dataframe, handle, title, units)
def loaddataset(config, handle): handles = [] handles.append(handle) hquery = formdatasetquery(handles, "") d = readdatasets("datasets", json.loads(hquery)) alldata = "" for dataset in d: alldata = str(dataset["csvframe"]) thishandle = dataset["handle"] units = dataset["units"] title = dataset["title"] csvio = StringIO(str(alldata)) dataframe = pd.read_csv(csvio, sep="\t", dtype="unicode") maincode = "Code" # df = adjustdataframe(dataframe) # df.index = df[maincode] # print df.index return (dataframe, handle, title, units)
for i in item: print i handles = [] handlequery = '' if filename: handle = "hdl:10622/4X6NCK" handles.append(handle) # handles.append(str(handle)) handle = "hdl:10622/I0YK5M" #handle = "'hdl:10622/LFQJEM" dataset = 'I0YK5M' handle = "hdl:10622/CSNWOF" handles.append(handle) hquery = formdatasetquery(handles,'') #,'{ title: 1, units: 1}') print hquery x = json.loads(hquery) #x = simplejson.load(hquery) #d = readdatasets('datasets', json.loads(hquery)) print hquery d = readdatasets('datasets', json.loads(hquery)) for x in d: print x['title'] for i in x: print 'F' + str(x[i]) for dataset in d: print dataset['data'] #$datasetadd(filename, handle, dataset)
from storage import data2store, readdata, readdataset, readdatasets, datasetadd, formdatasetquery from pandas.io.json import json_normalize import re import openpyxl from openpyxl.cell import get_column_letter import requests from StringIO import StringIO url = "http://dpe.sandbox.socialhistoryservices.org/api/datasets?handle=Panel[%27hdl:10622/MRPFJ0:243:244%27]&format=csv" onlinetest = '' if onlinetest: res = requests.get(url) csvio = StringIO(res.content) dataframe = pd.read_csv(csvio, sep='\t', dtype='unicode') print dataframe.to_html else: handles = [] handle = "hdl:10622/MRPFJ0" handle = "hdl:10622/DIUBXI" handles.append(handle) hquery = formdatasetquery(handles,'') print hquery d = readdatasets('datasets', json.loads(hquery)) for x in d: csvframe = StringIO(x['csvframe']) dataframe = pd.read_csv(csvframe, sep='\t', dtype='unicode') print dataframe.to_html
def chartlib(): (thismapurl, apilink, ctrlist, title, units, switch, hist) = ('', '', '', 'Title', 'Units', 'modern', '') handleface = [] config = configuration() ctrlist = config['ctrlist'] if config['error']: return config['error'] urlmatch = re.search(r'(.+)\&face', request.url) try: if urlmatch.group(0): thismapurl = urlmatch.group(1) except: thismapurl = request.url if 'sandbox' not in thismapurl: thismapurl = thismapurl.replace('http://', 'https://') handles = [] showpanel = 'yes' try: if request.args.get('print'): showpanel = '' except: showpanel = 'yes' f = request.args handle = '' for q in f: value = f[q] if value: handle = str(handle) + '&' + str(q) + '=' + str(f[q]) if request.args.get('ctrlist'): ctrlist = request.args.get('ctrlist') if request.args.get('hist'): switch = 'historical' hist = request.args.get('hist') if request.args.get('handle'): handledataset = request.args.get('handle') try: (pids, pidslist) = pidfrompanel(handledataset) handles.append(pids[0]) except: handles.append(handledataset) nopanel = 'yes' if pids[0]: apilink = "/api/tabledata?handle=" + str(pids[0]) if ctrlist: apilink = apilink + '&ctrlist=' + ctrlist if request.args.get('hist'): apilink = apilink + '&hist=' + hist if request.args.get('face'): handles = [] handleface = request.args.get('face') handles.append(handleface) if handleface: apilink = "/api/tabledata?handle=" + str(handleface) if ctrlist: apilink = apilink + '&ctrlist=' + ctrlist if request.args.get('hist'): apilink = apilink + '&hist=' + hist try: pids.remove(handleface) except: nothing = 1 if set(pids) == set(handles): pids[:] = [] links = graphlinks('&face=' + str(handles[0]) + '&hist=' + hist) (geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, '') # vty hist (origdata, maindata, metadata) = request_datasets(config, switch, modern, historical, handles, geolist) try: title = metadata[handles[0]]['title'] units = metadata[handles[0]]['units'] except: skip = 0 handledict = {} if handles: handle = handles[0] if pids: try: if handles[1]: pids.remove(handles[0]) except: skip = 1 hquery = formdatasetquery(pids,'') d = readdatasets('datasets', json.loads(hquery)) for x in d: thishandle = x['handle'] if thishandle != handle: handledict[thishandle] = x['title'] resp = make_response(render_template('chartlib.html', thismapurl=thismapurl, indicators=handledict, apilink=apilink, title=title, units=units, showpanel=showpanel, handle=handle, chartlib=links['chartlib'], barlib=links['barlib'], panellib=links['panellib'], treemaplib=links['treemaplib'])) return resp
def mapslider(): (title, steps, customcountrycodes, fromyear, toyear, customyear, catmax, histo) = ('', 0, '', '1500', '2012', '', 6, '') config = configuration() datafilter = {} datafilter['ctrlist'] = '' datafilter['startyear'] = fromyear datafilter['endyear'] = toyear if config['error']: return config['error'] handleface = '' urlmatch = re.search(r'(.+)\&face', request.url) try: if urlmatch.group(0): thismapurl = urlmatch.group(1) except: thismapurl = request.url thismapurl = thismapurl.replace('http://', 'https://') geocoder = '' pids = [] handledataset = '' logscale = 0 handles = [] datahub = {} dataset = '' warning = '' hist = {} if request.args.get('ctrlist'): customcountrycodes = '' tmpcustomcountrycodes = request.args.get('ctrlist') c = tmpcustomcountrycodes.split(',') for ids in sorted(c): if ids: customcountrycodes = str(customcountrycodes) + str(ids) + ',' customcountrycodes = customcountrycodes[:-1] datafilter['ctrlist'] = customcountrycodes if request.args.get('dataset'): dataset = request.args.get('dataset') handles.append(dataset) if request.args.get('handle'): handledataset = request.args.get('handle') try: (pids, pidslist) = pidfrompanel(handledataset) except: nopanel = 'yes' handlestring = request.args.get('handle') ishandle = re.search(r'(hdl:\d+\/\w+)', handlestring) if ishandle: handle = ishandle.group(1) handle = handle.replace("'", "") else: handle = handlestring (dataset, revid, cliopid, clearpid) = findpid(handle) #handles.append(dataset) handles.append(handle) handleface = handle if request.args.get('logscale'): logscale = 1 if request.args.get('catmax'): catmax = request.args.get('catmax') if request.args.get('yearmin'): fromyear = request.args.get('yearmin') datafilter['startyear'] = fromyear if request.args.get('yearmax'): toyear = request.args.get('yearmax') datafilter['endyear'] = toyear if request.args.get('geocoder'): geocoder = request.args.get('geocoder') if request.args.get('hist'): geocoder = request.args.get('hist') histo = 'on' if request.args.get('face'): handleface = request.args.get('face') if handleface: handles = [] handle = handleface handles.append(handleface) try: pids.remove(handleface) except: nothing = 1 historical = 0 hubyears = [] if config: switch = 'modern' if histo: switch = 'historical' (geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, '') (origdata, maindata, metadata) = request_datasets(config, switch, modern, historical, handles, geolist) (hubyears, notyears) = selectint(origdata.columns) title = metadata[handles[0]]['title'] for handle in handles: (datasubset, ctrlist) = datasetfilter(maindata[handle], datafilter) datasubset['handle'] = handle if not datasubset.empty: datasubset = datasubset.dropna(how='all') (allyears, notyears) = selectint(datasubset.columns) for year in datasubset: if datasubset[year].count() == 0: datasubset = datasubset.drop(year, axis=1) (hubyears, notyears) = selectint(datasubset.columns) validyears = [] lastyear = '' for year in sorted(hubyears): validyears.append(year) lastyear = year steps = steps + 1 handledict = {} if pids: hquery = formdatasetquery(pids,'') d = readdatasets('datasets', json.loads(hquery)) for x in d: thishandle = x['handle'] handledict[thishandle] = x['title'] #validyears.reverse() return make_response(render_template('mapslider.html', handle=handle, years=validyears, warning=warning, steps=steps, title=title, geocoder=histo, dataset=dataset, customcountrycodes=customcountrycodes, catmax=catmax, lastyear=lastyear, indicators=pids, thismapurl=thismapurl, handledict=handledict))