Exemple #1
0
ctrlist = '380,250,52,850'
ctrlist = ''
result = ''
thisyear = 2010
ctrlimit = 10

modern = moderncodes(config['modernnames'], config['apiroot'])
dataframe = load_api_data(jsonapi, '')

if dataframe:
    allcodes = {}
    panel = []

    for dataitem in dataframe:
        handle = dataitem['handle']
        (dataset, codes) = paneldatafilter(dataitem['data'], int(yearmin),
                                           int(yearmax), ctrlist, handle)
        if not dataset.empty:
            panel.append(dataset)

    if panel:
        totalpanel = pd.concat(panel)
        cleanedpanel = totalpanel.dropna(axis=1, how='any')
        cleanedpanel = totalpanel

        (header, data, countries, handles, vhandles) = panel2dict(cleanedpanel)
        print data
        result = panel2csv(header, data, thisyear, countries, handles,
                           vhandles, ctrlimit, modern)
        print result
Exemple #2
0
def panel():
    (thisyear, datafilter, handle, yearmin, yearmax, thisyear, ctrlist,
     lastyear, logscale) = (0, {}, '', '1500', '2020', 1950, '', 2010, '')
    handles = []
    config = configuration()
    datafilter['startyear'] = yearmin
    datafilter['endyear'] = lastyear
    datafilter['ctrlist'] = config['ctrlist']

    #modern = moderncodes(config['modernnames'], config['apiroot'])
    if request.args.get('handle'):
        handle = str(request.args.get('handle'))
        handle = handle.replace(" ", "")
        handle = handle.replace("'", "")
        try:
            (pids, pidslist) = pidfrompanel(handle)
            handles = pids
        except:
            nopanel = 'yes'
            handles.append(handle)
    if request.args.get('face'):
        facehandle = request.args.get('face')
        if facehandle not in handles:
            handles.append(facehandle)
    if request.args.get('dataset'):
        dataset = request.args.get('dataset')
    if request.args.get('ctrlist'):
        customcountrycodes = ''
        ctrlist = request.args.get('ctrlist')
        datafilter['ctrlist'] = ctrlist
    if request.args.get('logscale'):
        logscale = request.args.get('logscale')
    if request.args.get('year'):
        thisyear = request.args.get('year')
        datafilter['startyear'] = int(thisyear)
        datafilter['endyear'] = int(thisyear)
    if request.args.get('yearmin'):
        fromyear = request.args.get('yearmin')
        datafilter['startyear'] = fromyear
    if request.args.get('yearmax'):
        toyear = request.args.get('yearmax')
        datafilter['endyear'] = toyear
    if request.args.get('hist'):
        switch = 'historical'
        if datafilter['ctrlist'] == '':
            datafilter['ctrlist'] = config['histctrlist']
    else:
        switch = 'modern'

    (geocoder, geolist, oecd2webmapper, modern,
     historical) = request_geocoder(config, '')
    (origdata, maindata, metadata) = request_datasets(config, switch, modern,
                                                      historical, handles,
                                                      geolist)
    (subsets, subsetyears, panel) = ({}, [], [])

    for handle in handles:
        (datasubset, ctrlist) = datasetfilter(maindata[handle], datafilter)
        datasubset['handle'] = handle
        if not datasubset.empty:
            datasubset = datasubset.dropna(how='all')
            try:
                if np.nan in datasubset.index:
                    datasubset = datasubset.drop(np.nan, axis=0)
            except:
                skip = 'yes'

            for year in datasubset:
                if datasubset[year].count() == 0:
                    datasubset = datasubset.drop(year, axis=1)

            (datayears, notyears) = selectint(datasubset.columns)
            panel.append(datasubset)
            subsets[handle] = datasubset
            subsetyears.append(datayears)

    dataframe = subsets
    ctrlimit = 10

    # Trying to find the best year with most filled data values
    try:
        bestyearlist = subsetyears[0]
        for i in range(1, len(subsetyears)):
            bestyearlist = list(set(bestyearlist) & set(subsetyears[i]))
#bestyearlist = bestyearlist.sort()
        thisyear = bestyearlist[0]
    except:
        bestyearlist = []

    allcodes = {}
    panel = []
    names = {}

    for handle in dataframe:
        try:
            names[handle] = metadata[handle]['title']
        except:
            names[handle] = 'title'
        try:
            #(dataset, codes) = paneldatafilter(dataframe[handle], int(yearmin), int(yearmax), ctrlist, handle)
            dataset = dataframe[handle]
            if not dataset.empty:
                panel.append(dataset)
        except:
            nodata = 0

    if panel:
        totalpanel = pd.concat(panel)
        cleanedpanel = totalpanel.dropna(axis=1, how='any')
        cleanedpanel = totalpanel

#return str(cleanedpanel.to_html())
    totalpanel = cleanedpanel
    if int(thisyear) <= 0:
        thisyear = totalpanel.columns[-2]
    result = ''
    original = {}
    if thisyear:
        if switch == 'historical':
            geocoder = historical
        if switch == 'hist':
            geocoder = historical
        else:
            geocoder = modern
        result = 'Country,'
        for handle in handles:
            result = result + str(metadata[handle]['title']) + ','
        result = result[:-1]

        known = {}
        for code in totalpanel.index:
            if str(code) not in known:
                result = result + '\n' + str(
                    geocoder.ix[int(code)][config['webmappercountry']])
                for handle in handles:
                    tmpframe = totalpanel.loc[totalpanel['handle'] == handle]
                    try:
                        (thisval,
                         original) = value2scale(tmpframe.ix[code][thisyear],
                                                 logscale, original)
                    except:
                        thisval = 'NaN'
                    result = result + ',' + str(thisval)
                known[str(code)] = code

        return Response(result, mimetype='text/plain')

        (allyears, notyears) = selectint(cleanedpanel.columns)
        (codes, notcodes) = selectint(cleanedpanel.index)
        cleanedpanel.index = codes
        (header, data, countries, handles,
         vhandles) = panel2dict(config, cleanedpanel, names)
        #return str(data)
        #thisyear = 1882
        #return str(countries)
        #return str(countries)
        years = []
        for year in sorted(data):
            try:
                years.append(int(year))
                lastyear = year
            except:
                skip = 1

        # Return only years
        if request.args.get('showyears'):
            yearsdata = {}
            yearsdata['years'] = years
            yearsdata['latestyear'] = lastyear
            #yearsdata['data'] = data
            yearsjson = json.dumps(yearsdata,
                                   ensure_ascii=False,
                                   sort_keys=True,
                                   indent=4)
            return Response(yearsjson, mimetype='application/json')

    return Response(result, mimetype='text/plain')
Exemple #3
0
def panel():
    (thisyear, datafilter, handle, yearmin, yearmax, thisyear, ctrlist, lastyear, logscale) = (0, {}, '', '1500', '2020', 1950, '', 2010, '')
    handles = []
    config = configuration()
    datafilter['startyear'] = yearmin
    datafilter['endyear'] = lastyear
    datafilter['ctrlist'] = config['ctrlist']

    #modern = moderncodes(config['modernnames'], config['apiroot'])
    if request.args.get('handle'):
        handle = str(request.args.get('handle'))
	handle = handle.replace(" ", "")
	handle = handle.replace("'", "")
        try:
            (pids, pidslist) = pidfrompanel(handle)
	    handles = pids
        except:
            nopanel = 'yes'
	    handles.append(handle)
    if request.args.get('face'):
	facehandle = request.args.get('face')
	if facehandle not in handles:
	    handles.append(facehandle)
    if request.args.get('dataset'):
        dataset = request.args.get('dataset')
    if request.args.get('ctrlist'):
        customcountrycodes = ''
        ctrlist = request.args.get('ctrlist')
	datafilter['ctrlist'] = ctrlist
    if request.args.get('logscale'):
        logscale = request.args.get('logscale')
    if request.args.get('year'):
        thisyear = request.args.get('year')
        datafilter['startyear'] = int(thisyear)
        datafilter['endyear'] = int(thisyear)
    if request.args.get('yearmin'):
        fromyear = request.args.get('yearmin')
        datafilter['startyear'] = fromyear
    if request.args.get('yearmax'):
        toyear = request.args.get('yearmax')
        datafilter['endyear'] = toyear
    if request.args.get('hist'):
	switch = 'historical'
	if datafilter['ctrlist'] == '':
	    datafilter['ctrlist'] = config['histctrlist']
    else:
	switch = 'modern'

    (geocoder, geolist, oecd2webmapper, modern, historical) = request_geocoder(config, '')
    (origdata, maindata, metadata) = request_datasets(config, switch, modern, historical, handles, geolist)
    (subsets, subsetyears, panel) = ({}, [], [])
    
    for handle in handles:
        (datasubset, ctrlist) = datasetfilter(maindata[handle], datafilter)
        datasubset['handle'] = handle
        if not datasubset.empty:
            datasubset = datasubset.dropna(how='all')
	    try:
	        if np.nan in datasubset.index:
	            datasubset = datasubset.drop(np.nan, axis=0)
	    except:
		skip = 'yes'

            for year in datasubset:
                if datasubset[year].count() == 0:
                    datasubset = datasubset.drop(year, axis=1)

	    (datayears, notyears) = selectint(datasubset.columns)
            panel.append(datasubset)
            subsets[handle] = datasubset    
	    subsetyears.append(datayears)

    dataframe = subsets
    ctrlimit = 10

    # Trying to find the best year with most filled data values
    try:
        bestyearlist = subsetyears[0]
        for i in range(1,len(subsetyears)):
	    bestyearlist = list(set(bestyearlist) & set(subsetyears[i]))
	#bestyearlist = bestyearlist.sort()
	thisyear = bestyearlist[0]
    except:
	bestyearlist = []

    allcodes = {}
    panel = []
    names = {}

    for handle in dataframe:
	try:
	    names[handle] = metadata[handle]['title']
	except:
	    names[handle] = 'title'
	try:
            #(dataset, codes) = paneldatafilter(dataframe[handle], int(yearmin), int(yearmax), ctrlist, handle)
	    dataset = dataframe[handle]
	    if not dataset.empty:
                panel.append(dataset)
	except:
	    nodata = 0
	
    if panel:
        totalpanel = pd.concat(panel)
        cleanedpanel = totalpanel.dropna(axis=1, how='any')
        cleanedpanel = totalpanel

	#return str(cleanedpanel.to_html())
    totalpanel = cleanedpanel
    if int(thisyear) <= 0:
        thisyear = totalpanel.columns[-2]
    result = ''
    original = {}
    if thisyear:
	if switch == 'historical':
	    geocoder = historical
	if switch == 'hist':
	    geocoder = historical
	else:
	    geocoder = modern
        result = 'Country,'
        for handle in handles:
            result = result + str(metadata[handle]['title']) + ','
	result = result[:-1]
    
        known = {}
        for code in totalpanel.index:
            if str(code) not in known:
                result = result + '\n' + str(geocoder.ix[int(code)][config['webmappercountry']])
                for handle in handles:
                    tmpframe = totalpanel.loc[totalpanel['handle'] == handle]
		    try:
                        (thisval, original) = value2scale(tmpframe.ix[code][thisyear], logscale, original)
		    except:
			thisval = 'NaN'
                    result = result + ',' + str(thisval)
                known[str(code)] = code

	return Response(result,  mimetype='text/plain')

	(allyears, notyears) = selectint(cleanedpanel.columns)
	(codes, notcodes) = selectint(cleanedpanel.index)
	cleanedpanel.index = codes
        (header, data, countries, handles, vhandles) = panel2dict(config, cleanedpanel, names)  
	#return str(data)
	#thisyear = 1882
	#return str(countries)
	#return str(countries)
	years = []
	for year in sorted(data):
            try:
        	years.append(int(year))
		lastyear = year
    	    except:
        	skip = 1

	# Return only years
	if request.args.get('showyears'):
	    yearsdata = {}
	    yearsdata['years'] = years
	    yearsdata['latestyear'] = lastyear
	    #yearsdata['data'] = data
	    yearsjson = json.dumps(yearsdata, ensure_ascii=False, sort_keys=True, indent=4)
	    return Response(yearsjson,  mimetype='application/json')

    return Response(result,  mimetype='text/plain')
yearmax = '2015'
x = ''
ctrlist = '380,250,52,850'
ctrlist = ''
result = ''
thisyear = 2010
ctrlimit = 10

modern = moderncodes(config['modernnames'], config['apiroot'])
dataframe = load_api_data(jsonapi, '')

if dataframe:
    allcodes = {}
    panel = []

    for dataitem in dataframe: 
        handle = dataitem['handle']        
        (dataset, codes) = paneldatafilter(dataitem['data'], int(yearmin), int(yearmax), ctrlist, handle)    
        if not dataset.empty:                
            panel.append(dataset)
        
    if panel:
        totalpanel = pd.concat(panel)
        cleanedpanel = totalpanel.dropna(axis=1, how='any')
        cleanedpanel = totalpanel

        (header, data, countries, handles, vhandles) = panel2dict(cleanedpanel)           
	print data
        result = panel2csv(header, data, thisyear, countries, handles, vhandles, ctrlimit, modern)
        print result