def geocoder():
config = configuration()
config['remote'] = ''
remote = 'on'
# Geocoder
handle = config['geocoderhandle']
(classification, geodataset, title, units) = content2dataframe(config, config['geocoderhandle'])
fromyear = 1500
toyear = 2016
cfilter = ''
if request.args.get('name'):
cfilter = request.args.get('name')
if request.args.get('name'):
cfilter = request.args.get('name')
if fromyear:
historical = ''
if historical == 'old':
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api, cfilter)
else:
(geocoder, geolist, oecd) = buildgeocoder(geodataset, config, cfilter)
data = json.dumps(geocoder, encoding="utf-8", sort_keys=True, indent=4)
return Response(data, mimetype='application/json')
def geocoder():
config = configuration()
config['remote'] = ''
remote = 'on'
# Geocoder
handle = config['geocoderhandle']
(classification, geodataset, title,
units) = content2dataframe(config, config['geocoderhandle'])
fromyear = 1500
toyear = 2016
cfilter = ''
if request.args.get('name'):
cfilter = request.args.get('name')
if request.args.get('name'):
cfilter = request.args.get('name')
if fromyear:
historical = ''
if historical == 'old':
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper,
geocoder) = histo(api, cfilter)
else:
(geocoder, geolist, oecd) = buildgeocoder(geodataset, config,
cfilter)
data = json.dumps(geocoder, encoding="utf-8", sort_keys=True, indent=4)
return Response(data, mimetype='application/json')
def webmapper():
config = configuration()
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api)
data = json.dumps(webmapper, encoding="utf-8", sort_keys=True, indent=4)
return Response(data, mimetype='application/json')
def webmapper():
config = configuration()
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api)
data = json.dumps(webmapper, encoding="utf-8", sort_keys=True, indent=4)
return Response(data, mimetype='application/json')
def downloadzip(pid):
DEBUG = 0
(fullpath) = ('')
fullmetadata = {}
logscale = 0
config = configuration()
config['remote'] = 'on'
API_TOKEN = config['key']
HOSTNAME = config['dataverseroot']
cmd = "--insecure -u " + API_TOKEN + ": " + HOSTNAME + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
tmpdir = config['tmpdir']
filerandom = randomword(10)
#filerandom = '12345'
arc = "data" + filerandom + ".zip"
filename = filerandom
finaldir = config['path'] + '/static/tmp'
# ToDO
if filename:
finaldir = str(finaldir) + '/' + str(filename)
tmpdir = str(tmpdir) + '/' + str(filename)
try:
os.mkdir(tmpdir)
os.mkdir(finaldir)
except:
donothing = 'ok'
customyear = ''
fromyear = request.args.get('y[min]')
toyear = request.args.get('y[max]')
historical = request.args.get('type[0]')
(handles, pidslist) = pidfrompanel(pid)
try:
if pidslist:
fullmetadata = load_fullmetadata(pidslist)
except:
showwarning = 1
# Log scales switch
if request.args.get('logscale'):
logscale = 1
# Select countries
customcountrycodes = ''
f = request.args
for key in f.keys():
if is_location(key):
for value in sorted(f.getlist(key)):
customcountrycodes = str(customcountrycodes) + str(value) + ','
if customcountrycodes:
customcountrycodes = customcountrycodes[:-1]
if handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
hist = countries
else:
hist = ''
(classification, geodataset, title, units) = content2dataframe(config, config['geocoderhandle'])
#geocoder = buildgeocoder(dataset, config)
(modern, historical) = loadgeocoder(config, dataset, 'geocoder')
for handle in handles:
#if remote:
# (class1, dataset) = loaddataset_fromurl(config, handle)
#else:
# dataset = loaddataset(handles)
#(cfilter, notint) = selectint(activeindex.values)
#(moderndata, historicaldata) = loadgeocoder(dataset, '')
# CHANGE
#return str(dataset.index)
(header, panelcells, codes, datahub, data, handle2ind, unit2ind, original) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale)
filename = filename + '.xls'
fullpath = panel2excel(finaldir, filename, header, panelcells, fullmetadata)
else:
# Clio format download
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, pid, tmpdir, arc, finaldir)
(alonepid, revid, cliohandle, clearpid) = findpid(pid)
if alonepid:
handles = [ clearpid ]
for pid in handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
hist = countries
else:
hist = ''
filename = filename + '.xls'
# 2DEBUG
(header, panelcells, codes, datahub, data, handle2ind, unit2ind, originalvalues) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale)
#codes = hist
#return str(fullmetadata)
metadata = fullmetadata
result = individual_dataset(finaldir, filename, handle2ind[pid], unit2ind[pid], datahub, data[pid], codes, metadata)
try:
for everypid in handles:
# Download papers
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, everypid, tmpdir, arc, finaldir)
except:
nopapers = 1
compile2zip(finaldir, arc)
filename = arc
return filename
import os
import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../modules')))
from tabulardata import loadcodes, load_api_data, countryset, json_dict, createframe, combinedata, data2panel
from config import configuration, dataverse2indicators, load_dataverse, findpid, load_metadata
from historical import load_historical, histo
from data2excel import panel2excel, individual_dataset
hist = {}
handles = ["4X6NCK", "F16UDU","ZWRBOY"]
customyear = ''
fromyear = '1500'
toyear = '2000'
customcountrycodes = '380,250,276,804,174,108,232,528,756'
handles = ["GDAO0N"]
handles = ["4X6NCK"]
years = 0
logflag = 0
config = configuration()
if fromyear:
historical = 0
if historical:
api = "http://clearance.sandbox.socialhistoryservices.org/collabs/static/data/historical.json"
(regions, countries, ctr2reg) = histo(api)
hist = countries
(header, panelcells, codes, datahub, data, handle2ind, unit2ind) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logflag)
for year in sorted(datahub):
print year
hist = {}
handles = ["4X6NCK", "F16UDU","ZWRBOY"]
customyear = ''
fromyear = '1500'
toyear = '2000'
customcountrycodes = '380,250,276,804,174,108,232,528,756'
#customcountrycodes = '528,756'
handles = ["hdl:10622/GDAO0N"]
logflag = 0
config = configuration()
if fromyear:
historical = 0
if historical:
api = config['clearance'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geo) = histo(api, '')
hist = countries
(header, panelcells, codes, datahub, data, handle2ind, unit2ind, original) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logflag)
datafile = "panel.xlsx"
#(handles, pidslist) = pidfrompanel(pid)
pidslist = handles
try:
if pidslist:
fullmetadata = load_fullmetadata(pidslist)
except:
fullmetadata = ''
# Create panel
panel2excel(config['webtest'], datafile, header, panelcells, fullmetadata)
# Create individual files
def downloadzip(pid):
DEBUG = 0
(fullpath) = ('')
fullmetadata = {}
logscale = 0
config = configuration()
config['remote'] = 'on'
API_TOKEN = config['key']
HOSTNAME = config['dataverseroot']
cmd = "--insecure -u " + API_TOKEN + ": " + HOSTNAME + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
tmpdir = config['tmpdir']
filerandom = randomword(10)
#filerandom = '12345'
arc = "data" + filerandom + ".zip"
filename = filerandom
finaldir = config['path'] + '/static/tmp'
# ToDO
if filename:
finaldir = str(finaldir) + '/' + str(filename)
tmpdir = str(tmpdir) + '/' + str(filename)
try:
os.mkdir(tmpdir)
os.mkdir(finaldir)
except:
donothing = 'ok'
customyear = ''
fromyear = request.args.get('y[min]')
toyear = request.args.get('y[max]')
historical = request.args.get('type[0]')
(handles, pidslist) = pidfrompanel(pid)
try:
if pidslist:
fullmetadata = load_fullmetadata(pidslist)
except:
showwarning = 1
# Log scales switch
if request.args.get('logscale'):
logscale = 1
# Select countries
customcountrycodes = ''
f = request.args
for key in f.keys():
if is_location(key):
for value in sorted(f.getlist(key)):
customcountrycodes = str(customcountrycodes) + str(value) + ','
if customcountrycodes:
customcountrycodes = customcountrycodes[:-1]
if handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
hist = countries
else:
hist = ''
(classification, geodataset, title,
units) = content2dataframe(config, config['geocoderhandle'])
#geocoder = buildgeocoder(dataset, config)
(modern, historical) = loadgeocoder(config, dataset, 'geocoder')
for handle in handles:
#if remote:
# (class1, dataset) = loaddataset_fromurl(config, handle)
#else:
# dataset = loaddataset(handles)
#(cfilter, notint) = selectint(activeindex.values)
#(moderndata, historicaldata) = loadgeocoder(dataset, '')
# CHANGE
#return str(dataset.index)
(header, panelcells, codes, datahub, data, handle2ind, unit2ind,
original) = data2panel(handles, customcountrycodes, fromyear, toyear,
customyear, hist, logscale)
filename = filename + '.xls'
fullpath = panel2excel(finaldir, filename, header, panelcells,
fullmetadata)
else:
# Clio format download
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, pid, tmpdir, arc,
finaldir)
(alonepid, revid, cliohandle, clearpid) = findpid(pid)
if alonepid:
handles = [clearpid]
for pid in handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper,
geocoder) = histo(api, '')
hist = countries
else:
hist = ''
filename = filename + '.xls'
# 2DEBUG
(header, panelcells, codes, datahub, data, handle2ind, unit2ind,
originalvalues) = data2panel(handles, customcountrycodes,
fromyear, toyear, customyear, hist,
logscale)
#codes = hist
#return str(fullmetadata)
metadata = fullmetadata
result = individual_dataset(finaldir, filename, handle2ind[pid],
unit2ind[pid], datahub, data[pid],
codes, metadata)
try:
for everypid in handles:
# Download papers
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, everypid, tmpdir,
arc, finaldir)
except:
nopapers = 1
compile2zip(finaldir, arc)
filename = arc
return filename
sys.path.append(
os.path.abspath(os.path.join(os.path.dirname("__file__"), '../modules')))
from tabulardata import loadcodes, load_api_data, countryset, json_dict, createframe, combinedata, data2panel
from config import configuration, dataverse2indicators, load_dataverse, findpid, load_metadata
from historical import load_historical, histo
hist = {}
# Test case
handles = ["4X6NCK", "F16UDU", "ZWRBOY"]
customyear = ''
fromyear = '1500'
toyear = '2000'
customcountrycodes = '380,250,276,804,174,108,232,528,756'
logflag = 1
config = configuration()
if fromyear:
historical = 1
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg) = histo(api)
hist = countries
(header, panelcells, codes, x1, x2, x3,
x4) = data2panel(handles, customcountrycodes, fromyear, toyear,
customyear, hist, logflag)
for item in panelcells:
print item
sys.path.append(os.path.abspath(os.path.join(os.path.dirname("__file__"), '../modules')))
from tabulardata import loadcodes, load_api_data, countryset, json_dict, createframe, combinedata, data2panel, moderncodes, data2json
from config import configuration, dataverse2indicators, load_dataverse, findpid, load_metadata
from historical import load_historical, histo
hist = {}
# Test case
handles = ["4X6NCK", "F16UDU","ZWRBOY"]
handles = ["4X6NCK"]
customyear = '1901'
fromyear = '1500'
toyear = '2000'
customcountrycodes = '380,250,276,804,174,108,232,528,756'
logflag = 0
DEBUG = 0
config = configuration()
cfilter = {}
if fromyear:
historical = 1
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
cfilter['query'] = 'ukr'
cfilter['startyear'] = 1900
cfilter['endyear'] = 2012
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api, cfilter)
hist = countries
print geocoder
hist = {}
handles = ["4X6NCK", "F16UDU", "ZWRBOY"]
customyear = ''
fromyear = '1500'
toyear = '2000'
customcountrycodes = '380,250,276,804,174,108,232,528,756'
#customcountrycodes = '528,756'
handles = ["hdl:10622/GDAO0N"]
logflag = 0
config = configuration()
if fromyear:
historical = 0
if historical:
api = config['clearance'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geo) = histo(api, '')
hist = countries
(header, panelcells, codes, datahub, data, handle2ind, unit2ind,
original) = data2panel(handles, customcountrycodes, fromyear, toyear,
customyear, hist, logflag)
datafile = "panel.xlsx"
#(handles, pidslist) = pidfrompanel(pid)
pidslist = handles
try:
if pidslist:
fullmetadata = load_fullmetadata(pidslist)
except:
fullmetadata = ''
# Create panel
panel2excel(config['webtest'], datafile, header, panelcells, fullmetadata)
from tabulardata import loadcodes, load_api_data, countryset, json_dict, createframe, combinedata, data2panel, moderncodes, data2json
from config import configuration, dataverse2indicators, load_dataverse, findpid, load_metadata
from historical import load_historical, histo
hist = {}
# Test case
handles = ["4X6NCK", "F16UDU", "ZWRBOY"]
handles = ["4X6NCK"]
customyear = '1901'
fromyear = '1500'
toyear = '2000'
customcountrycodes = '380,250,276,804,174,108,232,528,756'
logflag = 0
DEBUG = 0
config = configuration()
cfilter = {}
if fromyear:
historical = 1
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
cfilter['query'] = 'ukr'
cfilter['startyear'] = 1900
cfilter['endyear'] = 2012
(regions, countries, ctr2reg, webmapper,
geocoder) = histo(api, cfilter)
hist = countries
print geocoder