def downloadzip(pid):
DEBUG = 0
(fullpath) = ('')
fullmetadata = {}
logscale = 0
config = configuration()
config['remote'] = 'on'
API_TOKEN = config['key']
HOSTNAME = config['dataverseroot']
cmd = "--insecure -u " + API_TOKEN + ": " + HOSTNAME + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
tmpdir = config['tmpdir']
filerandom = randomword(10)
#filerandom = '12345'
arc = "data" + filerandom + ".zip"
filename = filerandom
finaldir = config['path'] + '/static/tmp'
# ToDO
if filename:
finaldir = str(finaldir) + '/' + str(filename)
tmpdir = str(tmpdir) + '/' + str(filename)
try:
os.mkdir(tmpdir)
os.mkdir(finaldir)
except:
donothing = 'ok'
customyear = ''
fromyear = request.args.get('y[min]')
toyear = request.args.get('y[max]')
historical = request.args.get('type[0]')
(handles, pidslist) = pidfrompanel(pid)
try:
if pidslist:
fullmetadata = load_fullmetadata(pidslist)
except:
showwarning = 1
# Log scales switch
if request.args.get('logscale'):
logscale = 1
# Select countries
customcountrycodes = ''
f = request.args
for key in f.keys():
if is_location(key):
for value in sorted(f.getlist(key)):
customcountrycodes = str(customcountrycodes) + str(value) + ','
if customcountrycodes:
customcountrycodes = customcountrycodes[:-1]
if handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
hist = countries
else:
hist = ''
(classification, geodataset, title, units) = content2dataframe(config, config['geocoderhandle'])
#geocoder = buildgeocoder(dataset, config)
(modern, historical) = loadgeocoder(config, dataset, 'geocoder')
for handle in handles:
#if remote:
# (class1, dataset) = loaddataset_fromurl(config, handle)
#else:
# dataset = loaddataset(handles)
#(cfilter, notint) = selectint(activeindex.values)
#(moderndata, historicaldata) = loadgeocoder(dataset, '')
# CHANGE
#return str(dataset.index)
(header, panelcells, codes, datahub, data, handle2ind, unit2ind, original) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale)
filename = filename + '.xls'
fullpath = panel2excel(finaldir, filename, header, panelcells, fullmetadata)
else:
# Clio format download
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, pid, tmpdir, arc, finaldir)
(alonepid, revid, cliohandle, clearpid) = findpid(pid)
if alonepid:
handles = [ clearpid ]
for pid in handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
hist = countries
else:
hist = ''
filename = filename + '.xls'
# 2DEBUG
(header, panelcells, codes, datahub, data, handle2ind, unit2ind, originalvalues) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale)
#codes = hist
#return str(fullmetadata)
metadata = fullmetadata
result = individual_dataset(finaldir, filename, handle2ind[pid], unit2ind[pid], datahub, data[pid], codes, metadata)
try:
for everypid in handles:
# Download papers
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, everypid, tmpdir, arc, finaldir)
except:
nopapers = 1
compile2zip(finaldir, arc)
filename = arc
return filename
def downloadzip(pid):
DEBUG = 0
(fullpath) = ('')
fullmetadata = {}
logscale = 0
config = configuration()
config['remote'] = 'on'
API_TOKEN = config['key']
HOSTNAME = config['dataverseroot']
cmd = "--insecure -u " + API_TOKEN + ": " + HOSTNAME + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
tmpdir = config['tmpdir']
filerandom = randomword(10)
#filerandom = '12345'
arc = "data" + filerandom + ".zip"
filename = filerandom
finaldir = config['path'] + '/static/tmp'
# ToDO
if filename:
finaldir = str(finaldir) + '/' + str(filename)
tmpdir = str(tmpdir) + '/' + str(filename)
try:
os.mkdir(tmpdir)
os.mkdir(finaldir)
except:
donothing = 'ok'
customyear = ''
fromyear = request.args.get('y[min]')
toyear = request.args.get('y[max]')
historical = request.args.get('type[0]')
(handles, pidslist) = pidfrompanel(pid)
try:
if pidslist:
fullmetadata = load_fullmetadata(pidslist)
except:
showwarning = 1
# Log scales switch
if request.args.get('logscale'):
logscale = 1
# Select countries
customcountrycodes = ''
f = request.args
for key in f.keys():
if is_location(key):
for value in sorted(f.getlist(key)):
customcountrycodes = str(customcountrycodes) + str(value) + ','
if customcountrycodes:
customcountrycodes = customcountrycodes[:-1]
if handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
hist = countries
else:
hist = ''
(classification, geodataset, title,
units) = content2dataframe(config, config['geocoderhandle'])
#geocoder = buildgeocoder(dataset, config)
(modern, historical) = loadgeocoder(config, dataset, 'geocoder')
for handle in handles:
#if remote:
# (class1, dataset) = loaddataset_fromurl(config, handle)
#else:
# dataset = loaddataset(handles)
#(cfilter, notint) = selectint(activeindex.values)
#(moderndata, historicaldata) = loadgeocoder(dataset, '')
# CHANGE
#return str(dataset.index)
(header, panelcells, codes, datahub, data, handle2ind, unit2ind,
original) = data2panel(handles, customcountrycodes, fromyear, toyear,
customyear, hist, logscale)
filename = filename + '.xls'
fullpath = panel2excel(finaldir, filename, header, panelcells,
fullmetadata)
else:
# Clio format download
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, pid, tmpdir, arc,
finaldir)
(alonepid, revid, cliohandle, clearpid) = findpid(pid)
if alonepid:
handles = [clearpid]
for pid in handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper,
geocoder) = histo(api, '')
hist = countries
else:
hist = ''
filename = filename + '.xls'
# 2DEBUG
(header, panelcells, codes, datahub, data, handle2ind, unit2ind,
originalvalues) = data2panel(handles, customcountrycodes,
fromyear, toyear, customyear, hist,
logscale)
#codes = hist
#return str(fullmetadata)
metadata = fullmetadata
result = individual_dataset(finaldir, filename, handle2ind[pid],
unit2ind[pid], datahub, data[pid],
codes, metadata)
try:
for everypid in handles:
# Download papers
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, everypid, tmpdir,
arc, finaldir)
except:
nopapers = 1
compile2zip(finaldir, arc)
filename = arc
return filename
#customcountrycodes = '528,756'
handles = ["hdl:10622/GDAO0N"]
logflag = 0
config = configuration()
if fromyear:
historical = 0
if historical:
api = config['clearance'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geo) = histo(api, '')
hist = countries
(header, panelcells, codes, datahub, data, handle2ind, unit2ind, original) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logflag)
datafile = "panel.xlsx"
#(handles, pidslist) = pidfrompanel(pid)
pidslist = handles
try:
if pidslist:
fullmetadata = load_fullmetadata(pidslist)
except:
fullmetadata = ''
# Create panel
panel2excel(config['webtest'], datafile, header, panelcells, fullmetadata)
# Create individual files
for pid in handles:
datafile = 'test1.xls'
print data[pid]
individual_dataset(config['webtest'], datafile, handle2ind[pid], unit2ind[pid], datahub, data[pid], codes, fullmetadata)
logflag = 0
config = configuration()
if fromyear:
historical = 0
if historical:
api = config['clearance'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geo) = histo(api, '')
hist = countries
(header, panelcells, codes, datahub, data, handle2ind, unit2ind,
original) = data2panel(handles, customcountrycodes, fromyear, toyear,
customyear, hist, logflag)
datafile = "panel.xlsx"
#(handles, pidslist) = pidfrompanel(pid)
pidslist = handles
try:
if pidslist:
fullmetadata = load_fullmetadata(pidslist)
except:
fullmetadata = ''
# Create panel
panel2excel(config['webtest'], datafile, header, panelcells, fullmetadata)
# Create individual files
for pid in handles:
datafile = 'test1.xls'
print data[pid]
individual_dataset(config['webtest'], datafile, handle2ind[pid],
unit2ind[pid], datahub, data[pid], codes, fullmetadata)