def testdownload():
config = configuration()
DEBUG = 0
API_TOKEN=config['key']
cmd = "--insecure -u " + API_TOKEN + ": " + config['dataverseroot'] + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
tmpdir = "/tmp/test"
filename = randomword(10)
arc = "data" + filename + ".zip"
finaldir = "/home/dpe/tmp"
if filename:
finaldir = str(finaldir) + '/' + str(filename)
tmpdir = str(tmpdir) + '/' + str(filename)
pid = "hdl:10622/73BBBI"
pid = "hdl:10622/4X6NCK"
pid = "hdl:10622/JPIRSD"
try:
os.mkdir(tmpdir)
os.mkdir(finaldir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
pass
zipfile = get_papers(config['dataverseroot'], config['key'], cmd, pid, tmpdir, arc, finaldir)
print zipfile
return
def download():
(classification, pid, root, switch, datafile) = ('modern', '', '', 'modern', '')
handle = ''
config = configuration()
cmd = "--insecure -u " + config['key'] + ": " + config['dataverseroot'] + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
config['remote'] = ''
datafilter = {}
datafilter['startyear'] = '1500'
datafilter['endyear'] = '2010'
datafilter['ctrlist'] = ''
tmpdir = config['tmpdir']
filerandom = randomword(10)
#filerandom = '12345'
arc = "data" + filerandom + ".zip"
filename = filerandom
finaldir = config['path'] + '/static/tmp'
# ToDO
if filename:
finaldir = str(finaldir) + '/' + str(filename)
tmpdir = str(tmpdir) + '/' + str(filename)
try:
os.mkdir(tmpdir)
os.mkdir(finaldir)
except:
donothing = 'ok'
if request.args.get('handle'):
handle = request.args.get('handle')
if request.args.get('type[0]') == 'historical':
classification = request.args.get('type[0]')
switch = classification
if request.args.get('y[min]'):
datafilter['startyear'] = request.args.get('y[min]')
if request.args.get('y[max]'):
datafilter['endyear'] = request.args.get('y[max]')
# Select countries
customcountrycodes = ''
f = request.args
for key in f.keys():
if is_location(key):
for value in sorted(f.getlist(key)):
customcountrycodes = str(customcountrycodes) + str(value) + ','
if customcountrycodes:
customcountrycodes = customcountrycodes[:-1]
datafilter['ctrlist'] = customcountrycodes
if request.args.get('ctrlist'):
datafilter['ctrlist'] = request.args.get('ctrlist')
if request.args.get('pid'):
pid = request.args.get('pid')
ispanel = ''
try:
(pids, pidslist) = pidfrompanel(pid)
handles = pids
handle = pids[0]
match = re.match(r'Panel\[(.+)\]', pid)
if match:
ispanel = 'yes'
except:
handles = pid
handle = pids[0]
if ispanel:
dirforzip = ''
for handle in handles:
dirforzip = get_papers(config['dataverseroot'], config['key'], cmd, handle, tmpdir, arc, finaldir)
(header, panelcells, metadata, totalpanel) = build_panel(config, switch, handles, datafilter)
filename = "paneldata.xlsx"
metadata = []
datadir = config['webtest']
localoutfile = panel2excel(dirforzip, filename, header, panelcells, metadata)
arc = 'dataarchive.zip'
compile2zip(dirforzip, arc)
root = config['apiroot'] + "/collabs/static/tmp/" + str(arc)
return redirect(root, code=301)
if classification:
outfile = "clioinfra.xlsx"
dirforzip = get_papers(config['dataverseroot'], config['key'], cmd, handle, tmpdir, arc, finaldir)
#fullpath = config['webtest'] + "/" + str(outfile)
fullpath = dirforzip
# Check selection
isselection = 'yes'
if datafilter['startyear'] == '1500':
if datafilter['ctrlist'] == '':
isselection = 'yes'
if isselection:
(datafile, outfilefinal, finalsubset) = dataframe_compiler(config, fullpath, handle, classification, datafilter)
#return datafile.to_html()
else:
# Copy original dataset
source = os.listdir(tmpdir)
for excelfile in source:
shutil.copy(tmpdir + '/' + excelfile, dirforzip)
#return outfilefinal
arc = 'dataarchive.zip'
if datafile:
arc = "%s_%s.zip" % (datafile, switch)
compile2zip(dirforzip, arc)
root = config['apiroot'] + "/collabs/static/tmp/" + str(arc)
#root = config['apiroot'] + "/collabs/static/tmp/" + str(outfile)
return redirect(root, code=301)
else:
zipfile = downloadzip(pid)
# CHANGE
#return zipfile
# DEBUG1
root = config['apiroot'] + "/collabs/static/tmp/" + zipfile
# HTML
#resp = make_response(render_template('progress.html', download=root))
#return "<a href=\"" + str(root) + "\">Download dataset(s) with all papers (zip archive)</a>"
#return resp
return redirect(root, code=301)
def downloadzip(pid):
DEBUG = 0
(fullpath) = ('')
fullmetadata = {}
logscale = 0
config = configuration()
config['remote'] = 'on'
API_TOKEN = config['key']
HOSTNAME = config['dataverseroot']
cmd = "--insecure -u " + API_TOKEN + ": " + HOSTNAME + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
tmpdir = config['tmpdir']
filerandom = randomword(10)
#filerandom = '12345'
arc = "data" + filerandom + ".zip"
filename = filerandom
finaldir = config['path'] + '/static/tmp'
# ToDO
if filename:
finaldir = str(finaldir) + '/' + str(filename)
tmpdir = str(tmpdir) + '/' + str(filename)
try:
os.mkdir(tmpdir)
os.mkdir(finaldir)
except:
donothing = 'ok'
customyear = ''
fromyear = request.args.get('y[min]')
toyear = request.args.get('y[max]')
historical = request.args.get('type[0]')
(handles, pidslist) = pidfrompanel(pid)
try:
if pidslist:
fullmetadata = load_fullmetadata(pidslist)
except:
showwarning = 1
# Log scales switch
if request.args.get('logscale'):
logscale = 1
# Select countries
customcountrycodes = ''
f = request.args
for key in f.keys():
if is_location(key):
for value in sorted(f.getlist(key)):
customcountrycodes = str(customcountrycodes) + str(value) + ','
if customcountrycodes:
customcountrycodes = customcountrycodes[:-1]
if handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
hist = countries
else:
hist = ''
(classification, geodataset, title, units) = content2dataframe(config, config['geocoderhandle'])
#geocoder = buildgeocoder(dataset, config)
(modern, historical) = loadgeocoder(config, dataset, 'geocoder')
for handle in handles:
#if remote:
# (class1, dataset) = loaddataset_fromurl(config, handle)
#else:
# dataset = loaddataset(handles)
#(cfilter, notint) = selectint(activeindex.values)
#(moderndata, historicaldata) = loadgeocoder(dataset, '')
# CHANGE
#return str(dataset.index)
(header, panelcells, codes, datahub, data, handle2ind, unit2ind, original) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale)
filename = filename + '.xls'
fullpath = panel2excel(finaldir, filename, header, panelcells, fullmetadata)
else:
# Clio format download
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, pid, tmpdir, arc, finaldir)
(alonepid, revid, cliohandle, clearpid) = findpid(pid)
if alonepid:
handles = [ clearpid ]
for pid in handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
hist = countries
else:
hist = ''
filename = filename + '.xls'
# 2DEBUG
(header, panelcells, codes, datahub, data, handle2ind, unit2ind, originalvalues) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale)
#codes = hist
#return str(fullmetadata)
metadata = fullmetadata
result = individual_dataset(finaldir, filename, handle2ind[pid], unit2ind[pid], datahub, data[pid], codes, metadata)
try:
for everypid in handles:
# Download papers
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, everypid, tmpdir, arc, finaldir)
except:
nopapers = 1
compile2zip(finaldir, arc)
filename = arc
return filename
def download():
(classification, pid, root, switch, datafile) = ('modern', '', '',
'modern', '')
handle = ''
config = configuration()
cmd = "--insecure -u " + config['key'] + ": " + config[
'dataverseroot'] + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
config['remote'] = ''
datafilter = {}
datafilter['startyear'] = '1500'
datafilter['endyear'] = '2010'
datafilter['ctrlist'] = ''
tmpdir = config['tmpdir']
filerandom = randomword(10)
#filerandom = '12345'
arc = "data" + filerandom + ".zip"
filename = filerandom
finaldir = config['path'] + '/static/tmp'
# ToDO
if filename:
finaldir = str(finaldir) + '/' + str(filename)
tmpdir = str(tmpdir) + '/' + str(filename)
try:
os.mkdir(tmpdir)
os.mkdir(finaldir)
except:
donothing = 'ok'
if request.args.get('handle'):
handle = request.args.get('handle')
if request.args.get('type[0]') == 'historical':
classification = request.args.get('type[0]')
switch = classification
if request.args.get('y[min]'):
datafilter['startyear'] = request.args.get('y[min]')
if request.args.get('y[max]'):
datafilter['endyear'] = request.args.get('y[max]')
# Select countries
customcountrycodes = ''
f = request.args
for key in f.keys():
if is_location(key):
for value in sorted(f.getlist(key)):
customcountrycodes = str(customcountrycodes) + str(value) + ','
if customcountrycodes:
customcountrycodes = customcountrycodes[:-1]
datafilter['ctrlist'] = customcountrycodes
if request.args.get('ctrlist'):
datafilter['ctrlist'] = request.args.get('ctrlist')
if request.args.get('pid'):
pid = request.args.get('pid')
ispanel = ''
try:
(pids, pidslist) = pidfrompanel(pid)
handles = pids
handle = pids[0]
match = re.match(r'Panel\[(.+)\]', pid)
if match:
ispanel = 'yes'
except:
handles = pid
handle = pids[0]
if ispanel:
dirforzip = ''
for handle in handles:
dirforzip = get_papers(config['dataverseroot'], config['key'],
cmd, handle, tmpdir, arc, finaldir)
(header, panelcells, metadata,
totalpanel) = build_panel(config, switch, handles, datafilter)
filename = "paneldata.xlsx"
metadata = []
datadir = config['webtest']
localoutfile = panel2excel(dirforzip, filename, header, panelcells,
metadata)
arc = 'dataarchive.zip'
compile2zip(dirforzip, arc)
root = config['apiroot'] + "/collabs/static/tmp/" + str(arc)
return redirect(root, code=301)
if classification:
outfile = "clioinfra.xlsx"
dirforzip = get_papers(config['dataverseroot'], config['key'], cmd,
handle, tmpdir, arc, finaldir)
#fullpath = config['webtest'] + "/" + str(outfile)
fullpath = dirforzip
# Check selection
isselection = 'yes'
if datafilter['startyear'] == '1500':
if datafilter['ctrlist'] == '':
isselection = 'yes'
if isselection:
(datafile, outfilefinal,
finalsubset) = dataframe_compiler(config, fullpath, handle,
classification, datafilter)
#return datafile.to_html()
else:
# Copy original dataset
source = os.listdir(tmpdir)
for excelfile in source:
shutil.copy(tmpdir + '/' + excelfile, dirforzip)
#return outfilefinal
arc = 'dataarchive.zip'
if datafile:
arc = "%s_%s.zip" % (datafile, switch)
compile2zip(dirforzip, arc)
root = config['apiroot'] + "/collabs/static/tmp/" + str(arc)
#root = config['apiroot'] + "/collabs/static/tmp/" + str(outfile)
return redirect(root, code=301)
else:
zipfile = downloadzip(pid)
# CHANGE
#return zipfile
# DEBUG1
root = config['apiroot'] + "/collabs/static/tmp/" + zipfile
# HTML
#resp = make_response(render_template('progress.html', download=root))
#return "<a href=\"" + str(root) + "\">Download dataset(s) with all papers (zip archive)</a>"
#return resp
return redirect(root, code=301)
def downloadzip(pid):
DEBUG = 0
(fullpath) = ('')
fullmetadata = {}
logscale = 0
config = configuration()
config['remote'] = 'on'
API_TOKEN = config['key']
HOSTNAME = config['dataverseroot']
cmd = "--insecure -u " + API_TOKEN + ": " + HOSTNAME + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
tmpdir = config['tmpdir']
filerandom = randomword(10)
#filerandom = '12345'
arc = "data" + filerandom + ".zip"
filename = filerandom
finaldir = config['path'] + '/static/tmp'
# ToDO
if filename:
finaldir = str(finaldir) + '/' + str(filename)
tmpdir = str(tmpdir) + '/' + str(filename)
try:
os.mkdir(tmpdir)
os.mkdir(finaldir)
except:
donothing = 'ok'
customyear = ''
fromyear = request.args.get('y[min]')
toyear = request.args.get('y[max]')
historical = request.args.get('type[0]')
(handles, pidslist) = pidfrompanel(pid)
try:
if pidslist:
fullmetadata = load_fullmetadata(pidslist)
except:
showwarning = 1
# Log scales switch
if request.args.get('logscale'):
logscale = 1
# Select countries
customcountrycodes = ''
f = request.args
for key in f.keys():
if is_location(key):
for value in sorted(f.getlist(key)):
customcountrycodes = str(customcountrycodes) + str(value) + ','
if customcountrycodes:
customcountrycodes = customcountrycodes[:-1]
if handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
hist = countries
else:
hist = ''
(classification, geodataset, title,
units) = content2dataframe(config, config['geocoderhandle'])
#geocoder = buildgeocoder(dataset, config)
(modern, historical) = loadgeocoder(config, dataset, 'geocoder')
for handle in handles:
#if remote:
# (class1, dataset) = loaddataset_fromurl(config, handle)
#else:
# dataset = loaddataset(handles)
#(cfilter, notint) = selectint(activeindex.values)
#(moderndata, historicaldata) = loadgeocoder(dataset, '')
# CHANGE
#return str(dataset.index)
(header, panelcells, codes, datahub, data, handle2ind, unit2ind,
original) = data2panel(handles, customcountrycodes, fromyear, toyear,
customyear, hist, logscale)
filename = filename + '.xls'
fullpath = panel2excel(finaldir, filename, header, panelcells,
fullmetadata)
else:
# Clio format download
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, pid, tmpdir, arc,
finaldir)
(alonepid, revid, cliohandle, clearpid) = findpid(pid)
if alonepid:
handles = [clearpid]
for pid in handles:
if historical:
api = config['apiroot'] + "/collabs/static/data/historical.json"
(regions, countries, ctr2reg, webmapper,
geocoder) = histo(api, '')
hist = countries
else:
hist = ''
filename = filename + '.xls'
# 2DEBUG
(header, panelcells, codes, datahub, data, handle2ind, unit2ind,
originalvalues) = data2panel(handles, customcountrycodes,
fromyear, toyear, customyear, hist,
logscale)
#codes = hist
#return str(fullmetadata)
metadata = fullmetadata
result = individual_dataset(finaldir, filename, handle2ind[pid],
unit2ind[pid], datahub, data[pid],
codes, metadata)
try:
for everypid in handles:
# Download papers
zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, everypid, tmpdir,
arc, finaldir)
except:
nopapers = 1
compile2zip(finaldir, arc)
filename = arc
return filename