Esempio n. 1
0
def testdownload():
    config = configuration()
    DEBUG = 0
    API_TOKEN=config['key']
    cmd = "--insecure -u " + API_TOKEN + ": " + config['dataverseroot'] + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
    tmpdir = "/tmp/test"
    filename = randomword(10)
    arc = "data" + filename + ".zip"
    finaldir = "/home/dpe/tmp"
    if filename:
	finaldir = str(finaldir) + '/' + str(filename)
	tmpdir = str(tmpdir) + '/' + str(filename)

    pid = "hdl:10622/73BBBI"
    pid = "hdl:10622/4X6NCK"
    pid = "hdl:10622/JPIRSD"
    try:
	os.mkdir(tmpdir)
        os.mkdir(finaldir)

    except OSError as e:
        if e.errno != errno.EEXIST:
            raise e
        pass

    zipfile = get_papers(config['dataverseroot'], config['key'], cmd, pid, tmpdir, arc, finaldir)
    print zipfile
    return
Esempio n. 2
0
def download():
    (classification, pid, root, switch, datafile) = ('modern', '', '', 'modern', '')
    handle = ''
    config = configuration()
    cmd = "--insecure -u " + config['key'] + ": " + config['dataverseroot'] + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"

    config['remote'] = ''
    datafilter = {}
    datafilter['startyear'] = '1500'
    datafilter['endyear'] = '2010'
    datafilter['ctrlist'] = ''

    tmpdir = config['tmpdir']
    filerandom = randomword(10)
    #filerandom = '12345'
    arc = "data" + filerandom + ".zip"
    filename = filerandom
    finaldir = config['path'] + '/static/tmp'
    # ToDO
    if filename:
        finaldir = str(finaldir) + '/' + str(filename)
        tmpdir = str(tmpdir) + '/' + str(filename)

    try:
        os.mkdir(tmpdir)
        os.mkdir(finaldir)
    except:
        donothing = 'ok'

    if request.args.get('handle'):
        handle = request.args.get('handle')
    if request.args.get('type[0]') == 'historical':
        classification = request.args.get('type[0]')
	switch = classification
    if request.args.get('y[min]'):
        datafilter['startyear'] = request.args.get('y[min]')
    if request.args.get('y[max]'):
        datafilter['endyear'] = request.args.get('y[max]')

    # Select countries
    customcountrycodes = ''
    f = request.args
    for key in f.keys():
        if is_location(key):
            for value in sorted(f.getlist(key)):
                customcountrycodes = str(customcountrycodes) + str(value) + ','
    if customcountrycodes:
        customcountrycodes = customcountrycodes[:-1]
        datafilter['ctrlist'] = customcountrycodes

    if request.args.get('ctrlist'):
	datafilter['ctrlist'] = request.args.get('ctrlist')

    if request.args.get('pid'):
        pid = request.args.get('pid')
	ispanel = ''
	try:
	    (pids, pidslist) = pidfrompanel(pid)
	    handles = pids
	    handle = pids[0]
   	    match = re.match(r'Panel\[(.+)\]', pid)
    	    if match:
	        ispanel = 'yes'
	except:
	    handles = pid
	    handle = pids[0]
	    
	if ispanel:
	    dirforzip = ''
	    for handle in handles:
	        dirforzip = get_papers(config['dataverseroot'], config['key'], cmd, handle, tmpdir, arc, finaldir)

            (header, panelcells, metadata, totalpanel) = build_panel(config, switch, handles, datafilter)
            filename = "paneldata.xlsx"
            metadata = []
	    datadir = config['webtest']
            localoutfile = panel2excel(dirforzip, filename, header, panelcells, metadata)
	    arc = 'dataarchive.zip'
	    compile2zip(dirforzip, arc)
	    root = config['apiroot'] + "/collabs/static/tmp/" + str(arc)
            return redirect(root, code=301)

    if classification:
	outfile = "clioinfra.xlsx"
	dirforzip = get_papers(config['dataverseroot'], config['key'], cmd, handle, tmpdir, arc, finaldir)
	#fullpath = config['webtest'] + "/" + str(outfile)
	fullpath = dirforzip

	# Check selection
	isselection = 'yes'
	if datafilter['startyear'] == '1500':
	    if datafilter['ctrlist'] == '':
		isselection = 'yes'

	if isselection:
	    (datafile, outfilefinal, finalsubset) = dataframe_compiler(config, fullpath, handle, classification, datafilter)
	    #return datafile.to_html()
	else:
	    # Copy original dataset
	    source = os.listdir(tmpdir)
	    for excelfile in source:
        	shutil.copy(tmpdir + '/' + excelfile, dirforzip)

	#return outfilefinal
        arc = 'dataarchive.zip'
  	if datafile:
	    arc = "%s_%s.zip" % (datafile, switch)
        compile2zip(dirforzip, arc)
        root = config['apiroot'] + "/collabs/static/tmp/" + str(arc)
	#root = config['apiroot'] + "/collabs/static/tmp/" + str(outfile)
	return redirect(root, code=301)
    else:
        zipfile = downloadzip(pid)
        # CHANGE
        #return zipfile
        # DEBUG1
        root = config['apiroot'] + "/collabs/static/tmp/" + zipfile
        # HTML
        #resp = make_response(render_template('progress.html', download=root))
        #return "<a href=\"" + str(root) + "\">Download dataset(s) with all papers (zip archive)</a>"
        #return resp
        return redirect(root, code=301)
Esempio n. 3
0
def downloadzip(pid):
    DEBUG = 0
    (fullpath) = ('')
    fullmetadata = {}
    logscale = 0

    config = configuration() 
    config['remote'] = 'on'
    API_TOKEN = config['key']
    HOSTNAME = config['dataverseroot']
    cmd = "--insecure -u " + API_TOKEN + ": " + HOSTNAME + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
    tmpdir = config['tmpdir']
    filerandom = randomword(10)
    #filerandom = '12345'
    arc = "data" + filerandom + ".zip"
    filename = filerandom  
    finaldir = config['path'] + '/static/tmp'
    # ToDO
    if filename:
        finaldir = str(finaldir) + '/' + str(filename)
        tmpdir = str(tmpdir) + '/' + str(filename)

    try:
        os.mkdir(tmpdir)
        os.mkdir(finaldir)
    except: 
	donothing = 'ok'

    customyear = ''
    fromyear = request.args.get('y[min]')
    toyear = request.args.get('y[max]')
    historical = request.args.get('type[0]')
    (handles, pidslist) = pidfrompanel(pid)
    try:
        if pidslist:
	    fullmetadata = load_fullmetadata(pidslist)
    except:
	showwarning = 1	

    # Log scales switch
    if request.args.get('logscale'):
	logscale = 1

    # Select countries
    customcountrycodes = ''
    f = request.args
    for key in f.keys():
        if is_location(key):
            for value in sorted(f.getlist(key)):
                customcountrycodes = str(customcountrycodes) + str(value) + ','
    if customcountrycodes:
        customcountrycodes = customcountrycodes[:-1]
    
    if handles:
	if historical:
            api = config['apiroot'] + "/collabs/static/data/historical.json"
            (regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
            hist = countries
	else:
	    hist = ''

    (classification, geodataset, title, units) = content2dataframe(config, config['geocoderhandle'])

    #geocoder = buildgeocoder(dataset, config)
    (modern, historical) = loadgeocoder(config, dataset, 'geocoder')
    for handle in handles:
	#if remote:
    	#    (class1, dataset) = loaddataset_fromurl(config, handle)
	#else:
    	#    dataset = loaddataset(handles)

	#(cfilter, notint) = selectint(activeindex.values)
	#(moderndata, historicaldata) = loadgeocoder(dataset, '')
	# CHANGE
	#return str(dataset.index)
        (header, panelcells, codes, datahub, data, handle2ind, unit2ind, original) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale)
	filename = filename + '.xls'
        fullpath = panel2excel(finaldir, filename, header, panelcells, fullmetadata)
    else:
	# Clio format download
	zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, pid, tmpdir, arc, finaldir)
	(alonepid, revid, cliohandle, clearpid) = findpid(pid)
	if alonepid:
	    handles = [ clearpid ]

	for pid in handles:
	    if historical:
                api = config['apiroot'] + "/collabs/static/data/historical.json"
                (regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
                hist = countries
	    else:
	        hist = ''
	    filename = filename + '.xls'
	    # 2DEBUG
	    (header, panelcells, codes, datahub, data, handle2ind, unit2ind, originalvalues) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale)
	    #codes = hist
	    #return str(fullmetadata)
	    metadata = fullmetadata
	    result = individual_dataset(finaldir, filename, handle2ind[pid], unit2ind[pid], datahub, data[pid], codes, metadata)

    try:
        for everypid in handles:
	    # Download papers
            zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, everypid, tmpdir, arc, finaldir)
    except:
	nopapers = 1

    compile2zip(finaldir, arc)
    filename = arc
    return filename
Esempio n. 4
0
def download():
    (classification, pid, root, switch, datafile) = ('modern', '', '',
                                                     'modern', '')
    handle = ''
    config = configuration()
    cmd = "--insecure -u " + config['key'] + ": " + config[
        'dataverseroot'] + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"

    config['remote'] = ''
    datafilter = {}
    datafilter['startyear'] = '1500'
    datafilter['endyear'] = '2010'
    datafilter['ctrlist'] = ''

    tmpdir = config['tmpdir']
    filerandom = randomword(10)
    #filerandom = '12345'
    arc = "data" + filerandom + ".zip"
    filename = filerandom
    finaldir = config['path'] + '/static/tmp'
    # ToDO
    if filename:
        finaldir = str(finaldir) + '/' + str(filename)
        tmpdir = str(tmpdir) + '/' + str(filename)

    try:
        os.mkdir(tmpdir)
        os.mkdir(finaldir)
    except:
        donothing = 'ok'

    if request.args.get('handle'):
        handle = request.args.get('handle')
    if request.args.get('type[0]') == 'historical':
        classification = request.args.get('type[0]')
        switch = classification
    if request.args.get('y[min]'):
        datafilter['startyear'] = request.args.get('y[min]')
    if request.args.get('y[max]'):
        datafilter['endyear'] = request.args.get('y[max]')

    # Select countries
    customcountrycodes = ''
    f = request.args
    for key in f.keys():
        if is_location(key):
            for value in sorted(f.getlist(key)):
                customcountrycodes = str(customcountrycodes) + str(value) + ','
    if customcountrycodes:
        customcountrycodes = customcountrycodes[:-1]
        datafilter['ctrlist'] = customcountrycodes

    if request.args.get('ctrlist'):
        datafilter['ctrlist'] = request.args.get('ctrlist')

    if request.args.get('pid'):
        pid = request.args.get('pid')
        ispanel = ''
        try:
            (pids, pidslist) = pidfrompanel(pid)
            handles = pids
            handle = pids[0]
            match = re.match(r'Panel\[(.+)\]', pid)
            if match:
                ispanel = 'yes'
        except:
            handles = pid
            handle = pids[0]

        if ispanel:
            dirforzip = ''
            for handle in handles:
                dirforzip = get_papers(config['dataverseroot'], config['key'],
                                       cmd, handle, tmpdir, arc, finaldir)

            (header, panelcells, metadata,
             totalpanel) = build_panel(config, switch, handles, datafilter)
            filename = "paneldata.xlsx"
            metadata = []
            datadir = config['webtest']
            localoutfile = panel2excel(dirforzip, filename, header, panelcells,
                                       metadata)
            arc = 'dataarchive.zip'
            compile2zip(dirforzip, arc)
            root = config['apiroot'] + "/collabs/static/tmp/" + str(arc)
            return redirect(root, code=301)

    if classification:
        outfile = "clioinfra.xlsx"
        dirforzip = get_papers(config['dataverseroot'], config['key'], cmd,
                               handle, tmpdir, arc, finaldir)
        #fullpath = config['webtest'] + "/" + str(outfile)
        fullpath = dirforzip

        # Check selection
        isselection = 'yes'
        if datafilter['startyear'] == '1500':
            if datafilter['ctrlist'] == '':
                isselection = 'yes'

        if isselection:
            (datafile, outfilefinal,
             finalsubset) = dataframe_compiler(config, fullpath, handle,
                                               classification, datafilter)
            #return datafile.to_html()
        else:
            # Copy original dataset
            source = os.listdir(tmpdir)
            for excelfile in source:
                shutil.copy(tmpdir + '/' + excelfile, dirforzip)

        #return outfilefinal
        arc = 'dataarchive.zip'
        if datafile:
            arc = "%s_%s.zip" % (datafile, switch)
        compile2zip(dirforzip, arc)
        root = config['apiroot'] + "/collabs/static/tmp/" + str(arc)
        #root = config['apiroot'] + "/collabs/static/tmp/" + str(outfile)
        return redirect(root, code=301)
    else:
        zipfile = downloadzip(pid)
        # CHANGE
        #return zipfile
        # DEBUG1
        root = config['apiroot'] + "/collabs/static/tmp/" + zipfile
        # HTML
        #resp = make_response(render_template('progress.html', download=root))
        #return "<a href=\"" + str(root) + "\">Download dataset(s) with all papers (zip archive)</a>"
        #return resp
        return redirect(root, code=301)
Esempio n. 5
0
def downloadzip(pid):
    DEBUG = 0
    (fullpath) = ('')
    fullmetadata = {}
    logscale = 0

    config = configuration()
    config['remote'] = 'on'
    API_TOKEN = config['key']
    HOSTNAME = config['dataverseroot']
    cmd = "--insecure -u " + API_TOKEN + ": " + HOSTNAME + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
    tmpdir = config['tmpdir']
    filerandom = randomword(10)
    #filerandom = '12345'
    arc = "data" + filerandom + ".zip"
    filename = filerandom
    finaldir = config['path'] + '/static/tmp'
    # ToDO
    if filename:
        finaldir = str(finaldir) + '/' + str(filename)
        tmpdir = str(tmpdir) + '/' + str(filename)

    try:
        os.mkdir(tmpdir)
        os.mkdir(finaldir)
    except:
        donothing = 'ok'

    customyear = ''
    fromyear = request.args.get('y[min]')
    toyear = request.args.get('y[max]')
    historical = request.args.get('type[0]')
    (handles, pidslist) = pidfrompanel(pid)
    try:
        if pidslist:
            fullmetadata = load_fullmetadata(pidslist)
    except:
        showwarning = 1

    # Log scales switch
    if request.args.get('logscale'):
        logscale = 1

    # Select countries
    customcountrycodes = ''
    f = request.args
    for key in f.keys():
        if is_location(key):
            for value in sorted(f.getlist(key)):
                customcountrycodes = str(customcountrycodes) + str(value) + ','
    if customcountrycodes:
        customcountrycodes = customcountrycodes[:-1]

    if handles:
        if historical:
            api = config['apiroot'] + "/collabs/static/data/historical.json"
            (regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
            hist = countries
        else:
            hist = ''

    (classification, geodataset, title,
     units) = content2dataframe(config, config['geocoderhandle'])

    #geocoder = buildgeocoder(dataset, config)
    (modern, historical) = loadgeocoder(config, dataset, 'geocoder')
    for handle in handles:
        #if remote:
        #    (class1, dataset) = loaddataset_fromurl(config, handle)
        #else:
        #    dataset = loaddataset(handles)

        #(cfilter, notint) = selectint(activeindex.values)
        #(moderndata, historicaldata) = loadgeocoder(dataset, '')
        # CHANGE
        #return str(dataset.index)
        (header, panelcells, codes, datahub, data, handle2ind, unit2ind,
         original) = data2panel(handles, customcountrycodes, fromyear, toyear,
                                customyear, hist, logscale)
        filename = filename + '.xls'
        fullpath = panel2excel(finaldir, filename, header, panelcells,
                               fullmetadata)
    else:
        # Clio format download
        zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, pid, tmpdir, arc,
                             finaldir)
        (alonepid, revid, cliohandle, clearpid) = findpid(pid)
        if alonepid:
            handles = [clearpid]

        for pid in handles:
            if historical:
                api = config['apiroot'] + "/collabs/static/data/historical.json"
                (regions, countries, ctr2reg, webmapper,
                 geocoder) = histo(api, '')
                hist = countries
            else:
                hist = ''
            filename = filename + '.xls'
            # 2DEBUG
            (header, panelcells, codes, datahub, data, handle2ind, unit2ind,
             originalvalues) = data2panel(handles, customcountrycodes,
                                          fromyear, toyear, customyear, hist,
                                          logscale)
            #codes = hist
            #return str(fullmetadata)
            metadata = fullmetadata
            result = individual_dataset(finaldir, filename, handle2ind[pid],
                                        unit2ind[pid], datahub, data[pid],
                                        codes, metadata)

    try:
        for everypid in handles:
            # Download papers
            zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, everypid, tmpdir,
                                 arc, finaldir)
    except:
        nopapers = 1

    compile2zip(finaldir, arc)
    filename = arc
    return filename