Exemplo n.º 1
0
def downloadzip(pid):
    DEBUG = 0
    (fullpath) = ('')
    fullmetadata = {}
    logscale = 0

    config = configuration() 
    config['remote'] = 'on'
    API_TOKEN = config['key']
    HOSTNAME = config['dataverseroot']
    cmd = "--insecure -u " + API_TOKEN + ": " + HOSTNAME + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
    tmpdir = config['tmpdir']
    filerandom = randomword(10)
    #filerandom = '12345'
    arc = "data" + filerandom + ".zip"
    filename = filerandom  
    finaldir = config['path'] + '/static/tmp'
    # ToDO
    if filename:
        finaldir = str(finaldir) + '/' + str(filename)
        tmpdir = str(tmpdir) + '/' + str(filename)

    try:
        os.mkdir(tmpdir)
        os.mkdir(finaldir)
    except: 
	donothing = 'ok'

    customyear = ''
    fromyear = request.args.get('y[min]')
    toyear = request.args.get('y[max]')
    historical = request.args.get('type[0]')
    (handles, pidslist) = pidfrompanel(pid)
    try:
        if pidslist:
	    fullmetadata = load_fullmetadata(pidslist)
    except:
	showwarning = 1	

    # Log scales switch
    if request.args.get('logscale'):
	logscale = 1

    # Select countries
    customcountrycodes = ''
    f = request.args
    for key in f.keys():
        if is_location(key):
            for value in sorted(f.getlist(key)):
                customcountrycodes = str(customcountrycodes) + str(value) + ','
    if customcountrycodes:
        customcountrycodes = customcountrycodes[:-1]
    
    if handles:
	if historical:
            api = config['apiroot'] + "/collabs/static/data/historical.json"
            (regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
            hist = countries
	else:
	    hist = ''

    (classification, geodataset, title, units) = content2dataframe(config, config['geocoderhandle'])

    #geocoder = buildgeocoder(dataset, config)
    (modern, historical) = loadgeocoder(config, dataset, 'geocoder')
    for handle in handles:
	#if remote:
    	#    (class1, dataset) = loaddataset_fromurl(config, handle)
	#else:
    	#    dataset = loaddataset(handles)

	#(cfilter, notint) = selectint(activeindex.values)
	#(moderndata, historicaldata) = loadgeocoder(dataset, '')
	# CHANGE
	#return str(dataset.index)
        (header, panelcells, codes, datahub, data, handle2ind, unit2ind, original) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale)
	filename = filename + '.xls'
        fullpath = panel2excel(finaldir, filename, header, panelcells, fullmetadata)
    else:
	# Clio format download
	zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, pid, tmpdir, arc, finaldir)
	(alonepid, revid, cliohandle, clearpid) = findpid(pid)
	if alonepid:
	    handles = [ clearpid ]

	for pid in handles:
	    if historical:
                api = config['apiroot'] + "/collabs/static/data/historical.json"
                (regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
                hist = countries
	    else:
	        hist = ''
	    filename = filename + '.xls'
	    # 2DEBUG
	    (header, panelcells, codes, datahub, data, handle2ind, unit2ind, originalvalues) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logscale)
	    #codes = hist
	    #return str(fullmetadata)
	    metadata = fullmetadata
	    result = individual_dataset(finaldir, filename, handle2ind[pid], unit2ind[pid], datahub, data[pid], codes, metadata)

    try:
        for everypid in handles:
	    # Download papers
            zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, everypid, tmpdir, arc, finaldir)
    except:
	nopapers = 1

    compile2zip(finaldir, arc)
    filename = arc
    return filename
Exemplo n.º 2
0
def downloadzip(pid):
    DEBUG = 0
    (fullpath) = ('')
    fullmetadata = {}
    logscale = 0

    config = configuration()
    config['remote'] = 'on'
    API_TOKEN = config['key']
    HOSTNAME = config['dataverseroot']
    cmd = "--insecure -u " + API_TOKEN + ": " + HOSTNAME + "/dvn/api/data-deposit/v1.1/swordv2/statement/study/"
    tmpdir = config['tmpdir']
    filerandom = randomword(10)
    #filerandom = '12345'
    arc = "data" + filerandom + ".zip"
    filename = filerandom
    finaldir = config['path'] + '/static/tmp'
    # ToDO
    if filename:
        finaldir = str(finaldir) + '/' + str(filename)
        tmpdir = str(tmpdir) + '/' + str(filename)

    try:
        os.mkdir(tmpdir)
        os.mkdir(finaldir)
    except:
        donothing = 'ok'

    customyear = ''
    fromyear = request.args.get('y[min]')
    toyear = request.args.get('y[max]')
    historical = request.args.get('type[0]')
    (handles, pidslist) = pidfrompanel(pid)
    try:
        if pidslist:
            fullmetadata = load_fullmetadata(pidslist)
    except:
        showwarning = 1

    # Log scales switch
    if request.args.get('logscale'):
        logscale = 1

    # Select countries
    customcountrycodes = ''
    f = request.args
    for key in f.keys():
        if is_location(key):
            for value in sorted(f.getlist(key)):
                customcountrycodes = str(customcountrycodes) + str(value) + ','
    if customcountrycodes:
        customcountrycodes = customcountrycodes[:-1]

    if handles:
        if historical:
            api = config['apiroot'] + "/collabs/static/data/historical.json"
            (regions, countries, ctr2reg, webmapper, geocoder) = histo(api, '')
            hist = countries
        else:
            hist = ''

    (classification, geodataset, title,
     units) = content2dataframe(config, config['geocoderhandle'])

    #geocoder = buildgeocoder(dataset, config)
    (modern, historical) = loadgeocoder(config, dataset, 'geocoder')
    for handle in handles:
        #if remote:
        #    (class1, dataset) = loaddataset_fromurl(config, handle)
        #else:
        #    dataset = loaddataset(handles)

        #(cfilter, notint) = selectint(activeindex.values)
        #(moderndata, historicaldata) = loadgeocoder(dataset, '')
        # CHANGE
        #return str(dataset.index)
        (header, panelcells, codes, datahub, data, handle2ind, unit2ind,
         original) = data2panel(handles, customcountrycodes, fromyear, toyear,
                                customyear, hist, logscale)
        filename = filename + '.xls'
        fullpath = panel2excel(finaldir, filename, header, panelcells,
                               fullmetadata)
    else:
        # Clio format download
        zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, pid, tmpdir, arc,
                             finaldir)
        (alonepid, revid, cliohandle, clearpid) = findpid(pid)
        if alonepid:
            handles = [clearpid]

        for pid in handles:
            if historical:
                api = config['apiroot'] + "/collabs/static/data/historical.json"
                (regions, countries, ctr2reg, webmapper,
                 geocoder) = histo(api, '')
                hist = countries
            else:
                hist = ''
            filename = filename + '.xls'
            # 2DEBUG
            (header, panelcells, codes, datahub, data, handle2ind, unit2ind,
             originalvalues) = data2panel(handles, customcountrycodes,
                                          fromyear, toyear, customyear, hist,
                                          logscale)
            #codes = hist
            #return str(fullmetadata)
            metadata = fullmetadata
            result = individual_dataset(finaldir, filename, handle2ind[pid],
                                        unit2ind[pid], datahub, data[pid],
                                        codes, metadata)

    try:
        for everypid in handles:
            # Download papers
            zipfile = get_papers(HOSTNAME, API_TOKEN, cmd, everypid, tmpdir,
                                 arc, finaldir)
    except:
        nopapers = 1

    compile2zip(finaldir, arc)
    filename = arc
    return filename
Exemplo n.º 3
0
#customcountrycodes = '528,756'
handles = ["hdl:10622/GDAO0N"]
logflag = 0
config = configuration()

if fromyear:
    historical = 0
    if historical:
        api = config['clearance'] + "/collabs/static/data/historical.json"
        (regions, countries, ctr2reg, webmapper, geo) = histo(api, '')
        hist = countries

    (header, panelcells, codes, datahub, data, handle2ind, unit2ind, original) = data2panel(handles, customcountrycodes, fromyear, toyear, customyear, hist, logflag)
    datafile = "panel.xlsx"
    #(handles, pidslist) = pidfrompanel(pid)
    pidslist = handles
    try:
        if pidslist:
            fullmetadata = load_fullmetadata(pidslist)
    except:
	fullmetadata = ''
    # Create panel
    panel2excel(config['webtest'], datafile, header, panelcells, fullmetadata)

# Create individual files 
for pid in handles:
    datafile = 'test1.xls'    
    print data[pid]
    individual_dataset(config['webtest'], datafile, handle2ind[pid], unit2ind[pid], datahub, data[pid], codes, fullmetadata)

Exemplo n.º 4
0
logflag = 0
config = configuration()

if fromyear:
    historical = 0
    if historical:
        api = config['clearance'] + "/collabs/static/data/historical.json"
        (regions, countries, ctr2reg, webmapper, geo) = histo(api, '')
        hist = countries

    (header, panelcells, codes, datahub, data, handle2ind, unit2ind,
     original) = data2panel(handles, customcountrycodes, fromyear, toyear,
                            customyear, hist, logflag)
    datafile = "panel.xlsx"
    #(handles, pidslist) = pidfrompanel(pid)
    pidslist = handles
    try:
        if pidslist:
            fullmetadata = load_fullmetadata(pidslist)
    except:
        fullmetadata = ''
    # Create panel
    panel2excel(config['webtest'], datafile, header, panelcells, fullmetadata)

# Create individual files
for pid in handles:
    datafile = 'test1.xls'
    print data[pid]
    individual_dataset(config['webtest'], datafile, handle2ind[pid],
                       unit2ind[pid], datahub, data[pid], codes, fullmetadata)