Beispiel #1
0
#!/usr/bin/python

import json
import sys
import os
import simplejson
import re
sys.path.append(os.path.abspath(os.path.join(os.path.dirname("__file__"), '../modules')))
from storage import data2store, readdata, readdataset, readdatasets, datasetadd, formdatasetquery
from sys import argv

script, filename, handle, title = argv

handles = []
handlequery = ''
if filename:
    print filename
    pid = re.search('\/(\w+)', handle)
    if pid:
        dataset = pid.group(1) 
        #handle = "hdl:10622/" + dataset
	datasetadd(filename, handle, dataset, title)
	print handle
	print dataset

Beispiel #2
0
#!/usr/bin/python

import json
import sys
import os
import simplejson
import re
sys.path.append(
    os.path.abspath(os.path.join(os.path.dirname("__file__"), '../modules')))
from storage import data2store, readdata, readdataset, readdatasets, datasetadd, formdatasetquery
from sys import argv

script, filename, handle, title = argv

handles = []
handlequery = ''
if filename:
    print filename
    pid = re.search('\/(\w+)', handle)
    if pid:
        dataset = pid.group(1)
        #handle = "hdl:10622/" + dataset
        datasetadd(filename, handle, dataset, title)
        print handle
        print dataset
Beispiel #3
0
def main():
    handle = ''
    dataverse = ''
    customkey = ''
    config = configuration()

    try:
        myopts, args = getopt.getopt(sys.argv[1:], "H:r:d:k:D:")
    except getopt.GetoptError as e:
        print(str(e))
        print(
            "Usage: %s -y year -d datatype -r region -f filename -DDEBUG -o output"
            % sys.argv[0])
        sys.exit(2)

    (handle, rhandle, customdv) = ('', '', '')
    for o, a in myopts:
        if o == '-H':
            handle = a
        if o == '-r':
            rhandle = a
        if o == '-d':
            dataverse = a
        if o == '-k':
            customkey = a
        if o == '-D':
            customdv = a

    dataset = {}
    DEBUG = ''
    path = config['path']
    # Default dataverse
    root = config['dataverseroot']
    key = config['key']
    dvname = config['branch']
    title = 'Title'
    units = 'Units'

    if dataverse:
        root = dataverse
    if customkey:
        key = customkey
    if customdv:
        dvname = customdv

    files = []
    if rhandle:
        contentsapi = root + "/api/dataverses/" + dvname + "/contents?key=" + key
        print contentsapi
        newdata = load_api_data(contentsapi, '')
        metadata = newdata['data']
        for item in metadata:
            dv = item['id']
            files = getfiles(root, dv, key)

    if handle:
        print handle
        (datahandle, datasetID, fileID) = parsehandle(handle)
        files.append(fileID)

    for fileID in files:
        fullpath = downloadfile(root, path, fileID, key)
        print fullpath
        (pid, revid, cliohandle, clearpid) = findpid(handle)
        (jsonfile, csvfile) = ('', '')
        #try:
        if pid:
            handle = pid
            try:
                (jsonfile, csvfile, tmptitle,
                 tmpunits) = dataextractor(fullpath, path, pid, fileID)
            except:
                resultfile = config['tmpdir'] + "/" + fileID
                (jsonfile, csvfile, tmptitle,
                 tmpunits) = excelvalidator(config['phantompath'], fullpath,
                                            resultfile, config['tmpdir'])

            if jsonfile:
                remove = removedata('datasets', 'handle', clearpid)
                try:
                    title = str(tmptitle)
                    units = str(tmpunits)
                except:
                    donothing = 1
                print "ADD " + str(jsonfile)
                datasetadd(jsonfile, csvfile, clearpid, handle, title, units,
                           datasetID)
                print handle
                print clearpid
                print datasetID
Beispiel #4
0
def main():
    handle = ''
    dataverse = ''
    customkey = ''
    config = configuration()

    try:
        myopts, args = getopt.getopt(sys.argv[1:], "H:r:d:k:D:")
    except getopt.GetoptError as e:
        print(str(e))
        print(
            "Usage: %s -y year -d datatype -r region -f filename -DDEBUG -o output"
            % sys.argv[0])
        sys.exit(2)

    (handle, rhandle, customdv) = ('', '', '')
    for o, a in myopts:
        if o == '-H':
            handle = a
        if o == '-r':
            rhandle = a
        if o == '-d':
            dataverse = a
        if o == '-k':
            customkey = a
        if o == '-D':
            customdv = a

    dataset = {}
    DEBUG = ''
    path = config['path']
    # Default dataverse
    root = config['dataverseroot']
    key = config['key']
    dvname = config['branch']

    if dataverse:
        root = dataverse
    if customkey:
        key = customkey
    if customdv:
        dvname = customdv

    files = []
    if rhandle:
        contentsapi = root + "/api/dataverses/" + dvname + "/contents?key=" + key
        print contentsapi
        newdata = load_api_data(contentsapi, '')
        metadata = newdata['data']
        for item in metadata:
            dv = item['id']
            files = getfiles(root, dv, key)

    if handle:
        print handle
        (datahandle, datasetID, fileID) = parsehandle(handle)
        files.append(fileID)

    for fileID in files:
        fullpath = downloadfile(root, path, fileID, key)
        print fullpath
        (pid, revid, cliohandle, clearpid) = findpid(handle)
        #try:
        if pid:
            handle = pid
            jsonfile = dataextractor(fullpath, path, pid, fileID)
            if jsonfile:
                title = 'Test'
                datasetadd(jsonfile, clearpid, handle, title)
                print handle
                print clearpid
Beispiel #5
0
def main():
    handle = ''
    dataverse = ''
    customkey = ''
    config = configuration()

    try:
        myopts, args = getopt.getopt(sys.argv[1:],"H:r:d:k:D:")
    except getopt.GetoptError as e:
        print (str(e))
        print("Usage: %s -y year -d datatype -r region -f filename -DDEBUG -o output" % sys.argv[0])
        sys.exit(2)

    (handle, rhandle, customdv) = ('', '', '')
    for o, a in myopts:
        if o == '-H':
            handle=a
	if o == '-r':
	    rhandle=a 
	if o == '-d':
	    dataverse=a
	if o == '-k':
	    customkey=a
	if o == '-D':
	    customdv=a

    dataset = {}
    DEBUG = ''
    path = config['path']
    # Default dataverse
    root = config['dataverseroot']
    key = config['key']
    dvname = config['branch']
    title = 'Title'
    units = 'Units'

    if dataverse:
	root = dataverse
    if customkey:
	key = customkey
    if customdv:
	dvname = customdv

    files = []
    if rhandle:
	contentsapi = root + "/api/dataverses/" + dvname +"/contents?key=" + key
	print contentsapi
	newdata = load_api_data(contentsapi, '')
	metadata = newdata['data']
	for item in metadata:
	    dv = item['id']
	    files = getfiles(root, dv, key)

    if handle:
         print handle
         (datahandle, datasetID, fileID) = parsehandle(handle)
	 files.append(fileID)

    for fileID in files: 
	 fullpath = downloadfile(root, path, fileID, key)
	 print fullpath
	 (pid, revid, cliohandle, clearpid) = findpid(handle)
	 (jsonfile, csvfile) =('', '')
	 #try:
	 if pid:
	     handle = pid
	     try:
	        (jsonfile, csvfile, tmptitle, tmpunits) = dataextractor(fullpath, path, pid, fileID)
	     except:
		resultfile = config['tmpdir'] + "/" + fileID
		(jsonfile, csvfile, tmptitle, tmpunits) = excelvalidator(config['phantompath'], fullpath, resultfile, config['tmpdir'])
	
	     if jsonfile:
		remove = removedata('datasets', 'handle', clearpid)
		try:
		    title = str(tmptitle)
		    units = str(tmpunits)
		except:
		    donothing = 1
		print "ADD " + str(jsonfile)
        	datasetadd(jsonfile, csvfile, clearpid, handle, title, units, datasetID)
                print handle
        	print clearpid
		print datasetID