def read(args):
"""perform read action"""
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
# retrieve and set extra values
extra_config = {}
# setup connection to handle server
client = EUDATHandleClient.instantiate_for_read_access(
credentials.get_server_URL(), **extra_config)
# set default return value
json_result = "None"
if args.key is None:
# retrieve whole handle
result = client.retrieve_handle_record_json(args.handle)
if result is not None:
json_result = json.dumps(result["values"])
else:
# retrieve single value from a handle
result = client.get_value_from_handle(args.handle, args.key)
if result is not None:
json_result = json.dumps(result)
# remove starting and finishing quotes.
json_result = json_result.lstrip('"')
json_result = json_result.rstrip('"')
sys.stdout.write(json_result)
def read(args):
"""perform read action"""
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials, **extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
if args.key is None:
result = read_execution(client, args.handle)
else:
result = read_execution(client, args.handle, args.key)
sys.stdout.write(result)
def connect_client(self, force_no_credentials=False, disable_logs=False):
if getattr(self, '_handle_client', None) is None:
if disable_logs:
import logging
logging.getLogger('b2handle').setLevel(logging.WARNING)
# With credentials
if force_no_credentials:
self._handle_client = b2handle.instantiate_for_read_access()
log.debug("HANDLE client connected [w/out credentials]")
else:
found = False
file = os.environ.get('HANDLE_CREDENTIALS', None)
if file is not None:
from utilities import path
credentials_path = path.build(file)
found = path.file_exists_and_nonzero(credentials_path)
if not found:
log.warning("B2HANDLE credentials file not found %s",
file)
if found:
self._handle_client = \
b2handle.instantiate_with_credentials(
credentials.load_from_JSON(file)
)
log.debug("HANDLE client connected [w/ credentials]")
return self._handle_client, True
return self._handle_client, False
def create(args):
"""perform create action"""
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
# create a handle to put. Concate the prefix with a new generated suffix
prefix = str(credentials.get_prefix())
uid = uuid.uuid1()
suffix = str(uid)
handle = prefix+"/"+suffix
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials,
**extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
overwrite=False
result = create_execution(client, handle, args.location, overwrite, args.checksum, args.loc10320, args.extratype)
sys.stdout.write(result)
def read(args):
"""perform read action"""
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials,
**extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
if args.key is None:
result = read_execution(client, args.handle)
else:
result = read_execution(client, args.handle, args.key)
sys.stdout.write(result)
def modify(args):
"""perform modify action"""
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
# retrieve and set extra values
extra_config = {}
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials, **extra_config)
kvpairs = dict([(args.key, args.value)])
# modify key/value pairs
result = client.modify_handle_value(args.handle,
ttl=None,
add_if_not_exist=True,
**kvpairs)
output_result = str(result)
if output_result == 'None':
output_result = 'True'
sys.stdout.write(output_result)
def register_handle(self, location):
"""Register a handle."""
pid = ''
try:
credential = PIDClientCredentials.load_from_JSON(
self.credential_path)
client = EUDATHandleClient.instantiate_with_credentials(credential)
pid = credential.get_prefix() + '/' \
+ "{:010d}".format(int(location.split('/records/')[1]))
handle = client.register_handle(pid, location)
current_app.logger.info(
'Registered successfully handle {}'.format(pid))
return handle
except (FileNotFoundError, CredentialsFormatError,
HandleAuthenticationError, GenericHandleError) as e:
current_app.logger.error('Registration failed of handle {}. {} in '
'HandleClient.register_handle'.format(
pid, e))
return None
except AttributeError:
current_app.logger.error('Missing Private Key!')
return None
except Exception as e:
current_app.logger.error(e)
return None
def create(args):
"""perform create action"""
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
# create a handle to put. Concate the prefix with a new generated suffix
prefix = str(credentials.get_prefix())
uid = uuid.uuid1()
suffix = str(uid)
handle = prefix + "/" + suffix
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials, **extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
overwrite = False
result = create_execution(client, handle, args.location, overwrite,
args.checksum, args.loc10320, args.extratype)
sys.stdout.write(result)
def search(args):
"""perform search action"""
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
# set username and password for search
if 'reverse_username' in credentials.get_config():
reverselookup_username = credentials.get_config()['reverse_username']
else:
reverselookup_username = credentials.get_prefix()
if 'reverse_password' in credentials.get_config():
reverselookup_password = credentials.get_config()['reverse_password']
else:
reverselookup_password = credentials.get_password()
# retrieve and set extra values
extra_config = {}
# setup connection to handle server
client = EUDATHandleClient.instantiate_for_read_and_search(
credentials.get_server_URL(), reverselookup_username,
reverselookup_password, **extra_config)
kvpairs = dict([(args.key, str(''.join(args.value)))])
# search for handle
result = client.search_handle(**kvpairs)
json_result = str(json.dumps(result))
if json_result == '[]':
json_result = 'empty'
sys.stdout.write(json_result)
def test_config_from_json(self):
"""Test credentials instantiation from JSON file, with config."""
path_to_json_credentials = PATH_CRED+'/credentials_correct_withconfig_PUBLIC.json'
inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
jsonfilecontent = json.loads(open(path_to_json_credentials, 'r').read())
self.assertEqual(inst.get_config()['foo'], jsonfilecontent['foo'],
'Config not the same as in json file.')
self.assertEqual(inst.get_config()['bar'], jsonfilecontent['bar'],
'Config not the same as in json file.')
def conn_handle(credentials='cred_21.T12995.json'):
cred = PIDClientCredentials.load_from_JSON(credentials)
print GREEN, "DEBUG"
print('PID prefix ' + cred.get_prefix())
print('Server ' + cred.get_server_URL())
print DEFAULT
ec = EUDATHandleClient.instantiate_with_credentials(cred)
return ec, cred
def test_config_from_json(self):
"""Test credentials instantiation from JSON file, with config."""
path_to_json_credentials = PATH_CRED+'/credentials_correct_withconfig_PUBLIC.json'
inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
jsonfilecontent = json.loads(open(path_to_json_credentials, 'r').read())
self.assertEqual(inst.get_config()['foo'], jsonfilecontent['foo'],
'Config not the same as in json file.')
self.assertEqual(inst.get_config()['bar'], jsonfilecontent['bar'],
'Config not the same as in json file.')
def retrieve_handle(self, handle):
"""Retrieve a handle."""
try:
credential = PIDClientCredentials.load_from_JSON(
self.credential_path)
client = EUDATHandleClient.instantiate_with_credentials(credential)
handle_record_json = client.retrieve_handle_record_json(handle)
return jsonify(handle_record_json)
except (CredentialsFormatError, FileNotFoundError) as e:
current_app.logger.error(
'{} in HandleClient.retrieve_handle_record_json({})'.format(
e, handle))
def test_getters(self):
"""Test credentials instantiation from JSON file."""
path_to_json_credentials = PATH_CRED+'/credentials_correct_PUBLIC.json'
inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
jsonfilecontent = json.loads(open(path_to_json_credentials, 'r').read())
self.assertEqual(inst.get_username(), jsonfilecontent['username'],
'Username not the same as in json file.')
self.assertEqual(inst.get_password(), jsonfilecontent['password'],
'Password not the same as in json file.')
self.assertEqual(inst.get_server_URL(), jsonfilecontent['baseuri'],
'Server URL not the same as in json file.')
def test_getters(self):
"""Test credentials instantiation from JSON file."""
path_to_json_credentials = PATH_CRED+'/credentials_correct_PUBLIC.json'
inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
jsonfilecontent = json.loads(open(path_to_json_credentials, 'r').read())
self.assertEqual(inst.get_username(), jsonfilecontent['username'],
'Username not the same as in json file.')
self.assertEqual(inst.get_password(), jsonfilecontent['password'],
'Password not the same as in json file.')
self.assertEqual(inst.get_server_URL(), jsonfilecontent['baseuri'],
'Server URL not the same as in json file.')
def read(args):
"""perform read action"""
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials,
**extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
# set default return value
json_result = "None"
if args.key is None:
try:
# retrieve whole handle
result = client.retrieve_handle_record_json(args.handle)
except HandleSyntaxError:
json_result = 'error'
if result is not None:
json_result = json.dumps(result["values"])
else:
try:
# retrieve single value from a handle
result = client.get_value_from_handle(args.handle, args.key)
except HandleSyntaxError:
json_result = 'error'
if result is not None:
json_result = json.dumps(result)
# remove starting and finishing quotes.
json_result = json_result.lstrip('"')
json_result = json_result.rstrip('"')
sys.stdout.write(json_result)
def main():
# parse command line options and arguments:
modes = [
'x', 'xmlfiles', 'j', 'jsonfiles', 'c', 'ckandatasets', 'p', 'pids',
'x-p', 'x-j', 'j-c', 'j-p'
]
p = options_parser(modes)
options, arguments = p.parse_args()
# check option 'mode' and generate process list:
(mode, pstat) = pstat_init(p, modes, options.mode, options.source,
options.host)
# check for quiet mode
if (options.quiet):
qmsg = 'would be'
mainmode = 'check'
else:
qmsg = 'is'
mainmode = 'deletion'
if options.host:
print "\tCKAN HOST:\t%s" % (options.host)
if options.handle_check:
print "\tCREDENTIAL:\t%s" % (options.handle_check)
print '=' * 90
# make jobdir
now = time.strftime("%Y-%m-%d %H:%M:%S")
jid = os.getpid()
print "\tStart of processing:\t%s" % (now)
global logger
OUT = Output(pstat, now, jid, options)
##HEW-D logger = log.getLogger()
## logger
logger = OUT.setup_custom_logger('root', options.verbose)
# create credentials if required
if (options.handle_check):
try:
cred = PIDClientCredentials.load_from_JSON('credentials_11098')
except Exception, err:
logger.critical(
"[CRITICAL] %s Could not create credentials from credstore %s"
% (err, options.handle_check))
p.print_help()
sys.exit(-1)
else:
logger.debug("Create EUDATHandleClient instance")
HandleClient = EUDATHandleClient.instantiate_with_credentials(
cred, HTTPS_verify=True)
def delete(args):
"""perform delete action"""
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials,
**extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
result = 'True'
if args.key is None:
# delete whole handle
try:
client.delete_handle(args.handle)
except HandleAuthenticationError:
result = 'error'
except HandleNotFoundException:
result = 'False'
except HandleSyntaxError:
result = 'error'
else:
# delete value
try:
client.delete_handle_value(args.handle, args.key)
except HandleAuthenticationError:
result = 'error'
except HandleNotFoundException:
result = 'False'
except HandleSyntaxError:
result = 'error'
sys.stdout.write(result)
def relation(args):
"""perform the relation action"""
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
# retrieve and set extra values
extra_config = {}
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials, **extra_config)
# add relation to 10320/LOC
result = client.add_additional_URL(args.ppid, args.cpid)
sys.stdout.write(str(result))
def modify(args):
"""perform modify action"""
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials,
**extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
kvpairs = dict([(args.key, args.value)])
result = 'True'
try:
# modify key/value pairs
client.modify_handle_value(
args.handle,
ttl=None,
add_if_not_exist=True,
**kvpairs)
except HandleAuthenticationError:
result = 'error'
except HandleNotFoundException:
result = 'False'
except HandleSyntaxError:
result = 'error'
sys.stdout.write(result)
def main():
# parse command line options and arguments:
modes=['x','xmlfiles','j','jsonfiles','c','ckandatasets','p','pids','x-p', 'x-j', 'j-c','j-p']
p = options_parser(modes)
options,arguments = p.parse_args()
# check option 'mode' and generate process list:
(mode, pstat) = pstat_init(p,modes,options.mode,options.source,options.host)
# check for quiet mode
if (options.quiet):
qmsg='would be'
mainmode='check'
else:
qmsg='is'
mainmode='deletion'
if options.host :
print "\tCKAN HOST:\t%s" % (options.host)
if options.handle_check :
print "\tCREDENTIAL:\t%s" % (options.handle_check)
print '='*90
# make jobdir
now = time.strftime("%Y-%m-%d %H:%M:%S")
jid = os.getpid()
print "\tStart of processing:\t%s" % (now)
global logger
OUT = Output(pstat,now,jid,options)
##HEW-D logger = log.getLogger()
## logger
logger = OUT.setup_custom_logger('root',options.verbose)
# create credentials if required
if (options.handle_check):
try:
cred = PIDClientCredentials.load_from_JSON('credentials_11098')
except Exception, err:
logger.critical("[CRITICAL] %s Could not create credentials from credstore %s" % (err,options.handle_check))
p.print_help()
sys.exit(-1)
else:
logger.debug("Create EUDATHandleClient instance")
HandleClient = EUDATHandleClient.instantiate_with_credentials(cred,HTTPS_verify=True)
def create(args):
"""perform create action"""
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
# retrieve and set extra values
extra_config = {}
# create a handle to put. Concate the prefix with a new generated suffix
prefix = str(credentials.get_prefix())
uid = uuid.uuid1()
suffix = str(uid)
handle = prefix + "/" + suffix
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials, **extra_config)
# pre-process the input parameters for the handle api
extype = {}
if args.extratype is not None:
exlist = args.extratype.split(';')
for item in exlist:
key = item.split('=')[0]
extype[key] = item.split('=')[1]
if args.loc10320 is not None:
l10320 = args.loc10320.split(';')
else:
l10320 = None
# create the new handle
result = client.register_handle(handle,
location=args.location,
checksum=args.checksum,
additional_URLs=l10320,
overwrite=False,
**extype)
if result is None:
sys.stdout.write("error")
else:
sys.stdout.write(result)
def process_upload(UP, rlist):
##HEW-D-ec credentials,ec = None,None
def print_extra(key,jsondata):
for v in jsondata['extras']:
if v['key'] == key:
print ' Key : %s | Value : %s |' % (v['key'],v['value'])
# create credentials and handle client if required
if (options.handle_check):
try:
cred = PIDClientCredentials.load_from_JSON('credentials_11098')
except Exception, err:
logger.critical("[CRITICAL %s ] : Could not create credentials from credstore %s" % (err,options.handle_check))
##p.print_help()
sys.exit(-1)
else:
logger.debug("Create EUDATHandleClient instance")
client = EUDATHandleClient.instantiate_with_credentials(cred)
def search(args):
"""perform search action"""
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials,
**extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
kvpairs = dict([(args.key, str(''.join(args.value)))])
try:
# search for handle
result = client.search_handle(**kvpairs)
except ReverseLookupException:
result = '{error}'
json_result = str(json.dumps(result))
if json_result == '[]':
json_result = 'empty'
elif json_result == '{error}':
json_result = 'error'
sys.stdout.write(json_result)
def delete(args):
"""perform delete action"""
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
# retrieve and set extra values
extra_config = {}
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials, **extra_config)
if args.key is None:
# delete whole handle
result = client.delete_handle(args.handle)
else:
# delete value
result = client.delete_handle_value(args.handle, args.key)
sys.stdout.write(str(result))
def relation(args):
"""perform the relation action"""
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials,
**extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
result = 'None'
# add relation to 10320/LOC
try:
client.add_additional_URL(args.ppid, args.cpid)
except HandleAuthenticationError:
result = 'error'
except HandleNotFoundException:
result = 'False'
except HandleSyntaxError:
result = 'error'
sys.stdout.write(result)
def connect_client(self, force_no_credentials=False):
found = False
# With credentials
if not force_no_credentials:
file = os.environ.get('HANDLE_CREDENTIALS', None)
if file is not None:
from utilities import path
credentials_path = path.build(file)
found = path.file_exists_and_nonzero(credentials_path)
if not found:
log.warning("B2HANDLE credentials file not found %s", file)
if found:
client = b2handle.instantiate_with_credentials(
credentials.load_from_JSON(file))
log.info("PID client connected: w/ credentials")
return client, True
client = b2handle.instantiate_for_read_access()
log.warning("PID client connected: NO credentials")
return client, False
def relation(args):
"""perform the relation action"""
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials, **extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
result = 'None'
# add relation to 10320/LOC
try:
client.add_additional_URL(args.ppid, args.cpid)
except HandleAuthenticationError:
result = 'error'
except HandleNotFoundException:
result = 'False'
except HandleSyntaxError:
result = 'error'
sys.stdout.write(result)
def test_credentials_from_json_username_without_index(self):
"""Exception occurs if user name in json file does not have an index."""
path_to_json_credentials = PATH_CRED+'/credentials_noindex_PUBLIC.json'
with self.assertRaises(HandleSyntaxError):
_inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
def test_credentials_from_json_invalid_username(self):
"""Exception occurs if user name in json file is not index:prefix/suffix."""
path_to_json_credentials = PATH_CRED+'/credentials_wrongusername_PUBLIC.json'
with self.assertRaises(HandleSyntaxError):
_inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
def test_credentials_from_json_missing_items(self):
"""Exception occurs if items are missing from json credentials file."""
path_to_json_credentials = PATH_CRED+'/credentials_usernamemissing_PUBLIC.json'
with self.assertRaises(CredentialsFormatError):
_inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
def bulk(args):
"""perform the bulk actions"""
try:
# open input file
bulk_input_file = open(args.input, "r")
except:
sys.stdout.write('error opening: '+args.input)
return
try:
# open result file
bulk_result_file = open(args.result, "w")
except:
sys.stdout.write('error opening: '+args.result)
return
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials,
**extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
for line in bulk_input_file:
bulk_array = line.split()
if bulk_array[0] == 'SEARCH':
# search key value # search handle which matches criteria
search_key = bulk_array[1]
search_value = bulk_array[2]
result = search_execution(client, search_key, search_value)
bulk_result_file.write('search handle key: '+search_key+' value: '+search_value+' result: '+result+'\n')
if bulk_array[0] == 'READ':
# READ handle # read whole handle
# READ handle key # read key/value pair from handle
read_handle = bulk_array[1]
if len(bulk_array) >= 3:
read_key = bulk_array[2]
result = read_execution(client, read_handle, read_key)
bulk_result_file.write('read handle: '+read_handle+' key: '+read_key+' result: '+result+'\n')
else:
result = read_execution(client, read_handle)
bulk_result_file.write('read handle: '+read_handle+' result: '+result+'\n')
if bulk_array[0] == 'CREATE':
# CREATE prefix/uuid URL # create handle, use uuid for suffix
# CREATE prefix/suffix URL # create handle, use suffix for handle, no check before if handle exists
# CREATE prefix/uuid URL CHECKSUM # create handle, use uuid for suffix, add checksum
# CREATE prefix/uuid URL CHECKSUM 10320/LOC # create handle, use uuid for suffix, add checksum, add 10320/LOC
# CREATE prefix/uuid URL CHECKSUM 10320/LOC EXTRATYPE # create handle, use uuid for suffix, add checksum, add 10320/LOC, add extratypes
overwrite=False
checksum = None
loc10320 = None
extratype = None
# create a handle to put.
prefix = str(credentials.get_prefix())
create_prefix = bulk_array[1].split("/")[0]
create_suffix = bulk_array[1].split("/")[1]
if create_suffix == 'uuid':
uid = uuid.uuid1()
suffix = str(uid)
handle = prefix+"/"+suffix
else:
handle = prefix+"/"+create_suffix
overwrite=True
if len(bulk_array) == 3:
result = create_execution(client, handle, bulk_array[2], overwrite)
else:
if len(bulk_array) >= 4 and bulk_array[3].lower() != 'none':
checksum = bulk_array[3]
if len(bulk_array) >= 5 and bulk_array[4].lower() != 'none':
loc10320 = bulk_array[4]
if len(bulk_array) >= 6 and bulk_array[5].lower() != 'none':
extratype = bulk_array[5]
result = create_execution(client, handle, bulk_array[2], overwrite, checksum, loc10320, extratype)
bulk_result_file.write('create handle: '+bulk_array[1]+' result: '+result+'\n')
if bulk_array[0] == 'DELETE':
# DELETE handle # delete whole handle
# DELETE handle key # delete key/value pair from handle
delete_handle = bulk_array[1]
if len(bulk_array) >= 3:
delete_key = bulk_array[2]
result = delete_execution(client, delete_handle, delete_key)
bulk_result_file.write('delete handle: '+delete_handle+' key: '+delete_key+' result: '+result+'\n')
else:
result = delete_execution(client, delete_handle)
bulk_result_file.write('delete handle: '+delete_handle+' result: '+result+'\n')
if bulk_array[0] == 'MODIFY':
# MODIFY handle key value # modify key/value pair in handle
if len(bulk_array) == 4:
modify_handle = bulk_array[1]
modify_key = bulk_array[2]
modify_value = bulk_array[3]
result = modify_execution(client, modify_handle, modify_key, modify_value)
bulk_result_file.write('modify handle: '+modify_handle+' key: '+modify_key+' value: '+modify_value+' result: '+result+'\n')
if bulk_array[0] == 'REPLACE':
# REPLACE handle key data1 data2 # replace data1 with data2 in value part of key/value pair in handle
if len(bulk_array) == 5:
replace_handle = bulk_array[1]
replace_key = bulk_array[2]
replace_data1 = bulk_array[3]
replace_data2 = bulk_array[4]
result = read_execution(client, replace_handle, replace_key)
if result != "None" and result != 'error':
new_value = result.replace(replace_data1, replace_data2)
if result != new_value:
result = modify_execution(client, replace_handle, replace_key, new_value)
else:
result = "None"
bulk_result_file.write('replace handle: '+replace_handle+' key: '+replace_key+' data1: '+replace_data1+' data2: '+replace_data2+' result: '+result+'\n')
bulk_input_file.close()
bulk_result_file.close()
def test_credentials_from_json(self):
"""Test credentials instantiation from JSON file."""
path_to_json_credentials = PATH_CRED+'/credentials_correct_PUBLIC.json'
inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
self.assertIsInstance(inst, PIDClientCredentials)
def create(args):
"""perform create action"""
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
# create a handle to put. Concate the prefix with a new generated suffix
prefix = str(credentials.get_prefix())
uid = uuid.uuid1()
suffix = str(uid)
handle = prefix+"/"+suffix
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials,
**extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
# pre-process the input parameters for the handle api
extype = {}
if args.extratype is not None:
exlist = args.extratype.split(';')
for item in exlist:
key = item.split('=')[0]
value = item.split('=')[1]
extype[key] = value
if args.loc10320 is not None:
l10320 = args.loc10320.split(';')
else:
l10320 = None
# replace "EUDAT/ROR=pid" with "EUDAT/ROR=handle"
key = 'EUDAT/ROR'
if key in extype:
if extype[key].lower() == 'pid':
extype[key] = handle
result = ''
try:
# create the new handle
result = client.register_handle(
handle,
location=args.location,
checksum=args.checksum,
additional_URLs=l10320,
overwrite=False,
**extype)
except HandleAlreadyExistsException:
result = 'False'
except HandleAuthenticationError:
result = 'error'
except HandleSyntaxError:
result = 'error'
sys.stdout.write(result)
def test_credentials_from_json_empty_items(self):
"""Exception occurs if items are empty in json credentials file."""
path_to_json_credentials = PATH_CRED+'/credentials_usernameempty_PUBLIC.json'
with self.assertRaises(CredentialsFormatError):
_inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
def bulk(args):
"""perform the bulk actions"""
try:
# open input file
bulk_input_file = open(args.input, "r")
except:
sys.stdout.write('error opening: ' + args.input)
return
try:
# open result file
bulk_result_file = open(args.result, "w")
except:
sys.stdout.write('error opening: ' + args.result)
return
try:
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
except CredentialsFormatError:
sys.stdout.write('error')
return
except HandleSyntaxError:
sys.stdout.write('error')
return
# retrieve and set extra values
extra_config = {}
try:
# setup connection to handle server
client = EUDATHandleClient.instantiate_with_credentials(
credentials, **extra_config)
except HandleNotFoundException:
sys.stdout.write('error')
return
for line in bulk_input_file:
bulk_array = line.split()
if bulk_array[0] == 'SEARCH':
# search key value # search handle which matches criteria
search_key = bulk_array[1]
search_value = bulk_array[2]
result = search_execution(client, search_key, search_value)
bulk_result_file.write('search handle key: ' + search_key +
' value: ' + search_value + ' result: ' +
result + '\n')
if bulk_array[0] == 'READ':
# READ handle # read whole handle
# READ handle key # read key/value pair from handle
read_handle = bulk_array[1]
if len(bulk_array) >= 3:
read_key = bulk_array[2]
result = read_execution(client, read_handle, read_key)
bulk_result_file.write('read handle: ' + read_handle +
' key: ' + read_key + ' result: ' +
result + '\n')
else:
result = read_execution(client, read_handle)
bulk_result_file.write('read handle: ' + read_handle +
' result: ' + result + '\n')
if bulk_array[0] == 'CREATE':
# CREATE prefix/uuid URL # create handle, use uuid for suffix
# CREATE prefix/suffix URL # create handle, use suffix for handle, no check before if handle exists
# CREATE prefix/uuid URL CHECKSUM # create handle, use uuid for suffix, add checksum
# CREATE prefix/uuid URL CHECKSUM 10320/LOC # create handle, use uuid for suffix, add checksum, add 10320/LOC
# CREATE prefix/uuid URL CHECKSUM 10320/LOC EXTRATYPE # create handle, use uuid for suffix, add checksum, add 10320/LOC, add extratypes
overwrite = False
checksum = None
loc10320 = None
extratype = None
# create a handle to put.
prefix = str(credentials.get_prefix())
create_prefix = bulk_array[1].split("/")[0]
create_suffix = bulk_array[1].split("/")[1]
if create_suffix == 'uuid':
uid = uuid.uuid1()
suffix = str(uid)
handle = prefix + "/" + suffix
else:
handle = prefix + "/" + create_suffix
overwrite = True
if len(bulk_array) == 3:
result = create_execution(client, handle, bulk_array[2],
overwrite)
else:
if len(bulk_array) >= 4 and bulk_array[3].lower() != 'none':
checksum = bulk_array[3]
if len(bulk_array) >= 5 and bulk_array[4].lower() != 'none':
loc10320 = bulk_array[4]
if len(bulk_array) >= 6 and bulk_array[5].lower() != 'none':
extratype = bulk_array[5]
result = create_execution(client, handle, bulk_array[2],
overwrite, checksum, loc10320,
extratype)
bulk_result_file.write('create handle: ' + bulk_array[1] +
' result: ' + result + '\n')
if bulk_array[0] == 'DELETE':
# DELETE handle # delete whole handle
# DELETE handle key # delete key/value pair from handle
delete_handle = bulk_array[1]
if len(bulk_array) >= 3:
delete_key = bulk_array[2]
result = delete_execution(client, delete_handle, delete_key)
bulk_result_file.write('delete handle: ' + delete_handle +
' key: ' + delete_key + ' result: ' +
result + '\n')
else:
result = delete_execution(client, delete_handle)
bulk_result_file.write('delete handle: ' + delete_handle +
' result: ' + result + '\n')
if bulk_array[0] == 'MODIFY':
# MODIFY handle key value # modify key/value pair in handle
if len(bulk_array) == 4:
modify_handle = bulk_array[1]
modify_key = bulk_array[2]
modify_value = bulk_array[3]
result = modify_execution(client, modify_handle, modify_key,
modify_value)
bulk_result_file.write('modify handle: ' + modify_handle +
' key: ' + modify_key + ' value: ' +
modify_value + ' result: ' + result +
'\n')
if bulk_array[0] == 'REPLACE':
# REPLACE handle key data1 data2 # replace data1 with data2 in value part of key/value pair in handle
if len(bulk_array) == 5:
replace_handle = bulk_array[1]
replace_key = bulk_array[2]
replace_data1 = bulk_array[3]
replace_data2 = bulk_array[4]
result = read_execution(client, replace_handle, replace_key)
if result != "None" and result != 'error':
new_value = result.replace(replace_data1, replace_data2)
if result != new_value:
result = modify_execution(client, replace_handle,
replace_key, new_value)
else:
result = "None"
bulk_result_file.write('replace handle: ' + replace_handle +
' key: ' + replace_key + ' data1: ' +
replace_data1 + ' data2: ' +
replace_data2 + ' result: ' + result +
'\n')
bulk_input_file.close()
bulk_result_file.close()
def test_credentials_from_json_broken_syntax(self):
""""""
path_to_json_credentials = PATH_CRED+'/credentials_brokensyntax_PUBLIC.json'
with self.assertRaises(CredentialsFormatError):
_inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
def test_credentials_from_json(self):
"""Test credentials instantiation from JSON file."""
path_to_json_credentials = PATH_CRED+'/credentials_correct_PUBLIC.json'
inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
self.assertIsInstance(inst, PIDClientCredentials)
def test_credentials_from_json_username_without_index(self):
"""Exception occurs if user name in json file does not have an index."""
path_to_json_credentials = PATH_CRED+'/credentials_noindex_PUBLIC.json'
with self.assertRaises(HandleSyntaxError):
_inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
def test_credentials_from_json_invalid_username(self):
"""Exception occurs if user name in json file is not index:prefix/suffix."""
path_to_json_credentials = PATH_CRED+'/credentials_wrongusername_PUBLIC.json'
with self.assertRaises(HandleSyntaxError):
_inst = PIDClientCredentials.load_from_JSON(path_to_json_credentials)
def process(options,pstat,OUT):
## process (options,pstat,OUT) - function
# Starts processing as specified in pstat['tbd'] and
# according the request list given bey the options
#
# Parameters:
# -----------
# 1. options (OptionsParser object)
# 2. pstat (process status dict)
#
# set list of request lsits for single or multi mode:
mode = None
procOptions=['community','source','verb','mdprefix','mdsubset','target_mdschema']
if(options.source):
mode = 'single'
mandParams=['community','verb','mdprefix'] # mandatory processing params
for param in mandParams :
if not getattr(options,param) :
logger.critical("Processing parameter %s is required in single mode" % param)
sys.exit(-1)
reqlist=[[
options.community,
options.source,
options.verb,
options.mdprefix,
options.mdsubset,
options.ckan_check,
options.handle_check,
options.target_mdschema
]]
elif(options.list):
mode = 'multi'
logger.debug(' |- Joblist: \t%s' % options.list)
reqlist=parse_list_file(options)
logger.debug(' |- Requestlist: \t%s' % reqlist)
## check job request (processing) options
logger.debug('|- Command line options')
for opt in procOptions :
if hasattr(options,opt) : logger.debug(' |- %s:\t%s' % (opt.upper(),getattr(options,opt)))
## GENERATION mode:
if (pstat['status']['g'] == 'tbd'):
logger.info('\n|- Generation started : %s' % time.strftime("%Y-%m-%d %H:%M:%S"))
GEN = Generator(OUT,pstat,options.outdir)
process_generate(GEN,reqlist)
## HARVESTING mode:
if (pstat['status']['h'] == 'tbd'):
logger.info('\n|- Harvesting started : %s' % time.strftime("%Y-%m-%d %H:%M:%S"))
HV = Harvester(OUT,pstat,options.outdir,options.fromdate)
process_harvest(HV,reqlist)
## MAPPINING - Mode:
if (pstat['status']['m'] == 'tbd'):
logger.info('\n|- Mapping started : %s' % time.strftime("%Y-%m-%d %H:%M:%S"))
MP = Mapper(OUT,options.outdir,options.fromdate)
process_map(MP,reqlist)
## VALIDATING - Mode:
if (pstat['status']['v'] == 'tbd'):
logger.info(' |- Validating started : %s' % time.strftime("%Y-%m-%d %H:%M:%S"))
VD = Validator(OUT,options.outdir,options.fromdate)
process_validate(VD,reqlist)
## CONVERTING - Mode:
if (pstat['status']['c'] == 'tbd'):
logger.info('\n|- Converting started : %s' % time.strftime("%Y-%m-%d %H:%M:%S"))
CV = Converter(OUT,options.outdir,options.fromdate)
process_convert(CV, reqlist)
## UPLOADING - Mode:
if (pstat['status']['u'] == 'tbd'):
logger.info('\n|- Uploading started : %s' % time.strftime("%Y-%m-%d %H:%M:%S"))
# create CKAN object
CKAN = CKAN_CLIENT(options.iphost,options.auth)
# create credentials and handle client if required
if (options.handle_check):
try:
cred = PIDClientCredentials.load_from_JSON('credentials_11098')
except Exception as err:
logger.critical("%s : Could not create credentials from credstore %s" % (err,options.handle_check))
##p.print_help()
sys.exit(-1)
else:
logger.debug("Create EUDATHandleClient instance")
HandleClient = EUDATHandleClient.instantiate_with_credentials(cred)
else:
cred=None
HandleClient=None
UP = Uploader(CKAN,options.ckan_check,HandleClient,cred,OUT,options.outdir,options.fromdate,options.iphost)
logger.info(' |- Host: \t%s' % CKAN.ip_host )
process_upload(UP, reqlist)
## DELETING - Mode:
if (pstat['status']['d'] == 'tbd'):
# start the process deleting:
logger.info('\n|- Deleting started : %s' % time.strftime("%Y-%m-%d %H:%M:%S"))
if mode is 'multi':
dir = options.outdir+'/delete'
if os.path.exists(dir):
process_delete(OUT, dir, options)
else:
logger.error('[ERROR] The directory "%s" does not exist! No files for deleting are found!' % (dir))
else:
logger.critical("[CRITICAL] Deleting mode only supported in 'multi mode' and an explicitly deleting script given !")
dataset_dict["CHECKSUM"]=hashlib.md5(json.dumps(dataset_dict, encoding='latin1').strip()).hexdigest()
# Verbose information about the data
if args.verbose > 0 : print(' The dataset to upload:\n\t%s' % dataset_dict)
# Use the json module to dump the dictionary to a string for posting.
data_string = urllib.quote(json.dumps(dataset_dict))
# By default package_create function is used to create a new dataset.
request = urllib2.Request(
'http://'+args.ipadress+'/api/action/package_create')
# Create a handle client and check handle if required
handlestatus="unknown"
if (args.handle_check):
try:
cred = PIDClientCredentials.load_from_JSON(args.handle_check)
client = EUDATHandleClient.instantiate_with_credentials(cred)
except Exception as err:
print ("%s : Could not create credentials from credstore %s" % (err,args.handle_check))
sys.exit(-1)
else:
if args.verbose > 1 : print ("HandleClient created")
pidRecord=dict()
try:
pid = cred.get_prefix() + '/eudat-jmd_' + dataset_dict['name']
rec = client.retrieve_handle_record_json(pid)
except Exception as err :
print ("ERROR : %s in client.retrieve_handle_record_json()" % (err))
else:
if args.verbose > 0 : print(" Retrieved Handle %s with\n |%-12s\t|%-30s\t|%-30s|\n %s" % (pid,'Attribute','Value','Changed value',80*'-'))
def process(options, pstat, OUT):
## process (options,pstat,OUT) - function
# Starts processing as specified in pstat['tbd'] and
# according the request list given bey the options
#
# Parameters:
# -----------
# 1. options (OptionsParser object)
# 2. pstat (process status dict)
#
# set list of request lsits for single or multi mode:
mode = None
procOptions = [
'community', 'source', 'verb', 'mdprefix', 'mdsubset',
'target_mdschema'
]
if (options.source):
mode = 'single'
mandParams = ['community', 'verb',
'mdprefix'] # mandatory processing params
for param in mandParams:
if not getattr(options, param):
logger.critical(
"Processing parameter %s is required in single mode" %
param)
sys.exit(-1)
reqlist = [[
options.community, options.source, options.verb, options.mdprefix,
options.mdsubset, options.ckan_check, options.handle_check,
options.target_mdschema
]]
elif (options.list):
mode = 'multi'
logger.debug(' |- Joblist: \t%s' % options.list)
reqlist = parse_list_file(options)
logger.debug(' |- Requestlist: \t%s' % reqlist)
## check job request (processing) options
logger.debug('|- Command line options')
for opt in procOptions:
if hasattr(options, opt):
logger.debug(' |- %s:\t%s' % (opt.upper(), getattr(options, opt)))
## HARVESTING mode:
if (pstat['status']['h'] == 'tbd'):
logger.info('\n|- Harvesting started : %s' %
time.strftime("%Y-%m-%d %H:%M:%S"))
HV = Harvester(OUT, pstat, options.outdir, options.fromdate)
process_harvest(HV, reqlist)
## MAPPINING - Mode:
if (pstat['status']['m'] == 'tbd'):
logger.info('\n|- Mapping started : %s' %
time.strftime("%Y-%m-%d %H:%M:%S"))
MP = Mapper(OUT, options.outdir, options.fromdate)
process_map(MP, reqlist)
## VALIDATING - Mode:
if (pstat['status']['v'] == 'tbd'):
logger.info(' |- Validating started : %s' %
time.strftime("%Y-%m-%d %H:%M:%S"))
MP = Mapper(OUT, options.outdir, options.fromdate)
process_validate(MP, reqlist)
## OAI-CONVERTING - Mode:
if (pstat['status']['o'] == 'tbd'):
logger.info('\n|- OAI-Converting started : %s' %
time.strftime("%Y-%m-%d %H:%M:%S"))
MP = Mapper(OUT, options.outdir, options.fromdate)
process_oaiconvert(MP, reqlist)
## UPLOADING - Mode:
if (pstat['status']['u'] == 'tbd'):
logger.info('\n|- Uploading started : %s' %
time.strftime("%Y-%m-%d %H:%M:%S"))
# create CKAN object
CKAN = CKAN_CLIENT(options.iphost, options.auth)
# create credentials and handle client if required
if (options.handle_check):
try:
cred = PIDClientCredentials.load_from_JSON('credentials_11098')
except Exception as err:
logger.critical(
"%s : Could not create credentials from credstore %s" %
(err, options.handle_check))
##p.print_help()
sys.exit(-1)
else:
logger.debug("Create EUDATHandleClient instance")
HandleClient = EUDATHandleClient.instantiate_with_credentials(
cred)
else:
cred = None
HandleClient = None
UP = Uploader(CKAN, options.ckan_check, HandleClient, cred, OUT,
options.outdir, options.fromdate, options.iphost)
logger.info(' |- Host: \t%s' % CKAN.ip_host)
process_upload(UP, reqlist)
## DELETING - Mode:
if (pstat['status']['d'] == 'tbd'):
# start the process deleting:
logger.info('\n|- Deleting started : %s' %
time.strftime("%Y-%m-%d %H:%M:%S"))
if mode is 'multi':
dir = options.outdir + '/delete'
if os.path.exists(dir):
process_delete(OUT, dir, options)
else:
logger.error(
'[ERROR] The directory "%s" does not exist! No files for deleting are found!'
% (dir))
else:
logger.critical(
"[CRITICAL] Deleting mode only supported in 'multi mode' and an explicitly deleting script given !"
)
def test_credentials_from_json_broken_syntax(self):
""""""
path_to_json_credentials = PATH_CRED + '/credentials_brokensyntax_PUBLIC.json'
with self.assertRaises(CredentialsFormatError):
_inst = PIDClientCredentials.load_from_JSON(
path_to_json_credentials)