def __init__(self,
db_host,
db_user,
db_password,
database_name,
admin_handle,
output_batch_filename,
queried_prefixes,
fixed_content,
input_batch_file=None,
handle_key_file=None,
handle_secret_key=None,
handle_server_url=None):
super(MigrationTool, self).__init__()
self.db_host = db_host
self.db_user = db_user
self.db_password = db_password
self.database_name = database_name
self.admin_handle = admin_handle
self.key_file = handle_key_file
self.secret_key = handle_secret_key
self.input_batch_filename = input_batch_file
self.output_batch_filename = output_batch_filename
self.queried_prefixes = queried_prefixes
if fixed_content:
self.fixed_content = "TRUE"
else:
self.fixed_content = "FALSE"
self.all_handles = []
self.handle_server_url = handle_server_url
self.b2handleclient = EUDATHandleClient.instantiate_for_read_access()
self.b2handleclient = EUDATHandleClient.instantiate_for_read_access(
handle_server_url)
def read(args):
"""perform read action"""
# load credentials
credentials = PIDClientCredentials.load_from_JSON(args.credpath)
# retrieve and set extra values
extra_config = {}
# setup connection to handle server
client = EUDATHandleClient.instantiate_for_read_access(
credentials.get_server_URL(), **extra_config)
# set default return value
json_result = "None"
if args.key is None:
# retrieve whole handle
result = client.retrieve_handle_record_json(args.handle)
if result is not None:
json_result = json.dumps(result["values"])
else:
# retrieve single value from a handle
result = client.get_value_from_handle(args.handle, args.key)
if result is not None:
json_result = json.dumps(result)
# remove starting and finishing quotes.
json_result = json_result.lstrip('"')
json_result = json_result.rstrip('"')
sys.stdout.write(json_result)
def download_dataset(pid, destination):
#Instantiate client for reading --> credentials necessary
ec = EUDATHandleClient.instantiate_for_read_access(
'https://hdl.handle.net')
record = ec.retrieve_handle_record(pid)
assert record != None
assert 'URL' in record
assert 'PROTOCOL' in record
assert 'SITE' in record
protocol = record['PROTOCOL']
site = record['SITE']
source = record['URL']
print GREEN, "DEBUG", DEFAULT, \
"PID", pid, "resolves to", protocol+"://"+site+source
exit_code = subprocess.call(["grid-proxy-init"])
print GREEN, "DEBUG downloading:"
exit_code = subprocess.call(
["globus-url-copy", "-list", protocol + "://" + site + source])
print "Destination", destination
print DEFAULT
exit_code = subprocess.call([
"globus-url-copy", "-cd", "-r", protocol + "://" + site + source,
destination
])
return exit_code
def check_issue_pid(self):
# Checking PID HANDLE
print('checkin PIDs')
sleep(5)
handle_client = EUDATHandleClient.instantiate_for_read_access()
self.uid = self.issue['uid']
for line in self.dsets:
print('CHECKING {} ERRATA IDS'.format(line))
exists = False
dataset = line.split('#')
dset_id = dataset[0]
dset_version = dataset[1]
hash_basis = dset_id+'.v'+dset_version
hash_basis_utf8 = hash_basis.encode('utf-8')
handle_string = uuid.uuid3(uuid.NAMESPACE_URL, hash_basis_utf8)
encoded_dict = handle_client.retrieve_handle_record(prefix + str(handle_string))
if encoded_dict is not None:
handle_record = {k.decode('utf8'): v.decode('utf8') for k, v in encoded_dict.items()}
if 'ERRATA_IDS' in handle_record.keys():
for uid in str(handle_record['ERRATA_IDS']).split(';'):
if uid == self.uid:
exists = True
break
if not exists:
print('An error occurred updating handle.')
return exists
def connect_client(self, force_no_credentials=False, disable_logs=False):
if getattr(self, '_handle_client', None) is None:
if disable_logs:
import logging
logging.getLogger('b2handle').setLevel(logging.WARNING)
# With credentials
if force_no_credentials:
self._handle_client = b2handle.instantiate_for_read_access()
log.debug("HANDLE client connected [w/out credentials]")
else:
found = False
file = os.environ.get('HANDLE_CREDENTIALS', None)
if file is not None:
from utilities import path
credentials_path = path.build(file)
found = path.file_exists_and_nonzero(credentials_path)
if not found:
log.warning("B2HANDLE credentials file not found %s",
file)
if found:
self._handle_client = \
b2handle.instantiate_with_credentials(
credentials.load_from_JSON(file)
)
log.debug("HANDLE client connected [w/ credentials]")
return self._handle_client, True
return self._handle_client, False
def _check_handle_status(dataset_id):
"""Checks handle exists or not.
:returns: Flag indicating whether handle status is such that it requires fiurther processing
:rtype: bool
"""
# Get handle information.
handle_string = resolve_input(dataset_id)
handle_client = EUDATHandleClient.instantiate_for_read_access()
encoded_dict = handle_client.retrieve_handle_record(handle_string)
# Error if not found.
if encoded_dict is None:
raise exceptions.HandleMismatch('Dataset {} has no published pid handle'.format(dataset_id))
# Reformat handle information.
handle_record = {k.decode('utf8'): v.decode('utf8') for k, v in encoded_dict.items()}
# Error if handle has no test value.
if '_TEST' not in handle_record.keys():
logger.log_pid('Dataset handle does not have test value, assuming not test...')
# raise exceptions.HandleMismatch('TEST VALUE WAS NOT FOUND IN HANDLE, ABORTING....')
else:
# Error if handle record value.
if handle_record['_TEST'].lower() != str(config.pid.is_test).lower():
raise exceptions.HandleMismatch('Dataset {} has mismatched test status [{}] with pid connector'.format(dataset_id, handle_record['_TEST']))
def test_instantiate_for_read_access(self):
"""Testing if instantiating with default handle server works
and if a handle is correctly retrieved. """
# Create client instance with username and password
inst = EUDATHandleClient.instantiate_for_read_access(HTTPS_verify=self.https_verify)
rec = self.inst.retrieve_handle_record_json(self.handle)
self.assertIsInstance(inst, EUDATHandleClient)
self.assertIn("handle", rec, 'Response lacks "handle".')
self.assertIn("responseCode", rec, 'Response lacks "responseCode".')
def test_instantiate_for_read_access(self):
"""Testing if instantiating with default handle server works
and if a handle is correctly retrieved. """
# Create client instance with username and password
inst = EUDATHandleClient.instantiate_for_read_access(
HTTPS_verify=self.https_verify)
rec = self.inst.retrieve_handle_record_json(self.handle)
self.assertIsInstance(inst, EUDATHandleClient)
self.assertIn('handle', rec, 'Response lacks "handle".')
self.assertIn('responseCode', rec, 'Response lacks "responseCode".')
def main():
""" Main function to test the script """
client = EUDATHandleClient.instantiate_for_read_access()
value = client.get_value_from_handle("11100/33ac01fc-6850-11e5-b66e-e41f13eb32b2", "URL")
print value
result = client.search_handle("irods://data.repo.cineca.it:1247/CINECA01/home/cin_staff/rmucci00/DSI_Test/test.txt")
print result
get_pid_info(pid='11100/0beb6af8-cbe5-11e3-a9da-e41f13eb41b2')
def harvest_errata_information(input_handle_string):
"""Given a handle, this will harvest all the errata data related to that handle as well as the previous versions.
:param input_handle_string: Handle identifier
:return: errata information, dset/file_id
"""
tick = time()
logger.log_pid("--CREATING HANDLE CLIENT--")
handle_client = EUDATHandleClient.instantiate_for_read_access()
logger.log_pid("--HANDLE CLIENT CREATED--")
logger.log_pid("----------------------------------BEGIN ISSUE TRACKING----------------------------------")
handle = get_handle_by_handle_string(input_handle_string, handle_client)
list_of_uids, incomplete_search = crawler(handle, handle_client)
logger.log_pid("ELAPSED TIME TILL COMPLETION : " + str(time()-tick) + " SECONDS")
logger.log_pid("-----------------------------------END ISSUE TRACKING-----------------------------------")
logger.log_pid("LIST OF UIDS GENERATED IS...")
logger.log_pid(list_of_uids)
return list_of_uids, incomplete_search
def connect_client(self, force_no_credentials=False):
found = False
# With credentials
if not force_no_credentials:
file = os.environ.get('HANDLE_CREDENTIALS', None)
if file is not None:
from utilities import path
credentials_path = path.build(file)
found = path.file_exists_and_nonzero(credentials_path)
if not found:
log.warning("B2HANDLE credentials file not found %s", file)
if found:
client = b2handle.instantiate_with_credentials(
credentials.load_from_JSON(file))
log.info("PID client connected: w/ credentials")
return client, True
client = b2handle.instantiate_for_read_access()
log.warning("PID client connected: NO credentials")
return client, False
def harvest_simple_errata(input_handle_string):
"""
A simplified version of the harvest original implementation.
:param input_handle_string: pid handle string
:return: errata_id list, empty if no errata is found.
"""
output = []
logger.log_pid("--CREATING HANDLE CLIENT--")
handle_client = EUDATHandleClient.instantiate_for_read_access()
logger.log_pid("--SUCCESSFULLY CREATED CLIENT--")
logger.log_pid("--RETRIEVING HANDLE FROM PID SERVER--")
handle = get_handle_by_handle_string(input_handle_string, handle_client)
if handle is not None:
if ERRATA_IDS in handle.keys():
output = str(handle[ERRATA_IDS].split(";"))
drs_id = handle[DRS]
version = handle[VERSION]
else:
logger.log_pid("--HANDLE NOT FOUND IN PID SERVER--")
return input_handle_string, drs_id, version, output, len(output) >= 1, len(drs_id) > 1
# Dataset A
# harvest_errata_information('21.14100/aae01ba2-8436-378d-84ed-5a06b9fbee46')
# Dataset B:
# harvest_errata_information('21.14100/37043d8e-ac5e-3843-a019-c03017cc68aa')
# Dataset C:
# harvest_errata_information('21.14100/e0560a9d-2227-3175-b943-fc26c427a923')
# Dataset D:
# harvest_errata_information('21.14100/bc3d4e81-bfbd-3a3f-a99f-4a2ec64b5962')
# temperature file
# harvest_errata_information('21.14100/d9053480-0e0d-11e6-a148-3e1d05defe78')
# rainfall file
# harvest_errata_information('21.14100/28ju73be-0e10-11e6-a148-a7751ce7ec0c')
# rainfall_1 file
# harvest_errata_information('21.14100/4ba213fc-f688-3d58-bd96-d984bb00f1d5')
# print(harvest_simple_errata('21.14100/4ba213fc-f688-3d58-bd96-d984bb00f1d5'))
from b2handle.handleclient import EUDATHandleClient
import uuid
import logging
import esgfpid
handle_client = EUDATHandleClient.instantiate_for_read_access()
def check_drs(drs_id):
drs_id = drs_id.split('#')
prefix = '21.14100/'
hash_basis = drs_id[0]+'.v'+drs_id[1]
hash_basis_utf8 = hash_basis.encode('utf-8')
handle_string = uuid.uuid3(uuid.NAMESPACE_URL, hash_basis_utf8)
encoded_dict = handle_client.retrieve_handle_record(prefix + str(handle_string))
if encoded_dict is not None:
return True
else:
print('Handle not found.')
return False
def clear_handle(drs_id, connector):
drs_id = drs_id.split('#')
prefix = '21.14100/'
hash_basis = drs_id[0]+'.v'+drs_id[1]
hash_basis_utf8 = hash_basis.encode('utf-8')
handle_string = uuid.uuid3(uuid.NAMESPACE_URL, hash_basis_utf8)
encoded_dict = handle_client.retrieve_handle_record(prefix + str(handle_string))
if encoded_dict is not None:
handle_record = {k.decode('utf8'): v.decode('utf8') for k, v in encoded_dict.items()}
def test_instantiate_for_read_access(self):
"""Testing if instantiating with default handle server works. """
# Create client instance with username and password
inst = EUDATHandleClient.instantiate_for_read_access()
self.assertIsInstance(inst, EUDATHandleClient)
def test_instantiate_for_read_access(self):
"""Testing if instantiating with default handle server works. """
# Create client instance with username and password
inst = EUDATHandleClient.instantiate_for_read_access()
self.assertIsInstance(inst, EUDATHandleClient)
