def __init__(self, db_host, db_user, db_password, database_name, admin_handle, output_batch_filename, queried_prefixes, fixed_content, input_batch_file=None, handle_key_file=None, handle_secret_key=None, handle_server_url=None): super(MigrationTool, self).__init__() self.db_host = db_host self.db_user = db_user self.db_password = db_password self.database_name = database_name self.admin_handle = admin_handle self.key_file = handle_key_file self.secret_key = handle_secret_key self.input_batch_filename = input_batch_file self.output_batch_filename = output_batch_filename self.queried_prefixes = queried_prefixes if fixed_content: self.fixed_content = "TRUE" else: self.fixed_content = "FALSE" self.all_handles = [] self.handle_server_url = handle_server_url self.b2handleclient = EUDATHandleClient.instantiate_for_read_access() self.b2handleclient = EUDATHandleClient.instantiate_for_read_access( handle_server_url)
def read(args): """perform read action""" # load credentials credentials = PIDClientCredentials.load_from_JSON(args.credpath) # retrieve and set extra values extra_config = {} # setup connection to handle server client = EUDATHandleClient.instantiate_for_read_access( credentials.get_server_URL(), **extra_config) # set default return value json_result = "None" if args.key is None: # retrieve whole handle result = client.retrieve_handle_record_json(args.handle) if result is not None: json_result = json.dumps(result["values"]) else: # retrieve single value from a handle result = client.get_value_from_handle(args.handle, args.key) if result is not None: json_result = json.dumps(result) # remove starting and finishing quotes. json_result = json_result.lstrip('"') json_result = json_result.rstrip('"') sys.stdout.write(json_result)
def download_dataset(pid, destination): #Instantiate client for reading --> credentials necessary ec = EUDATHandleClient.instantiate_for_read_access( 'https://hdl.handle.net') record = ec.retrieve_handle_record(pid) assert record != None assert 'URL' in record assert 'PROTOCOL' in record assert 'SITE' in record protocol = record['PROTOCOL'] site = record['SITE'] source = record['URL'] print GREEN, "DEBUG", DEFAULT, \ "PID", pid, "resolves to", protocol+"://"+site+source exit_code = subprocess.call(["grid-proxy-init"]) print GREEN, "DEBUG downloading:" exit_code = subprocess.call( ["globus-url-copy", "-list", protocol + "://" + site + source]) print "Destination", destination print DEFAULT exit_code = subprocess.call([ "globus-url-copy", "-cd", "-r", protocol + "://" + site + source, destination ]) return exit_code
def check_issue_pid(self): # Checking PID HANDLE print('checkin PIDs') sleep(5) handle_client = EUDATHandleClient.instantiate_for_read_access() self.uid = self.issue['uid'] for line in self.dsets: print('CHECKING {} ERRATA IDS'.format(line)) exists = False dataset = line.split('#') dset_id = dataset[0] dset_version = dataset[1] hash_basis = dset_id+'.v'+dset_version hash_basis_utf8 = hash_basis.encode('utf-8') handle_string = uuid.uuid3(uuid.NAMESPACE_URL, hash_basis_utf8) encoded_dict = handle_client.retrieve_handle_record(prefix + str(handle_string)) if encoded_dict is not None: handle_record = {k.decode('utf8'): v.decode('utf8') for k, v in encoded_dict.items()} if 'ERRATA_IDS' in handle_record.keys(): for uid in str(handle_record['ERRATA_IDS']).split(';'): if uid == self.uid: exists = True break if not exists: print('An error occurred updating handle.') return exists
def connect_client(self, force_no_credentials=False, disable_logs=False): if getattr(self, '_handle_client', None) is None: if disable_logs: import logging logging.getLogger('b2handle').setLevel(logging.WARNING) # With credentials if force_no_credentials: self._handle_client = b2handle.instantiate_for_read_access() log.debug("HANDLE client connected [w/out credentials]") else: found = False file = os.environ.get('HANDLE_CREDENTIALS', None) if file is not None: from utilities import path credentials_path = path.build(file) found = path.file_exists_and_nonzero(credentials_path) if not found: log.warning("B2HANDLE credentials file not found %s", file) if found: self._handle_client = \ b2handle.instantiate_with_credentials( credentials.load_from_JSON(file) ) log.debug("HANDLE client connected [w/ credentials]") return self._handle_client, True return self._handle_client, False
def _check_handle_status(dataset_id): """Checks handle exists or not. :returns: Flag indicating whether handle status is such that it requires fiurther processing :rtype: bool """ # Get handle information. handle_string = resolve_input(dataset_id) handle_client = EUDATHandleClient.instantiate_for_read_access() encoded_dict = handle_client.retrieve_handle_record(handle_string) # Error if not found. if encoded_dict is None: raise exceptions.HandleMismatch('Dataset {} has no published pid handle'.format(dataset_id)) # Reformat handle information. handle_record = {k.decode('utf8'): v.decode('utf8') for k, v in encoded_dict.items()} # Error if handle has no test value. if '_TEST' not in handle_record.keys(): logger.log_pid('Dataset handle does not have test value, assuming not test...') # raise exceptions.HandleMismatch('TEST VALUE WAS NOT FOUND IN HANDLE, ABORTING....') else: # Error if handle record value. if handle_record['_TEST'].lower() != str(config.pid.is_test).lower(): raise exceptions.HandleMismatch('Dataset {} has mismatched test status [{}] with pid connector'.format(dataset_id, handle_record['_TEST']))
def test_instantiate_for_read_access(self): """Testing if instantiating with default handle server works and if a handle is correctly retrieved. """ # Create client instance with username and password inst = EUDATHandleClient.instantiate_for_read_access(HTTPS_verify=self.https_verify) rec = self.inst.retrieve_handle_record_json(self.handle) self.assertIsInstance(inst, EUDATHandleClient) self.assertIn("handle", rec, 'Response lacks "handle".') self.assertIn("responseCode", rec, 'Response lacks "responseCode".')
def test_instantiate_for_read_access(self): """Testing if instantiating with default handle server works and if a handle is correctly retrieved. """ # Create client instance with username and password inst = EUDATHandleClient.instantiate_for_read_access( HTTPS_verify=self.https_verify) rec = self.inst.retrieve_handle_record_json(self.handle) self.assertIsInstance(inst, EUDATHandleClient) self.assertIn('handle', rec, 'Response lacks "handle".') self.assertIn('responseCode', rec, 'Response lacks "responseCode".')
def main(): """ Main function to test the script """ client = EUDATHandleClient.instantiate_for_read_access() value = client.get_value_from_handle("11100/33ac01fc-6850-11e5-b66e-e41f13eb32b2", "URL") print value result = client.search_handle("irods://data.repo.cineca.it:1247/CINECA01/home/cin_staff/rmucci00/DSI_Test/test.txt") print result get_pid_info(pid='11100/0beb6af8-cbe5-11e3-a9da-e41f13eb41b2')
def harvest_errata_information(input_handle_string): """Given a handle, this will harvest all the errata data related to that handle as well as the previous versions. :param input_handle_string: Handle identifier :return: errata information, dset/file_id """ tick = time() logger.log_pid("--CREATING HANDLE CLIENT--") handle_client = EUDATHandleClient.instantiate_for_read_access() logger.log_pid("--HANDLE CLIENT CREATED--") logger.log_pid("----------------------------------BEGIN ISSUE TRACKING----------------------------------") handle = get_handle_by_handle_string(input_handle_string, handle_client) list_of_uids, incomplete_search = crawler(handle, handle_client) logger.log_pid("ELAPSED TIME TILL COMPLETION : " + str(time()-tick) + " SECONDS") logger.log_pid("-----------------------------------END ISSUE TRACKING-----------------------------------") logger.log_pid("LIST OF UIDS GENERATED IS...") logger.log_pid(list_of_uids) return list_of_uids, incomplete_search
def connect_client(self, force_no_credentials=False): found = False # With credentials if not force_no_credentials: file = os.environ.get('HANDLE_CREDENTIALS', None) if file is not None: from utilities import path credentials_path = path.build(file) found = path.file_exists_and_nonzero(credentials_path) if not found: log.warning("B2HANDLE credentials file not found %s", file) if found: client = b2handle.instantiate_with_credentials( credentials.load_from_JSON(file)) log.info("PID client connected: w/ credentials") return client, True client = b2handle.instantiate_for_read_access() log.warning("PID client connected: NO credentials") return client, False
def harvest_simple_errata(input_handle_string): """ A simplified version of the harvest original implementation. :param input_handle_string: pid handle string :return: errata_id list, empty if no errata is found. """ output = [] logger.log_pid("--CREATING HANDLE CLIENT--") handle_client = EUDATHandleClient.instantiate_for_read_access() logger.log_pid("--SUCCESSFULLY CREATED CLIENT--") logger.log_pid("--RETRIEVING HANDLE FROM PID SERVER--") handle = get_handle_by_handle_string(input_handle_string, handle_client) if handle is not None: if ERRATA_IDS in handle.keys(): output = str(handle[ERRATA_IDS].split(";")) drs_id = handle[DRS] version = handle[VERSION] else: logger.log_pid("--HANDLE NOT FOUND IN PID SERVER--") return input_handle_string, drs_id, version, output, len(output) >= 1, len(drs_id) > 1 # Dataset A # harvest_errata_information('21.14100/aae01ba2-8436-378d-84ed-5a06b9fbee46') # Dataset B: # harvest_errata_information('21.14100/37043d8e-ac5e-3843-a019-c03017cc68aa') # Dataset C: # harvest_errata_information('21.14100/e0560a9d-2227-3175-b943-fc26c427a923') # Dataset D: # harvest_errata_information('21.14100/bc3d4e81-bfbd-3a3f-a99f-4a2ec64b5962') # temperature file # harvest_errata_information('21.14100/d9053480-0e0d-11e6-a148-3e1d05defe78') # rainfall file # harvest_errata_information('21.14100/28ju73be-0e10-11e6-a148-a7751ce7ec0c') # rainfall_1 file # harvest_errata_information('21.14100/4ba213fc-f688-3d58-bd96-d984bb00f1d5') # print(harvest_simple_errata('21.14100/4ba213fc-f688-3d58-bd96-d984bb00f1d5'))
from b2handle.handleclient import EUDATHandleClient import uuid import logging import esgfpid handle_client = EUDATHandleClient.instantiate_for_read_access() def check_drs(drs_id): drs_id = drs_id.split('#') prefix = '21.14100/' hash_basis = drs_id[0]+'.v'+drs_id[1] hash_basis_utf8 = hash_basis.encode('utf-8') handle_string = uuid.uuid3(uuid.NAMESPACE_URL, hash_basis_utf8) encoded_dict = handle_client.retrieve_handle_record(prefix + str(handle_string)) if encoded_dict is not None: return True else: print('Handle not found.') return False def clear_handle(drs_id, connector): drs_id = drs_id.split('#') prefix = '21.14100/' hash_basis = drs_id[0]+'.v'+drs_id[1] hash_basis_utf8 = hash_basis.encode('utf-8') handle_string = uuid.uuid3(uuid.NAMESPACE_URL, hash_basis_utf8) encoded_dict = handle_client.retrieve_handle_record(prefix + str(handle_string)) if encoded_dict is not None: handle_record = {k.decode('utf8'): v.decode('utf8') for k, v in encoded_dict.items()}
def test_instantiate_for_read_access(self): """Testing if instantiating with default handle server works. """ # Create client instance with username and password inst = EUDATHandleClient.instantiate_for_read_access() self.assertIsInstance(inst, EUDATHandleClient)