def __handle_terminated_replicas(replicas, prepend_str=''): """ Used by finisher to handle available and unavailable replicas. :param replicas: List of replicas. :param prepend_str: String to prepend to logging. """ for req_type in replicas: for rule_id in replicas[req_type]: try: __update_bulk_replicas(replicas[req_type][rule_id]) except (UnsupportedOperation, ReplicaNotFound): # one replica in the bulk cannot be found. register it one by one logging.warn( '%s Problem to bulk update the replicas states. Will try one by one', prepend_str) for replica in replicas[req_type][rule_id]: try: __update_replica(replica) except (DatabaseException, DatabaseError) as error: if re.match('.*ORA-00054.*', error.args[0]) or re.match( '.*ORA-00060.*', error.args[0] ) or 'ERROR 1205 (HY000)' in error.args[0]: logging.warn( "%s Locks detected when handling replica %s:%s at RSE %s", prepend_str, replica['scope'], replica['name'], replica['rse_id']) else: logging.error( "%s Could not finish handling replicas %s:%s at RSE %s (%s)", prepend_str, replica['scope'], replica['name'], replica['rse_id'], traceback.format_exc()) except Exception as error: logging.error( "%s Something unexpected happened when updating replica state for transfer %s:%s at %s (%s)", prepend_str, replica['scope'], replica['name'], replica['rse_id'], str(error)) except (DatabaseException, DatabaseError) as error: if re.match('.*ORA-00054.*', error.args[0]) or re.match( '.*ORA-00060.*', error.args[0] ) or 'ERROR 1205 (HY000)' in error.args[0]: logging.warn( "%s Locks detected when handling replicas on %s rule %s, update updated time.", prepend_str, req_type, rule_id) try: request_core.touch_requests_by_rule(rule_id) except (DatabaseException, DatabaseError): logging.error( "%s Failed to touch requests by rule(%s): %s", prepend_str, rule_id, traceback.format_exc()) else: logging.error( "%s Could not finish handling replicas on %s rule %s: %s", prepend_str, req_type, rule_id, traceback.format_exc()) except Exception as error: logging.error( "%s Something unexpected happened when handling replicas on %s rule %s: %s", prepend_str, req_type, rule_id, str(error))
def __handle_terminated_replicas(replicas): """ Used by finisher to handle available and unavailable replicas. :param replicas: List of replicas. """ for req_type in replicas: for rule_id in replicas[req_type]: try: __update_bulk_replicas(replicas[req_type][rule_id], req_type, rule_id) except (UnsupportedOperation, ReplicaNotFound): # one replica in the bulk cannot be found. register it one by one for replica in replicas[req_type][rule_id]: try: __update_replica(replica, req_type, rule_id) except (DatabaseException, DatabaseError) as error: if isinstance(error.args[0], tuple) and ( re.match('.*ORA-00054.*', error.args[0][0]) or ('ERROR 1205 (HY000)' in error.args[0][0])): logging.warn( "Locks detected when handling replica %s:%s at RSE %s" % (replica['scope'], replica['name'], replica['rse_id'])) else: logging.error( "Could not finish handling replicas %s:%s at RSE %s (%s)" % (replica['scope'], replica['name'], replica['rse_id'], traceback.format_exc())) except: logging.error( "Something unexpected happened when updating replica state for transfer %s:%s at %s (%s)" % (replica['scope'], replica['name'], replica['rse_id'], traceback.format_exc())) except (DatabaseException, DatabaseError) as error: if isinstance(error.args[0], tuple) and ( re.match('.*ORA-00054.*', error.args[0][0]) or ('ERROR 1205 (HY000)' in error.args[0][0])): logging.warn( "Locks detected when handling replicas on %s rule %s, update updated time." % (req_type, rule_id)) try: request_core.touch_requests_by_rule(rule_id) except (DatabaseException, DatabaseError): logging.error( "Failed to touch requests by rule(%s): %s" % (rule_id, traceback.format_exc())) else: logging.error( "Could not finish handling replicas on %s rule %s: %s" % (req_type, rule_id, traceback.format_exc())) except: logging.error( "Something unexpected happened when handling replicas on %s rule %s: %s" % (req_type, rule_id, traceback.format_exc()))