def list_new_dids(type=None, worker_number=None, total_workers=None, chunk_size=1000):
"""
List recent identifiers.
:param type : The DID type.
"""
return did.list_new_dids(did_type=type and DIDType.from_sym(type), worker_number=worker_number, total_workers=total_workers, chunk_size=chunk_size)
def list_new_dids(type=None, thread=None, total_threads=None, chunk_size=1000):
"""
List recent identifiers.
:param type : The DID type.
"""
return did.list_new_dids(did_type=type and DIDType.from_sym(type), thread=thread, total_threads=total_threads, chunk_size=chunk_size)
def list_new_dids(type=None, thread=None, total_threads=None, chunk_size=1000):
"""
List recent identifiers.
:param type : The DID type.
"""
dids = did.list_new_dids(did_type=type and DIDType.from_sym(type), thread=thread, total_threads=total_threads, chunk_size=chunk_size)
for d in dids:
yield api_update_return_dict(d)
def list_new_dids(did_type=None, thread=None, total_threads=None, chunk_size=1000, vo='def'):
"""
List recent identifiers.
:param did_type : The DID type.
:param thread: The assigned thread for this necromancer.
:param total_threads: The total number of threads of all necromancers.
:param chunk_size: Number of requests to return per yield.
:param vo: The VO to act on.
"""
dids = did.list_new_dids(did_type=did_type and DIDType[did_type.upper()], thread=thread, total_threads=total_threads, chunk_size=chunk_size)
for d in dids:
if d['scope'].vo == vo:
yield api_update_return_dict(d)
def transmogrifier(bulk=5, once=False, sleep_time=60):
"""
Creates a Transmogrifier Worker that gets a list of new DIDs for a given hash,
identifies the subscriptions matching the DIDs and
submit a replication rule for each DID matching a subscription.
:param thread: Thread number at startup.
:param bulk: The number of requests to process.
:param once: Run only once.
:param sleep_time: Time between two cycles.
"""
executable = 'transmogrifier'
hostname = socket.getfqdn()
pid = os.getpid()
hb_thread = threading.current_thread()
heartbeat.sanity_check(executable=executable, hostname=hostname)
while not graceful_stop.is_set():
heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
dids, subscriptions = [], []
tottime = 0
prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'],
heart_beat['nr_threads'])
try:
# Get the new DIDs based on the is_new flag
for did in list_new_dids(thread=heart_beat['assign_thread'],
total_threads=heart_beat['nr_threads'],
chunk_size=bulk,
did_type=None):
dids.append({
'scope': did['scope'],
'did_type': str(did['did_type']),
'name': did['name']
})
sub_dict = {3: []}
# Get the list of subscriptions. The default priority of the subscription is 3. 0 is the highest priority, 5 the lowest
# The priority is defined as 'policyid'
for sub in list_subscriptions(None, None):
if sub['state'] != SubscriptionState.INACTIVE and sub[
'lifetime'] and (datetime.now() > sub['lifetime']):
update_subscription(
name=sub['name'],
account=sub['account'],
metadata={'state': SubscriptionState.INACTIVE})
elif sub['state'] in [
SubscriptionState.ACTIVE, SubscriptionState.UPDATED
]:
priority = 3
if 'policyid' in sub:
if int(sub['policyid']) not in sub_dict:
sub_dict[int(sub['policyid'])] = []
priority = int(sub['policyid'])
sub_dict[priority].append(sub)
priorities = list(sub_dict.keys())
priorities.sort()
# Order the subscriptions according to their priority
for priority in priorities:
subscriptions.extend(sub_dict[priority])
except SubscriptionNotFound as error:
logging.warning(prepend_str + 'No subscriptions defined: %s' %
(str(error)))
time.sleep(10)
continue
except Exception as error:
logging.error(
prepend_str +
'Failed to get list of new DIDs or subscriptions: %s' %
(str(error)))
try:
results = {}
start_time = time.time()
blacklisted_rse_id = [
rse['id'] for rse in list_rses({'availability_write': False})
]
logging.debug(prepend_str + 'In transmogrifier worker')
identifiers = []
# Loop over all the new dids
for did in dids:
did_success = True
if did['did_type'] == str(
DIDType.DATASET) or did['did_type'] == str(
DIDType.CONTAINER):
did_tag = '%s:%s' % (did['scope'].internal, did['name'])
results[did_tag] = []
try:
metadata = get_metadata(did['scope'], did['name'])
# Loop over all the subscriptions
for subscription in subscriptions:
# Check if the DID match the subscription
if is_matching_subscription(
subscription, did, metadata) is True:
filter_string = loads(subscription['filter'])
split_rule = filter_string.get(
'split_rule', False)
stime = time.time()
results[did_tag].append(subscription['id'])
logging.info(prepend_str +
'%s:%s matches subscription %s' %
(did['scope'], did['name'],
subscription['name']))
rules = loads(
subscription['replication_rules'])
created_rules = {}
cnt = 0
for rule_dict in rules:
cnt += 1
created_rules[cnt] = []
# Get all the rule and subscription parameters
grouping = rule_dict.get(
'grouping', 'DATASET')
lifetime = rule_dict.get('lifetime', None)
ignore_availability = rule_dict.get(
'ignore_availability', None)
weight = rule_dict.get('weight', None)
source_replica_expression = rule_dict.get(
'source_replica_expression', None)
locked = rule_dict.get('locked', None)
if locked == 'True':
locked = True
else:
locked = False
purge_replicas = rule_dict.get(
'purge_replicas', False)
if purge_replicas == 'True':
purge_replicas = True
else:
purge_replicas = False
rse_expression = str(
rule_dict['rse_expression'])
comment = str(subscription['comments'])
subscription_id = str(subscription['id'])
account = subscription['account']
copies = int(rule_dict['copies'])
activity = rule_dict.get(
'activity', 'User Subscriptions')
try:
validate_schema(name='activity',
obj=activity)
except InputValidationError as error:
logging.error(
prepend_str +
'Error validating the activity %s'
% (str(error)))
activity = 'User Subscriptions'
if lifetime:
lifetime = int(lifetime)
str_activity = "".join(activity.split())
success = False
nattempt = 5
attemptnr = 0
skip_rule_creation = False
selected_rses = []
chained_idx = rule_dict.get(
'chained_idx', None)
if chained_idx:
params = {}
if rule_dict.get(
'associated_site_idx', None):
params[
'associated_site_idx'] = rule_dict.get(
'associated_site_idx',
None)
logging.debug(
'%s Chained subscription identified. Will use %s',
prepend_str,
str(created_rules[chained_idx]))
algorithm = rule_dict.get(
'algorithm', None)
selected_rses = select_algorithm(
algorithm,
created_rules[chained_idx], params)
else:
# In the case of chained subscription, don't use rseselector but use the rses returned by the algorithm
if split_rule:
vo = account.vo
rses = parse_expression(
rse_expression,
filter={'vo': vo})
list_of_rses = [
rse['id'] for rse in rses
]
# Check that some rule doesn't already exist for this DID and subscription
preferred_rse_ids = []
for rule in list_rules(
filters={
'subscription_id':
subscription_id,
'scope': did['scope'],
'name': did['name']
}):
already_existing_rses = [
(rse['rse'], rse['id']) for
rse in parse_expression(
rule['rse_expression'],
filter={'vo': vo})
]
for rse, rse_id in already_existing_rses:
if (rse_id in list_of_rses
) and (
rse_id not in
preferred_rse_ids):
preferred_rse_ids.append(
rse_id)
if len(preferred_rse_ids
) >= copies:
skip_rule_creation = True
rse_id_dict = {}
for rse in rses:
rse_id_dict[
rse['id']] = rse['rse']
try:
rseselector = RSESelector(
account=account,
rses=rses,
weight=weight,
copies=copies -
len(preferred_rse_ids))
selected_rses = [
rse_id_dict[rse_id]
for rse_id, _, _ in
rseselector.select_rse(
0,
preferred_rse_ids=
preferred_rse_ids,
copies=copies,
blacklist=
blacklisted_rse_id)
]
except (InsufficientTargetRSEs,
InsufficientAccountLimit,
InvalidRuleWeight,
RSEOverQuota) as error:
logging.warning(
prepend_str +
'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blacklisted sites'
% (subscription['name'],
account, str(error)))
# Now including the blacklisted sites
try:
rseselector = RSESelector(
account=account,
rses=rses,
weight=weight,
copies=copies -
len(preferred_rse_ids))
selected_rses = [
rse_id_dict[rse_id]
for rse_id, _, _ in
rseselector.select_rse(
0,
preferred_rse_ids=
preferred_rse_ids,
copies=copies,
blacklist=[])
]
ignore_availability = True
except (InsufficientTargetRSEs,
InsufficientAccountLimit,
InvalidRuleWeight,
RSEOverQuota) as error:
logging.error(
prepend_str +
'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.'
%
(subscription['name'],
account, str(error)))
monitor.record_counter(
counters=
'transmogrifier.addnewrule.errortype.%s'
% (str(error.__class__.
__name__)),
delta=1)
# The DID won't be reevaluated at the next cycle
did_success = did_success and True
continue
for attempt in range(0, nattempt):
attemptnr = attempt
nb_rule = 0
# Try to create the rule
try:
if split_rule:
if not skip_rule_creation:
for rse in selected_rses:
if isinstance(
selected_rses,
dict):
source_replica_expression = selected_rses[
rse].get(
'source_replica_expression',
None)
weight = selected_rses[
rse].get(
'weight',
None)
logging.info(
prepend_str +
'Will insert one rule for %s:%s on %s'
%
(did['scope'],
did['name'], rse))
rule_ids = add_rule(
dids=[{
'scope':
did['scope'],
'name':
did['name']
}],
account=account,
copies=1,
rse_expression=rse,
grouping=grouping,
weight=weight,
lifetime=lifetime,
locked=locked,
subscription_id=
subscription_id,
source_replica_expression
=source_replica_expression,
activity=activity,
purge_replicas=
purge_replicas,
ignore_availability=
ignore_availability,
comment=comment)
created_rules[
cnt].append(
rule_ids[0])
nb_rule += 1
if nb_rule == copies:
success = True
break
else:
rule_ids = add_rule(
dids=[{
'scope': did['scope'],
'name': did['name']
}],
account=account,
copies=copies,
rse_expression=
rse_expression,
grouping=grouping,
weight=weight,
lifetime=lifetime,
locked=locked,
subscription_id=
subscription['id'],
source_replica_expression=
source_replica_expression,
activity=activity,
purge_replicas=
purge_replicas,
ignore_availability=
ignore_availability,
comment=comment)
created_rules[cnt].append(
rule_ids[0])
nb_rule += 1
monitor.record_counter(
counters=
'transmogrifier.addnewrule.done',
delta=nb_rule)
monitor.record_counter(
counters=
'transmogrifier.addnewrule.activity.%s'
% str_activity,
delta=nb_rule)
success = True
break
except (InvalidReplicationRule,
InvalidRuleWeight,
InvalidRSEExpression,
StagingAreaRuleRequiresLifetime,
DuplicateRule) as error:
# Errors that won't be retried
success = True
logging.error(prepend_str + '%s' %
(str(error)))
monitor.record_counter(
counters=
'transmogrifier.addnewrule.errortype.%s'
% (str(
error.__class__.__name__)),
delta=1)
break
except (ReplicationRuleCreationTemporaryFailed,
InsufficientTargetRSEs,
InsufficientAccountLimit,
DatabaseException,
RSEBlacklisted,
RSEWriteBlocked) as error:
# Errors to be retried
logging.error(
prepend_str +
'%s Will perform an other attempt %i/%i'
% (str(error), attempt + 1,
nattempt))
monitor.record_counter(
counters=
'transmogrifier.addnewrule.errortype.%s'
% (str(
error.__class__.__name__)),
delta=1)
except Exception:
# Unexpected errors
monitor.record_counter(
counters=
'transmogrifier.addnewrule.errortype.unknown',
delta=1)
exc_type, exc_value, exc_traceback = exc_info(
)
logging.critical(
prepend_str + ''.join(
format_exception(
exc_type, exc_value,
exc_traceback)).strip(
))
did_success = (did_success and success)
if (attemptnr +
1) == nattempt and not success:
logging.error(
prepend_str +
'Rule for %s:%s on %s cannot be inserted'
% (did['scope'], did['name'],
rse_expression))
else:
logging.info(
prepend_str +
'%s rule(s) inserted in %f seconds'
% (str(nb_rule),
time.time() - stime))
except DataIdentifierNotFound as error:
logging.warning(prepend_str + error)
if did_success:
if did['did_type'] == str(DIDType.FILE):
monitor.record_counter(
counters='transmogrifier.did.file.processed',
delta=1)
elif did['did_type'] == str(DIDType.DATASET):
monitor.record_counter(
counters='transmogrifier.did.dataset.processed',
delta=1)
elif did['did_type'] == str(DIDType.CONTAINER):
monitor.record_counter(
counters='transmogrifier.did.container.processed',
delta=1)
monitor.record_counter(
counters='transmogrifier.did.processed', delta=1)
identifiers.append({
'scope':
did['scope'],
'name':
did['name'],
'did_type':
DIDType.from_sym(did['did_type'])
})
time1 = time.time()
# Mark the DIDs as processed
for identifier in chunks(identifiers, 100):
_retrial(set_new_dids, identifier, None)
logging.info(prepend_str + 'Time to set the new flag : %f' %
(time.time() - time1))
tottime = time.time() - start_time
for sub in subscriptions:
update_subscription(
name=sub['name'],
account=sub['account'],
metadata={'last_processed': datetime.now()})
logging.info(prepend_str +
'It took %f seconds to process %i DIDs' %
(tottime, len(dids)))
logging.debug(prepend_str + 'DIDs processed : %s' % (str(dids)))
monitor.record_counter(counters='transmogrifier.job.done', delta=1)
monitor.record_timer(stat='transmogrifier.job.duration',
time=1000 * tottime)
except Exception:
exc_type, exc_value, exc_traceback = exc_info()
logging.critical(prepend_str + ''.join(
format_exception(exc_type, exc_value, exc_traceback)).strip())
monitor.record_counter(counters='transmogrifier.job.error',
delta=1)
monitor.record_counter(counters='transmogrifier.addnewrule.error',
delta=1)
if once is True:
break
if tottime < sleep_time:
logging.info(prepend_str + 'Will sleep for %s seconds' %
(sleep_time - tottime))
time.sleep(sleep_time - tottime)
heartbeat.die(executable, hostname, pid, hb_thread)
logging.info(prepend_str + 'Graceful stop requested')
logging.info(prepend_str + 'Graceful stop done')
def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int,
**_kwargs) -> bool:
worker_number, total_workers, logger = heartbeat_handler.live()
dids, subscriptions = [], []
tottime = 0
try:
# Get the new DIDs based on the is_new flag
logger(logging.DEBUG, "Listing new dids")
for did in list_new_dids(
thread=worker_number,
total_threads=total_workers,
chunk_size=bulk,
did_type=None,
):
dids.append({
"scope": did["scope"],
"did_type": str(did["did_type"]),
"name": did["name"],
})
logger(logging.INFO, "%i new dids to process", len(dids))
sub_dict = {3: []}
# Get the list of subscriptions. The default priority of the subscription is 3. 0 is the highest priority, 5 the lowest
# The priority is defined as 'policyid'
logger(logging.DEBUG, "Listing active subscriptions")
for sub in list_subscriptions(None, None):
if (sub["state"] != SubscriptionState.INACTIVE and sub["lifetime"]
and (datetime.now() > sub["lifetime"])):
update_subscription(
name=sub["name"],
account=sub["account"],
metadata={"state": SubscriptionState.INACTIVE},
)
elif sub["state"] in [
SubscriptionState.ACTIVE, SubscriptionState.UPDATED
]:
priority = 3
if "policyid" in sub:
if int(sub["policyid"]) not in sub_dict:
sub_dict[int(sub["policyid"])] = []
priority = int(sub["policyid"])
sub_dict[priority].append(sub)
priorities = list(sub_dict.keys())
priorities.sort()
# Order the subscriptions according to their priority
for priority in priorities:
subscriptions.extend(sub_dict[priority])
logger(logging.INFO, "%i active subscriptions", len(subscriptions))
except SubscriptionNotFound as error:
logger(logging.WARNING, "No subscriptions defined: %s" % (str(error)))
must_sleep = True
return must_sleep
except Exception as error:
logger(
logging.ERROR,
"Failed to get list of new DIDs or subscriptions: %s" %
(str(error)),
)
must_sleep = False
return must_sleep
results = {}
start_time = time.time()
blocklisted_rse_id = [
rse["id"] for rse in list_rses({"availability_write": False})
]
identifiers = []
# Loop over all the new dids
for did in dids:
_, _, logger = heartbeat_handler.live()
did_success = True
if did["did_type"] == str(DIDType.DATASET) or did["did_type"] == str(
DIDType.CONTAINER):
did_tag = "%s:%s" % (did["scope"].internal, did["name"])
results[did_tag] = []
try:
metadata = get_metadata(did["scope"], did["name"])
# Loop over all the subscriptions
for subscription in subscriptions:
# Check if the DID match the subscription
if is_matching_subscription(subscription, did,
metadata) is True:
filter_string = loads(subscription["filter"])
split_rule = filter_string.get("split_rule", False)
stime = time.time()
results[did_tag].append(subscription["id"])
logger(
logging.INFO,
"%s:%s matches subscription %s" %
(did["scope"], did["name"], subscription["name"]),
)
rules = loads(subscription["replication_rules"])
created_rules = {}
cnt = 0
for rule_dict in rules:
cnt += 1
created_rules[cnt] = []
# Get all the rule and subscription parameters
grouping = rule_dict.get("grouping", "DATASET")
lifetime = rule_dict.get("lifetime", None)
ignore_availability = rule_dict.get(
"ignore_availability", None)
weight = rule_dict.get("weight", None)
source_replica_expression = rule_dict.get(
"source_replica_expression", None)
locked = rule_dict.get("locked", None)
if locked == "True":
locked = True
else:
locked = False
purge_replicas = rule_dict.get(
"purge_replicas", False)
if purge_replicas == "True":
purge_replicas = True
else:
purge_replicas = False
rse_expression = str(rule_dict["rse_expression"])
comment = str(subscription["comments"]
)[:RULES_COMMENT_LENGTH]
if "comments" in rule_dict:
comment = str(rule_dict["comments"])
subscription_id = str(subscription["id"])
account = subscription["account"]
copies = int(rule_dict["copies"])
activity = rule_dict.get("activity",
"User Subscriptions")
try:
validate_schema(name="activity",
obj=activity,
vo=account.vo)
except InputValidationError as error:
logger(
logging.ERROR,
"Error validating the activity %s" %
(str(error)),
)
activity = "User Subscriptions"
if lifetime:
lifetime = int(lifetime)
str_activity = "".join(activity.split())
success = False
nattempt = 5
attemptnr = 0
skip_rule_creation = False
selected_rses = []
chained_idx = rule_dict.get("chained_idx", None)
if chained_idx:
params = {}
if rule_dict.get("associated_site_idx", None):
params[
"associated_site_idx"] = rule_dict.get(
"associated_site_idx", None)
logger(
logging.DEBUG,
"Chained subscription identified. Will use %s",
str(created_rules[chained_idx]),
)
algorithm = rule_dict.get("algorithm", None)
selected_rses = select_algorithm(
algorithm, created_rules[chained_idx],
params)
else:
# In the case of chained subscription, don't use rseselector but use the rses returned by the algorithm
if split_rule:
preferred_rses = set()
for rule in list_rules(
filters={
"subscription_id":
subscription_id,
"scope": did["scope"],
"name": did["name"],
}):
for rse_dict in parse_expression(
rule["rse_expression"],
filter_={"vo": account.vo},
):
preferred_rses.add(rse_dict["rse"])
preferred_rses = list(preferred_rses)
try:
(
selected_rses,
preferred_unmatched,
) = resolve_rse_expression(
rse_expression,
account,
weight=weight,
copies=copies,
size=0,
preferred_rses=preferred_rses,
blocklist=blocklisted_rse_id,
)
except (
InsufficientTargetRSEs,
InsufficientAccountLimit,
InvalidRuleWeight,
RSEOverQuota,
) as error:
logger(
logging.WARNING,
'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blocklisted sites'
% (
subscription["name"],
account,
str(error),
),
)
# Now including the blocklisted sites
try:
(
selected_rses,
preferred_unmatched,
) = resolve_rse_expression(
rse_expression,
account,
weight=weight,
copies=copies,
size=0,
preferred_rses=preferred_rses,
)
ignore_availability = True
except (
InsufficientTargetRSEs,
InsufficientAccountLimit,
InvalidRuleWeight,
RSEOverQuota,
) as error:
logger(
logging.ERROR,
'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.'
% (
subscription["name"],
account,
str(error),
),
)
monitor.record_counter(
name=
"transmogrifier.addnewrule.errortype.{exception}",
labels={
"exception":
str(error.__class__.
__name__)
},
)
# The DID won't be reevaluated at the next cycle
did_success = did_success and True
continue
if (len(preferred_rses) -
len(preferred_unmatched) >=
copies):
skip_rule_creation = True
for attempt in range(0, nattempt):
attemptnr = attempt
nb_rule = 0
# Try to create the rule
try:
if split_rule:
if not skip_rule_creation:
for rse in selected_rses:
if isinstance(
selected_rses, dict):
source_replica_expression = (
selected_rses[rse].get(
"source_replica_expression",
None,
))
weight = selected_rses[
rse].get(
"weight", None)
logger(
logging.INFO,
"Will insert one rule for %s:%s on %s"
% (did["scope"],
did["name"], rse),
)
rule_ids = add_rule(
dids=[{
"scope": did["scope"],
"name": did["name"],
}],
account=account,
copies=1,
rse_expression=rse,
grouping=grouping,
weight=weight,
lifetime=lifetime,
locked=locked,
subscription_id=
subscription_id,
source_replica_expression=
source_replica_expression,
activity=activity,
purge_replicas=
purge_replicas,
ignore_availability=
ignore_availability,
comment=comment,
)
created_rules[cnt].append(
rule_ids[0])
nb_rule += 1
if nb_rule == copies:
success = True
break
else:
rule_ids = add_rule(
dids=[{
"scope": did["scope"],
"name": did["name"],
}],
account=account,
copies=copies,
rse_expression=rse_expression,
grouping=grouping,
weight=weight,
lifetime=lifetime,
locked=locked,
subscription_id=subscription["id"],
source_replica_expression=
source_replica_expression,
activity=activity,
purge_replicas=purge_replicas,
ignore_availability=
ignore_availability,
comment=comment,
)
created_rules[cnt].append(rule_ids[0])
nb_rule += 1
monitor.record_counter(
name="transmogrifier.addnewrule.done",
delta=nb_rule,
)
monitor.record_counter(
name=
"transmogrifier.addnewrule.activity.{activity}",
delta=nb_rule,
labels={"activity": str_activity},
)
success = True
break
except (
InvalidReplicationRule,
InvalidRuleWeight,
InvalidRSEExpression,
StagingAreaRuleRequiresLifetime,
DuplicateRule,
) as error:
# Errors that won't be retried
success = True
logger(logging.ERROR, str(error))
monitor.record_counter(
name=
"transmogrifier.addnewrule.errortype.{exception}",
labels={
"exception":
str(error.__class__.__name__)
},
)
break
except (
ReplicationRuleCreationTemporaryFailed,
InsufficientTargetRSEs,
InsufficientAccountLimit,
DatabaseException,
RSEWriteBlocked,
) as error:
# Errors to be retried
logger(
logging.ERROR,
"%s Will perform an other attempt %i/%i"
% (str(error), attempt + 1, nattempt),
)
monitor.record_counter(
name=
"transmogrifier.addnewrule.errortype.{exception}",
labels={
"exception":
str(error.__class__.__name__)
},
)
except Exception:
# Unexpected errors
monitor.record_counter(
name=
"transmogrifier.addnewrule.errortype.{exception}",
labels={"exception": "unknown"},
)
logger(logging.ERROR,
"Unexpected error",
exc_info=True)
did_success = did_success and success
if (attemptnr + 1) == nattempt and not success:
logger(
logging.ERROR,
"Rule for %s:%s on %s cannot be inserted" %
(did["scope"], did["name"],
rse_expression),
)
else:
logger(
logging.INFO,
"%s rule(s) inserted in %f seconds" %
(str(nb_rule), time.time() - stime),
)
except DataIdentifierNotFound as error:
logger(logging.WARNING, str(error))
if did_success:
if did["did_type"] == str(DIDType.FILE):
monitor.record_counter(
name="transmogrifier.did.file.processed")
elif did["did_type"] == str(DIDType.DATASET):
monitor.record_counter(
name="transmogrifier.did.dataset.processed")
elif did["did_type"] == str(DIDType.CONTAINER):
monitor.record_counter(
name="transmogrifier.did.container.processed", delta=1)
monitor.record_counter(name="transmogrifier.did.processed",
delta=1)
identifiers.append({
"scope": did["scope"],
"name": did["name"],
"did_type": did["did_type"],
})
time1 = time.time()
# Mark the DIDs as processed
for identifier in chunks(identifiers, 100):
_retrial(set_new_dids, identifier, None)
logger(logging.DEBUG,
"Time to set the new flag : %f" % (time.time() - time1))
tottime = time.time() - start_time
for sub in subscriptions:
update_subscription(
name=sub["name"],
account=sub["account"],
metadata={"last_processed": datetime.now()},
)
logger(logging.INFO,
"It took %f seconds to process %i DIDs" % (tottime, len(dids)))
logger(logging.DEBUG, "DIDs processed : %s" % (str(dids)))
monitor.record_counter(name="transmogrifier.job.done", delta=1)
monitor.record_timer(name="transmogrifier.job.duration",
time=1000 * tottime)
must_sleep = True
return must_sleep
