def delete_rule(self, did, rse):
hash = did.split('-')[-1]
dtype = did.split('-')[0].split(':')[-1]
number = int(did.split(':')[0].split('_')[-1])
print("Deleting the rule {0} from {1}".format(did, rse))
print("Run number: {0}".format(number))
print("Data type: {0}".format(dtype))
print("Hash: {0}".format(hash))
run = self.db.db.find_one({'number': number})
#Checks if the datum exists in the DB
datum = None
for d in run['data']:
if d['type'] == dtype and d['host'] == 'rucio-catalogue' and d[
'location'] == rse:
datum = d
break
#Delete the datum
if datum is not None:
self.db.RemoveDatafield(run['_id'], datum)
print("Datum deleted in DB.")
else:
print('There is no datum to delete')
#Get the rule of a given DID
rucio_rule = self.rc.GetRule(upload_structure=did, rse=rse)
#Delete the rule
if rucio_rule['exists']:
self.rc.DeleteRule(rucio_rule['id'])
print("Rucio rule deleted.")
else:
print('There is no Rucio rule to delete')
#In case it is datamanager, directly delete files
if rse == self.UPLOAD_TO:
files = list_file_replicas(number, dtype, hash, self.UPLOAD_TO)
print("Deleting rucio data in datamanager disk. Deleting",
len(files), "files")
for file in files:
try:
os.remove(file)
except:
print("File: {0} not found".format(file))
print("Done.")
def remove_datatype_from_db_and_datamanager(did):
config = Config()
helper.make_global("admix_config", os.path.abspath(config.get('Admix','config_file')))
RSES = helper.get_hostconfig()['rses']
DB = ConnectMongoDB()
rc = RucioSummoner()
hash = did.split('-')[-1]
dtype = did.split('-')[0].split(':')[-1]
number = int(did.split(':')[0].split('_')[-1])
print("Removing",number,dtype,hash)
rundoc = DB.db.find_one({'number' : number})
print("status = ",rundoc['status'])
run_id = "%06d" % number
# make it uploadable
DB.db.find_one_and_update({'number':number},{'$set':{"status": "eb_ready_to_upload"}})
# Remove DB entries for all RSEs
for d in rundoc['data']:
if d['type'] == dtype and d['host'] == 'rucio-catalogue':
# Remove the data entry in DB
print("Deleting data = ",d)
DB.db.update_one({"_id" : rundoc['_id']},
{"$pull" : {"data" : d} })
# Remove Rucio rules for all RSEs
for rse in RSES:
rucio_rule = rc.GetRule(upload_structure=did, rse=rse)
if rucio_rule['exists']:
print("Deleting rucio rule = ", rucio_rule['id'])
rc.DeleteRule(rucio_rule['id'])
files = list_file_replicas(number, dtype, hash, "LNGS_USERDISK")
print("Deleting rucio data in datamanager disk. Deleting",len(files),"files")
for file in files:
os.remove(file)
def showrun(self, arg_number, arg_to, arg_dtypes, arg_compact,
arg_dumpjson, arg_status, arg_latest, arg_pending):
#Define data types
RAW_RECORDS_TPC_TYPES = helper.get_hostconfig(
)['raw_records_tpc_types']
RAW_RECORDS_MV_TYPES = helper.get_hostconfig()['raw_records_mv_types']
RAW_RECORDS_NV_TYPES = helper.get_hostconfig()['raw_records_nv_types']
LIGHT_RAW_RECORDS_TPC_TYPES = helper.get_hostconfig(
)['light_raw_records_tpc_types']
LIGHT_RAW_RECORDS_MV_TYPES = helper.get_hostconfig(
)['light_raw_records_mv_types']
LIGHT_RAW_RECORDS_NV_TYPES = helper.get_hostconfig(
)['light_raw_records_nv_types']
HIGH_LEVEL_TYPES = helper.get_hostconfig()['high_level_types']
RECORDS_TYPES = helper.get_hostconfig()['records_types']
#Get other parameters
DATADIR = helper.get_hostconfig()['path_data_to_upload']
RSES = helper.get_hostconfig()['rses']
minimum_number_acceptable_rses = 2
minimum_deltadays_allowed = 3
# Storing some backup hashes in case DID information is not available
bkp_hashes = {
'raw_records': 'rfzvpzj4mf',
'raw_records_he': 'rfzvpzj4mf',
'raw_records_mv': 'rfzvpzj4mf',
'raw_records_aqmon': 'rfzvpzj4mf',
'records': '56ausr64s7',
'lone_hits': 'b7dgmtzaef'
}
context = 'xenonnt_online'
#Init the runDB
db = ConnectMongoDB()
#Init Rucio for later uploads and handling:
rc = RucioSummoner(helper.get_hostconfig("rucio_backend"))
rc.SetRucioAccount(helper.get_hostconfig('rucio_account'))
rc.SetConfigPath(helper.get_hostconfig("rucio_cli"))
rc.SetProxyTicket(helper.get_hostconfig('rucio_x509'))
rc.SetHost(helper.get_hostconfig('host'))
rc.ConfigHost()
rc.SetProxyTicket("rucio_x509")
data_types = RAW_RECORDS_TPC_TYPES + RAW_RECORDS_MV_TYPES + RAW_RECORDS_NV_TYPES + LIGHT_RAW_RECORDS_TPC_TYPES + LIGHT_RAW_RECORDS_MV_TYPES + LIGHT_RAW_RECORDS_NV_TYPES + HIGH_LEVEL_TYPES + RECORDS_TYPES
# if arg_number has been given
if arg_number != "":
# if the "number" argument is a number, it is converted as integer
if arg_number.isdigit():
arg_number = int(arg_number)
# otherwise it is assumed that a DID has been given and run number and other parameters are extracted from the DID
else:
arg_number, dtype, hash = get_did(arg_number)
arg_dtypes = [dtype]
# if no arg_number has been given, then the "latest" option is activated (with 5 run numbers by default) in compact modality
else:
if arg_latest == 0:
arg_latest = 5
arg_compact = True
if arg_latest > 0:
cursor = db.db.find({}).sort('number', pymongo.DESCENDING).limit(1)
cursor = list(cursor)
arg_to = cursor[0]['number']
arg_number = arg_to - arg_latest + 1
print('Processing latest {0} runs'.format(arg_latest))
if arg_to > arg_number:
cursor = db.db.find({
'number': {
'$gte': arg_number,
'$lte': arg_to
}
}).sort('number', pymongo.ASCENDING)
print('Runs that will be processed are from {0} to {1}'.format(
arg_number, arg_to))
else:
cursor = db.db.find({'number': arg_number})
print('Run that will be processed is {0}'.format(arg_number))
cursor = list(cursor)
# Runs over all listed runs
for run in cursor:
print("")
# Gets run number
number = run['number']
print('Run: {0}'.format(number))
# Gets the status
if 'status' in run:
print('Status: {0}'.format(run['status']))
else:
print('Status: {0}'.format('Not available'))
if arg_status:
continue
# Extracts the correct Event Builder machine who processed this run
# Then also the bootstrax state and, in case it was abandoned, the reason
if 'bootstrax' in run:
bootstrax = run['bootstrax']
eb = bootstrax['host'].split('.')[0]
print('Processed by: {0}'.format(eb))
if 'state' in bootstrax:
print('Bootstrax state: {0}'.format(bootstrax['state']))
if bootstrax['state'] == 'abandoned':
if 'reason' in bootstrax:
print('Reason: {0}'.format(bootstrax['reason']))
else:
print('Not processed')
# Gets the date
if 'start' in run:
start_time = run['start'].replace(tzinfo=timezone.utc)
print("Date: ", start_time.astimezone(tz=None))
# Calculates the duration
if 'end' in run:
if run['end'] is not None:
end_time = run['end'].replace(tzinfo=timezone.utc)
duration = end_time - start_time
print("Duration: ", duration)
else:
print("Duration: ", "unknown")
# Prints if run is still enough recent (three days from now)
now_time = datetime.now().replace(tzinfo=timezone.utc)
delta_time = now_time - start_time
if delta_time < timedelta(days=minimum_deltadays_allowed):
print("Less than {0} days old".format(
minimum_deltadays_allowed))
else:
print("Warning : no time info available")
# Gets the comments
if 'comments' in run:
if len(run['comments']) > 0:
last_comment = run['comments'][-1]
print("Latest comment ({0}): {1}".format(
last_comment['user'], last_comment['comment']))
# Dumps the entire rundoc under json format
if arg_dumpjson:
print(dumps(run, indent=4))
if arg_compact:
continue
# Merges data and deleted_data
# if 'deleted_data' in run:
# data = run['data'] + run['deleted_data']
# else:
data = run['data']
# Check is there are more instances in more EventBuilders
extra_ebs = set()
for d in data:
if 'eb' in d['host'] and eb not in d['host']:
extra_ebs.add(d['host'].split('.')[0])
if len(extra_ebs) > 0:
print(
'\t\t Warning : The run has been processed by more than one EventBuilder: {0}'
.format(extra_ebs))
# Runs over all data types to be monitored
for dtype in data_types:
if len(arg_dtypes) > 0:
if dtype not in arg_dtypes:
continue
# Take the official number of files accordingto run DB
# and the eb status
Nfiles = -1
ebstatus = ""
for d in data:
if d['type'] == dtype and eb in d['host']:
if 'file_count' in d:
Nfiles = d['file_count']
if 'status' in d:
ebstatus = d['status']
if arg_pending:
if ebstatus in ["", "transferred"]:
continue
# Data type name
print('{0}'.format(dtype))
if Nfiles == -1:
print('\t Number of files: missing in DB')
else:
print('\t Number of files: {0}'.format(Nfiles))
if ebstatus != "":
print('\t EB status: {0}'.format(ebstatus))
else:
print('\t EB status: not available')
# Check if data are still in the data list and not in deleted_data
DB_InEB = False
for d in run['data']:
if d['type'] == dtype and eb in d['host']:
DB_InEB = True
DB_NotInEB = False
if 'deleted_data' in run:
for d in run['deleted_data']:
if d['type'] == dtype and eb in d['host']:
DB_NotInEB = True
if DB_InEB and not DB_NotInEB:
print('\t DB : still in EB')
if not DB_InEB and DB_NotInEB:
print('\t DB : deleted from EB')
if DB_InEB and DB_NotInEB:
print(
'\t\t Incoherency in DB: it is both in data list and in deleted_data list'
)
#if (DB_InEB and DB_NotInEB) or (not DB_InEB and not DB_NotInEB):
# print('\t\t incoherency in DB: it is neither in data list nor in deleted_data list')
# Check if data are still in the EB disks without using the DB
upload_path = ""
for d in run['data']:
if d['type'] == dtype and eb in d['host']:
file = d['location'].split('/')[-1]
upload_path = os.path.join(DATADIR, eb, file)
path_exists = os.path.exists(upload_path)
if upload_path != "" and path_exists:
path, dirs, files = next(os.walk(upload_path))
print('\t Disk: still in EB disk and with', len(files),
'files')
else:
print('\t Disk: not in EB disk')
if DB_InEB and not path_exists:
print(
'\t\t Incoherency in DB and disk: it is in DB data list but it is not in the disk'
)
if DB_NotInEB and path_exists:
print(
'\t\t Incoherency in DB and disk: it is in DB deleted_data list but it is still in the disk'
)
# The list of DIDs (usually just one)
dids = set()
for d in data:
if d['type'] == dtype and d['host'] == 'rucio-catalogue':
if 'did' in d:
dids.add(d['did'])
print('\t DID:', dids)
# Check the presence in each available RSE
Nrses = 0
for rse in RSES:
is_in_rse = False
for d in run['data']:
if d['type'] == dtype and rse in d['location']:
if 'status' in d:
status = d['status']
else:
status = 'Not available'
if 'did' in d:
hash = d['did'].split('-')[-1]
did = d['did']
else:
print(
'\t\t Warning : DID information is absent in DB data list (old admix version). Using standard hashes for RSEs'
)
#hash = bkp_hashes.get(dtype)
#hash = utilix.db.get_hash(context, dtype)
hash = db.GetHashByContext(context, dtype)
did = make_did(number, dtype, hash)
rucio_rule = rc.GetRule(upload_structure=did,
rse=rse)
files = list_file_replicas(number, dtype, hash,
rse)
if rucio_rule['exists']:
print('\t', rse + ': DB Yes, Status', status,
', Rucio Yes, State',
rucio_rule['state'], ",", len(files),
'files')
if len(files) < Nfiles and rucio_rule[
'state'] != "REPLICATING":
print(
'\t\t Warning : Wrong number of files in Rucio!!!'
)
else:
print('\t', rse + ': DB Yes, Status', status,
', Rucio No')
# print(files)
is_in_rse = True
Nrses += 1
if not is_in_rse:
# print('\t\t Warning : data information is absent in DB data list. Trying using standard hashes to query Rucio')
# hash = bkp_hashes.get(dtype)
#hash = utilix.db.get_hash(context, dtype)
hash = db.GetHashByContext(context, dtype)
did = make_did(number, dtype, hash)
print('\t Guessed DID:', did)
rucio_rule = rc.GetRule(upload_structure=did, rse=rse)
files = list_file_replicas(number, dtype, hash, rse)
if rucio_rule['exists']:
print('\t', rse + ': DB No, Rucio Yes, State',
rucio_rule['state'], ",", len(files),
'files')
if len(files) < Nfiles and rucio_rule[
'state'] != "REPLICATING":
print(
'\t\t Warning : Wrong number of files in Rucio!!!'
)
else:
print('\t', rse + ': DB No, Rucio No')
print('\t Number of sites: ', Nrses)
def reset_upload(self, did):
hash = did.split('-')[-1]
dtype = did.split('-')[0].split(':')[-1]
number = int(did.split(':')[0].split('_')[-1])
print("Resetting the upload associated to the DID: {0}".format(did))
print("Run number: {0}".format(number))
print("Data type: {0}".format(dtype))
print("Hash: {0}".format(hash))
run = self.db.db.find_one({'number': number})
# Gets the status
if 'status' in run:
print('Run status: {0}'.format(run['status']))
else:
print('Run status: {0}'.format('Not available'))
# Extracts the correct Event Builder machine who processed this run
# Then also the bootstrax state and, in case it was abandoned, the reason
if 'bootstrax' in run:
bootstrax = run['bootstrax']
eb = bootstrax['host'].split('.')[0]
else:
print('Not processed')
return (0)
# Get the EB datum and its status
ebstatus = ""
datum = None
for d in run['data']:
if d['type'] == dtype and eb in d['host']:
datum = d
if 'status' in d:
ebstatus = d['status']
if datum is None:
print('There is no EB datum. No reset is possible')
return (0)
if ebstatus != "":
print('EB status: {0}'.format(ebstatus))
else:
print('EB status: not available')
# Step zero (normally not needed): change the run status to "transferring"
# self.db.db.find_one_and_update({'number':number},{'$set':{"status": "transferring"}})
# First action: remove the files stored in datamanager
files = list_file_replicas(number, dtype, hash, self.UPLOAD_TO)
print("Deleting rucio data in datamanager disk. Deleting", len(files),
"files")
for file in files:
try:
os.remove(file)
except:
print("File: {0} not found".format(file))
# Second action: remove the LNGS Rucio rule
deleted_any_rule = False
for rse in self.RSES:
rucio_rule = self.rc.GetRule(upload_structure=did, rse=rse)
if rucio_rule['exists']:
print("Deleting rucio rule = ", rucio_rule['id'],
"from RSE = ", rse)
self.rc.DeleteRule(rucio_rule['id'])
deleted_any_rule = True
# Third action: remove possible files in datamanager in case the Rucio rule does not exists
datamanager_rucio_rule = self.rc.GetRule(upload_structure=did,
rse=self.UPLOAD_TO)
if not datamanager_rucio_rule['exists']:
print(
"Rucio rule not existing. Deleting data in datamanager without Rucio"
)
filelistname = os.path.join(
"/archive/data/rucio/xnt_%06d/*/*/" % number,
dtype + "-" + hash + "*")
filelist = glob.glob(filelistname)
for filePath in filelist:
try:
os.remove(filePath)
except:
print("Error while deleting file : ", filePath)
# If some rule has been deleted, wait for 1 hour (plus 5 minutes of margin)
if deleted_any_rule:
print(
"We have to wait until the rule is fully deleted before changing the status of the datum. It could take at least an hour"
)
while True:
datamanager_rucio_rule = self.rc.GetRule(upload_structure=did,
rse=self.UPLOAD_TO)
if not datamanager_rucio_rule['exists']:
print("Rule for did {0} finally deleted".format(did))
break
delay = 60 * 10
time.sleep(delay)
else:
print("There is no rule to delete")
# Fourth action: set the EB status as 'eb_ready_to_upload'
self.db.db.find_one_and_update(
{
'_id': run['_id'],
'data': {
'$elemMatch': {
'type': datum['type'],
'location': datum['location'],
'host': datum['host']
}
}
}, {'$set': {
"data.$.status": 'eb_ready_to_upload'
}})
print("EB status changed to eb_ready_to_upload")
# Reload the run
run = self.db.db.find_one({'number': number})
# Gets the status
if 'status' in run:
print('New run status: {0}'.format(run['status']))
else:
print('Ru status: {0}'.format('Not available'))
# Get the EB datum and its status
ebstatus = ""
datum = None
for d in run['data']:
if d['type'] == dtype and eb in d['host']:
datum = d
if 'status' in d:
ebstatus = d['status']
# Prints the eb status as a confirmation of the performed change
if ebstatus != "":
print('New EB status: {0}'.format(ebstatus))
else:
print('New EB status: not available')
def run(self, *args, **kwargs):
helper.global_dictionary['logger'].Info(
f'Run task {self.__class__.__name__}')
data_types = self.RAW_RECORDS_TPC_TYPES + self.RAW_RECORDS_MV_TYPES + self.RAW_RECORDS_NV_TYPES + self.LIGHT_RAW_RECORDS_TPC_TYPES + self.LIGHT_RAW_RECORDS_MV_TYPES + self.LIGHT_RAW_RECORDS_NV_TYPES + self.HIGH_LEVEL_TYPES + self.RECORDS_TYPES
# Get all runs that are already transferred and that still have some data_types in eb
cursor = self.db.db.find(
{
# 'number': {"$lt": 7600, "$gte": 7500},
# 'number': {"$lt": 7600, "$gte": 7200},
# 'number': {"$lt": 8570, "$gte": 8550},
# 'number': {"$gte": 7330},
# 'number': {"$gte": 8500},
'number': {
"$gte": 10800
},
# 'number': {"$gte": 8013},
# 'number': 8075,
# 'data' : { "$elemMatch": { "host" : {"$regex" : ".*eb.*"} , "type" : {"$in" : data_types}} },
# 'status': 'transferred'
'status': {
'$in': ['transferred', 'transferring']
}
},
{
'_id': 1,
'number': 1,
'data': 1,
'bootstrax': 1
})
cursor = list(cursor)
# helper.global_dictionary['logger'].Info('Runs that will be processed are {0}'.format([c["number"] for c in cursor]))
helper.global_dictionary['logger'].Info(
'Runs that will be processed are {0}'.format(len(cursor)))
# Runs over all listed runs
for run in cursor:
#Gets the run number
number = run['number']
# Extracts the correct Event Builder machine who processed this run
bootstrax = run['bootstrax']
eb = bootstrax['host'].split('.')[0]
# helper.global_dictionary['logger'].Info('Treating run {0}'.format(number))
# helper.global_dictionary['logger'].Info('Run {0} has been processed by {1}'.format(number,eb))
# Checks how much date are old
if 'time' in run['bootstrax']:
run_time = run['bootstrax']['time'].replace(
tzinfo=timezone.utc)
now_time = datetime.now().replace(tzinfo=timezone.utc)
delta_time = now_time - run_time
else:
delta_time = timedelta(days=self.minimum_deltadays_allowed)
# Loops on all datatypes that have to be cleaned
for dtype in data_types:
# helper.global_dictionary['logger'].Info('\t==> Looking for data type {0}'.format(dtype))
# checks the age of the data type
is_enough_old = True
# for some data types, it they are not yet older than three days, it skips deleting them
if dtype in self.dtype_delayed_delete:
if delta_time < timedelta(
days=self.minimum_deltadays_allowed):
helper.global_dictionary['logger'].Info(
'Run {0}, data type {1} is not yet older than {2} days. Skip it'
.format(number, dtype,
self.minimum_deltadays_allowed))
is_enough_old = False
# for some heavy data types (records and raw_records), if they are not yet older than one day, it skips deleting them
if dtype in self.dtype_delayed_delete_heavy:
if delta_time < timedelta(
days=self.minimum_deltadays_allowed_heavy):
helper.global_dictionary['logger'].Info(
'Run {0}, data type {1} is not yet older than {2} days. Skip it'
.format(number, dtype,
self.minimum_deltadays_allowed_heavy))
is_enough_old = False
# check first with runDB if the data type already exists in external RSEs
rses_in_db = []
for d in run['data']:
if d['type'] == dtype and d[
'host'] == 'rucio-catalogue' and d[
'location'] != self.UPLOAD_TO and d[
'status'] == 'transferred':
rses_in_db.append(d['location'])
# helper.global_dictionary['logger'].Info('\t==> According to DB, found in following external RSEs : {0}'.format(rses_in_db))
# if this is not the case, just skip any attempt of deleting anything
if len(rses_in_db) < self.minimum_number_acceptable_rses:
# helper.global_dictionary['logger'].Info('\t==> Nothing will be deleted : not enough external RSEs')
continue
# check first if data are, according to the DB, still in EB
datum = None
for d in run['data']:
if d['type'] == dtype and eb in d['host']:
datum = d
# skip this data type in case the eb status field is missing or is different from "transferred"
if datum is not None and ('status' not in datum
or datum['status'] != "transferred"):
continue
#
# Phase 1 : Deleting data in EB
#
# if datum is None:
# helper.global_dictionary['logger'].Info('Data type not in eb')
# start deleting data in EB
if datum is not None and dtype not in self.dtype_never_delete and is_enough_old:
file = datum['location'].split('/')[-1]
hash = file.split('-')[-1]
# create the DID from DB
did = make_did(number, dtype, hash)
# check if a rule already exists with this exact DID in external RSEs
# and take also the number of files in each RSE
rses_with_rule = []
rses_with_correct_nfiles = []
for rse in self.RSES:
rucio_rule = self.rc.GetRule(upload_structure=did,
rse=rse)
if rucio_rule['exists'] and rucio_rule['state'] == 'OK':
if self.UPLOAD_TO == rucio_rule['rse']:
continue
rses_with_rule.append(rucio_rule['rse'])
nfiles = len(
list_file_replicas(number, dtype, hash,
rucio_rule['rse']))
if 'file_count' in datum:
if nfiles == datum['file_count']:
rses_with_correct_nfiles.append(
rucio_rule['rse'])
# helper.global_dictionary['logger'].Info('\t==> According to Rucio, found in following external RSEs : {0}'.format(rses_with_rule))
# if len(rses_with_correct_nfiles) == len(rses_with_rule):
# helper.global_dictionary['logger'].Info('\t==> All of them with the expected number of files')
# else:
# helper.global_dictionary['logger'].Info('\t==> Error, these RSEs have wrong number of files : {0}'.format(rses_with_correct_nfiles))
# if so, start deleting
# if len(rses_with_rule)>=self.minimum_number_acceptable_rses and len(rses_with_correct_nfiles) == len(rses_with_rule):
if len(rses_with_rule
) >= self.minimum_number_acceptable_rses and len(
rses_with_correct_nfiles
) >= self.minimum_number_acceptable_rses:
# if len(rses_with_rule)>=self.minimum_number_acceptable_rses:
# delete from DB
# print(run['_id'],datum['type'],datum['host'])
self.db.RemoveDatafield(run['_id'], datum)
full_path = os.path.join(self.DATADIR, eb, file)
# print(full_path)
helper.global_dictionary['logger'].Info(
'\t==> Run {0}, data type {1}. Deleted EB info from DB'
.format(number, dtype))
# delete from disk
try:
shutil.rmtree(full_path)
except OSError as e:
helper.global_dictionary['logger'].Info(
'\t==> Error, cannot delete directory : {0}'.
format(e))
else:
helper.global_dictionary['logger'].Info(
'\t==> Run {0}, data type {1}. Deleted data from EB disk'
.format(number, dtype))
#
# Phase 2 : Deleting data in LNGS_USERDISK
#
# check if data are, according to the DB, still in datamanager (LNGS_USERDISK)
datum = None
for d in run['data']:
if d['type'] == dtype and d[
'host'] == 'rucio-catalogue' and self.UPLOAD_TO in d[
'location']:
datum = d
# if so, start deleting data in datamanager (LNGS_USERDISK)
# if datum is None:
# helper.global_dictionary['logger'].Info('Data type not in LNGS_USERDISK')
if datum is not None:
# create the DID from DB
did = datum['did']
hash = did.split('-')[-1]
nfiles_upload_to = len(
list_file_replicas(number, dtype, hash,
self.UPLOAD_TO))
# check if a rule already exists with this exact DID in external RSEs
rses_with_rule = []
rses_with_correct_nfiles = []
for rse in self.RSES:
rucio_rule = self.rc.GetRule(upload_structure=did,
rse=rse)
if rucio_rule['exists'] and rucio_rule['state'] == 'OK':
if self.UPLOAD_TO == rucio_rule['rse']:
continue
rses_with_rule.append(rucio_rule['rse'])
nfiles = len(
list_file_replicas(number, dtype, hash,
rucio_rule['rse']))
if nfiles == nfiles_upload_to:
rses_with_correct_nfiles.append(
rucio_rule['rse'])
# helper.global_dictionary['logger'].Info('\t==> According to Rucio, found in following external RSEs : {0}'.format(rses_with_rule))
# if len(rses_with_correct_nfiles) == len(rses_with_rule):
# helper.global_dictionary['logger'].Info('\t==> All of them with the expected number of files')
# else:
# helper.global_dictionary['logger'].Info('\t==> Error, these RSEs have wrong number of files : {0}'.format(rses_with_correct_nfiles))
# if so, start deleting
# if len(rses_with_rule)>=self.minimum_number_acceptable_rses and len(rses_with_correct_nfiles) == len(rses_with_rule):
if len(rses_with_rule
) >= self.minimum_number_acceptable_rses and len(
rses_with_correct_nfiles
) >= self.minimum_number_acceptable_rses:
rucio_rule = self.rc.GetRule(upload_structure=did,
rse=self.UPLOAD_TO)
if rucio_rule['exists'] and rucio_rule[
'state'] == 'OK' and rucio_rule[
'rse'] == self.UPLOAD_TO:
self.rc.DeleteRule(rucio_rule['id'])
helper.global_dictionary['logger'].Info(
'\t==> Run {0}, data type {1}. Deleted LNGS_USERDISK Rucio rule'
.format(number, dtype))
hash = did.split('-')[-1]
files = list_file_replicas(number, dtype, hash,
"LNGS_USERDISK")
for file in files:
os.remove(file)
helper.global_dictionary['logger'].Info(
'\t==> Run {0}, data type {1}. Deleted data from LNGS_USERDISK disk'
.format(number, dtype))
self.db.RemoveDatafield(run['_id'], datum)
helper.global_dictionary['logger'].Info(
'\t==> Run {0}, data type {1}. Deleted LNGS_USERDISK info from DB'
.format(number, dtype))
return 0
def reset_upload(did):
config = Config()
helper.make_global("admix_config", os.path.abspath(config.get('Admix','config_file')))
RSES = helper.get_hostconfig()['rses']
DB = ConnectMongoDB()
rc = RucioSummoner()
hash = did.split('-')[-1]
dtype = did.split('-')[0].split(':')[-1]
number = int(did.split(':')[0].split('_')[-1])
print("Resetting the upload associated to the DID: {0}".format(did))
print("Run number: {0}".format(number))
print("Data type: {0}".format(dtype))
print("Hash: {0}".format(hash))
run = DB.db.find_one({'number' : number})
# Gets the status
if 'status' in run:
print('Run status: {0}'.format(run['status']))
else:
print('Run status: {0}'.format('Not available'))
# Extracts the correct Event Builder machine who processed this run
# Then also the bootstrax state and, in case it was abandoned, the reason
if 'bootstrax' in run:
bootstrax = run['bootstrax']
eb = bootstrax['host'].split('.')[0]
else:
print('Not processed')
return(0)
# Get the EB datum and its status
ebstatus = ""
datum = None
for d in run['data']:
if d['type'] == dtype and eb in d['host']:
datum = d
if 'status' in d:
ebstatus = d['status']
if datum is None:
print('There is no EB datum. No reset is possible')
return(0)
if ebstatus != "":
print('EB status: {0}'.format(ebstatus))
else:
print('EB status: not available')
# Step zero (normally not needed): change the run status to "transferring"
# DB.db.find_one_and_update({'number':number},{'$set':{"status": "transferring"}})
# First action: remove the files stored in datamanager
files = list_file_replicas(number, dtype, hash, "LNGS_USERDISK")
print("Deleting rucio data in datamanager disk. Deleting",len(files),"files")
for file in files:
os.remove(file)
# Second action: remove the LNGS Rucio rule
rucio_rule = rc.GetRule(upload_structure=did, rse='LNGS_USERDISK')
if rucio_rule['exists']:
print("Deleting rucio rule = ", rucio_rule['id'])
rc.DeleteRule(rucio_rule['id'])
# Wait for 1 hour (plus 5 minutes of margin)
delay = 3600+60*5
print("We have to wait for {0} seconds before proceeding to the next step".format(delay))
time.sleep(delay)
else:
print("There is no rule to delete")
# Third action: set the EB status as 'eb_ready_to_upload'
DB.db.find_one_and_update({'_id': run['_id'],'data': {'$elemMatch': datum}},
{'$set': {'data.$.status': 'eb_ready_to_upload'}})
print("EB status changed to eb_ready_to_upload")
# Reload the run
run = DB.db.find_one({'number' : number})
# Gets the status
if 'status' in run:
print('New run status: {0}'.format(run['status']))
else:
print('Ru status: {0}'.format('Not available'))
# Get the EB datum and its status
ebstatus = ""
datum = None
for d in run['data']:
if d['type'] == dtype and eb in d['host']:
datum = d
if 'status' in d:
ebstatus = d['status']
# Prints the eb status as a confirmation of the performed change
if ebstatus != "":
print('New EB status: {0}'.format(ebstatus))
else:
print('New EB status: not available')