def remove_datatype_from_db(did,rse):
config = Config()
helper.make_global("admix_config", os.path.abspath(config.get('Admix','config_file')))
RSES = helper.get_hostconfig()['rses']
DB = ConnectMongoDB()
rc = RucioSummoner()
hash = did.split('-')[-1]
dtype = did.split('-')[0].split(':')[-1]
number = int(did.split(':')[0].split('_')[-1])
print("Removing",number,dtype,hash)
rundoc = DB.db.find_one({'number' : number})
print("status = ",rundoc['status'])
run_id = "%06d" % number
# make it uploadable
DB.db.find_one_and_update({'number':number},{'$set':{"status": "transferring"}})
# Remove DB entries for all RSEs
for d in rundoc['data']:
if d['type'] == dtype and d['host'] == 'rucio-catalogue' and d['did'] == did and d['location'] == rse:
# Remove the data entry in DB
print("Deleting data = ",d)
DB.db.update_one({"_id" : rundoc['_id']},
{"$pull" : {"data" : d} })
def main():
parser = ArgumentParser("admix-showrun")
config = Config()
parser.add_argument("number",
nargs='?',
help="Run number or DID to show",
default="")
parser.add_argument("--dtypes",
nargs="*",
help="Restricts infos on the given data types")
parser.add_argument("--to",
type=int,
help="Shows runs from the run number up to this value",
default=0)
parser.add_argument(
"--compact",
help="Just list few DB infos as run number, status, date, comments",
action='store_true')
parser.add_argument("--status",
help="Just list the run name and its global status",
action='store_true')
parser.add_argument("--pending",
help="Shows only pending data types",
action='store_true')
parser.add_argument("--json",
help="Dumps the whole DB rundoc in pretty style",
action='store_true')
parser.add_argument("--latest",
type=int,
help="Shows latest runs",
default=0)
parser.add_argument("--new",
help="New version under development",
action='store_true')
args = parser.parse_args()
if args.dtypes:
dtypes = args.dtypes
else:
dtypes = []
helper.make_global("admix_config",
os.path.abspath(config.get('Admix', 'config_file')))
showrun = ShowRun()
try:
if args.new:
showrun.showrun_new(args.number, args.to, dtypes, args.compact,
args.json, args.status, args.latest,
args.pending)
else:
showrun.showrun(args.number, args.to, dtypes, args.compact,
args.json, args.status, args.latest, args.pending)
except KeyboardInterrupt:
return 0
def showcontexts():
config = Config()
helper.make_global("admix_config", os.path.abspath(config.get('Admix','config_file')))
#Define data types
NORECORDS_DTYPES = helper.get_hostconfig()['norecords_types']
RAW_RECORDS_DTYPES = helper.get_hostconfig()['raw_records_types']
RECORDS_DTYPES = helper.get_hostconfig()['records_types']
#Get other parameters
DATADIR = helper.get_hostconfig()['path_data_to_upload']
periodic_check = helper.get_hostconfig()['upload_periodic_check']
RSES = helper.get_hostconfig()['rses']
#Init the runDB
db = ConnectMongoDB()
data_types = RAW_RECORDS_DTYPES + RECORDS_DTYPES + NORECORDS_DTYPES
context = 'xenonnt_online'
for dtype in data_types:
hash = utilix.db.get_hash(context, dtype)
print('Data type {0}, hash {1}'.format(dtype,hash))
def remove_datatype_from_db_and_datamanager(did):
config = Config()
helper.make_global("admix_config", os.path.abspath(config.get('Admix','config_file')))
RSES = helper.get_hostconfig()['rses']
DB = ConnectMongoDB()
rc = RucioSummoner()
hash = did.split('-')[-1]
dtype = did.split('-')[0].split(':')[-1]
number = int(did.split(':')[0].split('_')[-1])
print("Removing",number,dtype,hash)
rundoc = DB.db.find_one({'number' : number})
print("status = ",rundoc['status'])
run_id = "%06d" % number
# make it uploadable
DB.db.find_one_and_update({'number':number},{'$set':{"status": "eb_ready_to_upload"}})
# Remove DB entries for all RSEs
for d in rundoc['data']:
if d['type'] == dtype and d['host'] == 'rucio-catalogue':
# Remove the data entry in DB
print("Deleting data = ",d)
DB.db.update_one({"_id" : rundoc['_id']},
{"$pull" : {"data" : d} })
# Remove Rucio rules for all RSEs
for rse in RSES:
rucio_rule = rc.GetRule(upload_structure=did, rse=rse)
if rucio_rule['exists']:
print("Deleting rucio rule = ", rucio_rule['id'])
rc.DeleteRule(rucio_rule['id'])
files = list_file_replicas(number, dtype, hash, "LNGS_USERDISK")
print("Deleting rucio data in datamanager disk. Deleting",len(files),"files")
for file in files:
os.remove(file)
def main():
print("")
print("--------------------------")
print("-- aDMIX Upload Manager --")
print("--------------------------")
print("")
parser = argparse.ArgumentParser(description="aDMIX Upload Manager")
config = Config()
# From here the input depends on the usage of the command
# Add arguments for the process manager:
parser.add_argument('--admix-config', dest="admix_config", type=str, default=config.get('Admix','config_file'),
help="Load your host configuration")
parser.add_argument('--once', dest='once', action='store_true',
help="Run aDMIX Upload Manager only once")
parser.add_argument('--high', dest='high', action='store_true',
help="Treat only high level data types")
parser.add_argument('--low', dest='low', action='store_true',
help="Treat only low level data types")
args = parser.parse_args()
#We make the individual arguments global available right after aDMIX starts:
helper.make_global("admix_config", os.path.abspath(args.admix_config))
helper.make_global("high", args.high)
helper.make_global("low", args.low)
helper.make_global("once", args.once)
#Pre tests:
# admix host configuration must match the hostname:
if helper.get_hostconfig()['hostname'] != helper.get_hostname():
print("admix configuration file for %s" % helper.get_hostconfig()['hostname'])
print(helper.get_hostname())
print("You are at {0}".format( helper.get_hostname()))
exit()
#Setup the logger in a very basic modi
# lg = Logger(logpath=helper.get_hostconfig()['log_path'],
# loglevel=logging.DEBUG)
# lg.Info("-----------------------------------------")
# lg.Info("aDMIX - advanced Data Management in XENON")
# helper.make_global("logger", lg)
upload_manager = UploadManager()
upload_manager.loop()
def remove_datatype_from_db_and_rule(number,dtype,rse):
config = Config()
helper.make_global("admix_config", os.path.abspath(config.get('Admix','config_file')))
RSES = helper.get_hostconfig()['rses']
DB = ConnectMongoDB()
rc = RucioSummoner()
print("Removing",number,dtype)
rundoc = DB.db.find_one({'number' : number})
print("status = ",rundoc['status'])
run_id = "%06d" % number
# make it uploadable
# DB.db.find_one_and_update({'number':number},{'$set':{"status": "transferring"}})
for d in rundoc['data']:
if d['type'] == dtype and d['host'] == 'rucio-catalogue' and d['location'] == rse:
# Remove the data entry in DB
print("Deleting datum from DB = ",d)
DB.db.update_one({"_id" : rundoc['_id']},
{"$pull" : {"data" : d} })
# Remove Rucio rule
rucio_rule = rc.GetRule(upload_structure=d['did'], rse=rse)
if rucio_rule['exists']:
print("Deleting rucio rule = ", rucio_rule['id'])
rc.DeleteRule(rucio_rule['id'])
def your_admix():
print("advanced Data Management in XENON")
parser = argparse.ArgumentParser(description="Run your favourite aDMIX")
config = Config()
# From here the input depends on the usage of the command
parser.add_argument('task',
nargs="?",
default="default",
help="Select an aDMIX task")
parser.add_argument('--admix-config',
dest="admix_config",
type=str,
default=config.get('Admix', 'config_file'),
help="Load your host configuration")
parser.add_argument('--once',
dest='once',
action='store_true',
help="Run aDMIX only once")
parser.add_argument('--sleep-time',
dest='sleep_time',
type=int,
help="Time to wait before running again the task")
args = parser.parse_args()
helper.make_global("admix_config", os.path.abspath(args.admix_config))
if args.sleep_time != None:
helper.make_global("sleep_time", args.sleep_time)
else:
helper.make_global("sleep_time", helper.get_hostconfig()['sleep_time'])
#Pre tests:
# admix host configuration must match the hostname:
if helper.get_hostconfig()['hostname'] != helper.get_hostname():
print("admix configuration file for %s" %
helper.get_hostconfig()['hostname'])
print(helper.get_hostname())
print("You are at {0}".format(helper.get_hostname()))
exit()
#Setup the logger in a very basic modi
lg = Logger(logpath=helper.get_hostconfig()['log_path'],
loglevel=logging.DEBUG)
lg.Info("-----------------------------------------")
lg.Info("aDMIX - advanced Data Management in XENON")
helper.make_global("logger", lg)
#Determine which tasks are addressed:
# - if it comes from args.task use it, nevertheless what is defined in hostconfig("task")
# - if args.task == default use hostconfig("task") information
task_list = []
if args.task == "default":
task_list.extend(helper.get_hostconfig("task"))
else:
task_list = [args.task]
#test if the list of tasks is available from the decorator
task_test = [
True if i_task in NameCollector else False for i_task in task_list
]
task_list = np.array(task_list)[task_test]
if len(task_list) == 0:
print("Select a task from this list:")
for i_task in NameCollector:
print(" <> {0}".format(i_task))
print("or adjust the 'task' field in your configuration")
print("file: {0}".format(helper.global_dictionary["admix_config"]))
exit()
#Create a tmp file named as the screen session that contains this process
process = psutil.Process()
screen = process.parent().parent().parent().parent().cmdline()[-1]
open("/tmp/admix-" + screen, 'a').close()
#Loop over the inizialization of all classes
for i_task in task_list:
ClassCollector[i_task].init()
#Go for the loop
while True:
for i_task in task_list:
ClassCollector[i_task].run()
if args.once == True:
end_admix()
break
if os.path.exists("/tmp/admix-stop"):
print("Exiting because of the presence of /tmp/admix-stop file")
end_admix()
break
wait_time = helper.global_dictionary['sleep_time']
if "CheckTransfers" in task_list or "CleanEB" in task_list:
wait_time = 600
print('Waiting for {0} seconds'.format(wait_time))
print("You can safely CTRL-C now if you need to stop me")
try:
time.sleep(wait_time)
except KeyboardInterrupt:
end_admix()
break
def main():
parser = ArgumentParser("admix-fix")
config = Config()
parser.add_argument(
"--reset_upload",
nargs=1,
help=
"Deletes everything related a given DID, except data in EB. The deletion includes the entries in the Rucio catalogue and the related data in the DB rundoc. This is ideal if you want to retry an upload that failed",
metavar=('DID'))
parser.add_argument(
"--fix_upload",
nargs=1,
help=
"Deletes everything related a given DID, then it retries the upload",
metavar=('DID'))
parser.add_argument(
"--add_rule",
nargs=3,
help=
"Add a new replication rule of a given DID from one RSE to another one. The rundoc in DB is updated with a new datum as well",
metavar=('DID', 'FROM_RSE', 'TO_RSE'))
parser.add_argument(
"--delete_rule",
nargs=2,
help=
"Delete a replication rule of a given DID from one RSE. The rundoc in DB is deleted as well",
metavar=('DID', 'RSE'))
parser.add_argument(
"--delete_db_datum",
nargs=2,
help=
"Deletes the db datum corresponding to a given DID. The SITE can be either a specific EB machine (ex: eb1) or a specific RSE",
metavar=('DID', 'SITE'))
parser.add_argument(
"--set_run_status",
nargs=2,
help=
"Set the run status to a given NAME (typical case is to set it to eb_ready_to_upload)",
metavar=('RUN_NUMBER', 'STATUS_NAME'))
parser.add_argument(
"--set_eb_status",
nargs=2,
help="Set the EB status of a given DID to a given NAME",
metavar=('DID', 'STATUS_NAME'))
parser.add_argument(
"--priority",
type=int,
help="Priority to assign to Rucio rules (default: %(default)s)",
default=3)
parser.add_argument(
"--skip_rucio",
help=
"Add this flag in context of add_rule in case you just want to update DB since Rucio rule exists already",
action='store_true')
parser.add_argument(
"--list_non_transferred_runs",
help="Lists all runs whose status is still not transferred",
action='store_true')
parser.add_argument(
"--test_db_modification",
nargs=2,
help="Test how quickly a modification in DB is registered",
metavar=('DID', 'STATUS'))
parser.add_argument(
"--fix_upload_db",
nargs=1,
help=
"To be used when the upload done by Rucio has been completed but then admix crashed before updating the DB",
metavar=('DID'))
parser.add_argument(
"--create_upload_rules",
nargs=1,
help=
"To be used when the upload done by Rucio has been completed but then admix crashed before creating the rules abroad",
metavar=('DID'))
parser.add_argument(
"--postpone",
help=
"To be used when an upload failed (for any reason) in a screen session and you want to free the session. Metadata on the failed dataset are copied in a directory and will be fixed by an expert",
action='store_true')
parser.add_argument(
"--add_rules_from_file",
nargs=3,
help=
"To be used when you want to transfer data from one RSE to another RSE, using rucio and without updating the database. The option requires a FILE containing the list of DIDs to be transferred. Each rule is copied only after the previous one is successfully completed. This is particularly suggested for tapes",
metavar=('FILE', 'FROM_RSE', 'TO_RSE'))
parser.add_argument("--test",
help="It's a test. Never use it",
action='store_true')
args = parser.parse_args()
helper.make_global("admix_config",
os.path.abspath(config.get('Admix', 'config_file')))
fix = Fix()
fix.skip_rucio = args.skip_rucio
fix.priority = args.priority
try:
if args.reset_upload:
fix.reset_upload(args.reset_upload[0])
if args.fix_upload:
fix.fix_upload(args.fix_upload[0])
if args.add_rule:
fix.add_rule(args.add_rule[0], args.add_rule[1], args.add_rule[2])
if args.add_rules_from_file:
fix.add_rules_from_file(args.add_rules_from_file[0],
args.add_rules_from_file[1],
args.add_rules_from_file[2])
if args.delete_rule:
fix.delete_rule(args.delete_rule[0], args.delete_rule[1])
if args.delete_db_datum:
fix.delete_db_datum(args.delete_db_datum[0],
args.delete_db_datum[1])
if args.set_run_status:
fix.set_run_status(args.set_run_status[0], args.set_run_status[1])
if args.set_eb_status:
fix.set_eb_status(args.set_eb_status[0], args.set_eb_status[1])
if args.list_non_transferred_runs:
fix.list_non_transferred_runs()
if args.test:
fix.test()
if args.test_db_modification:
fix.test_db_modification(args.test_db_modification[0],
args.test_db_modification[1])
if args.fix_upload_db:
fix.fix_upload_db(args.fix_upload_db[0])
if args.create_upload_rules:
fix.create_upload_rules(args.create_upload_rules[0])
if args.postpone:
fix.postpone()
# if args.action == "reset_upload" and args.did:
# fix.reset_upload(args.did)
# if args.action == "add_rule" and args.did and args.fromrse and args.torse:
# fix.add_rule(args.did,args.fromrse,args.torse)
print("")
except KeyboardInterrupt:
return 0
def add_rule_run_dtype(number,dtype,from_rse,to_rse,priority=3):
config = Config()
helper.make_global("admix_config", os.path.abspath(config.get('Admix','config_file')))
RSES = helper.get_hostconfig()['rses']
DB = ConnectMongoDB()
rc = RucioSummoner()
print("Adding a new rule from {0} to {1}".format(from_rse,to_rse))
print("Run number: {0}".format(number))
print("Data type: {0}".format(dtype))
run = DB.db.find_one({'number' : number})
# Gets the status
if 'status' in run:
print('Run status: {0}'.format(run['status']))
else:
print('Run status: {0}'.format('Not available'))
#Checks if the datum of the sender exists in the DB
datum = None
for d in run['data']:
if d['type'] == dtype and d['host'] == 'rucio-catalogue' and d['location'] == from_rse:
datum = d
continue
if datum is None:
print('The datum concerning data type {0} and site {1} is missing in the DB. Forced to stop'.format(dtype,from_rse))
return(0)
print("DID: {0}".format(datum['did']))
#Checks if the destination datum exists already in the DB
for d in run['data']:
if d['type'] == dtype and d['host'] == 'rucio-catalogue' and d['location'] == to_rse:
print('Rule already exists')
return(0)
# Checks the rule status of the sender RSE
rucio_rule = rc.GetRule(upload_structure=datum['did'], rse=from_rse)
if rucio_rule['state'] != 'OK':
print('The rule in {0} is not OK. Forced to stop'.format(from_rse))
return(0)
# set the new rule
result = rc.AddConditionalRule(datum['did'], from_rse, to_rse, lifetime=None, priority=priority)
rucio_rule = rc.GetRule(datum['did'], rse=to_rse)
# Update run status
DB.db.find_one_and_update({'number': number},{'$set': {'status': 'transferring'}})
# Add a new datum in the run document
updated_fields = {'host': "rucio-catalogue",
'type': dtype,
'location': to_rse,
'lifetime': rucio_rule['expires'],
'status': 'transferring',
'did': datum['did'],
'protocol': 'rucio'
}
data_dict = datum.copy()
data_dict.update(updated_fields)
DB.AddDatafield(run['_id'], data_dict)
def reset_upload(did):
config = Config()
helper.make_global("admix_config", os.path.abspath(config.get('Admix','config_file')))
RSES = helper.get_hostconfig()['rses']
DB = ConnectMongoDB()
rc = RucioSummoner()
hash = did.split('-')[-1]
dtype = did.split('-')[0].split(':')[-1]
number = int(did.split(':')[0].split('_')[-1])
print("Resetting the upload associated to the DID: {0}".format(did))
print("Run number: {0}".format(number))
print("Data type: {0}".format(dtype))
print("Hash: {0}".format(hash))
run = DB.db.find_one({'number' : number})
# Gets the status
if 'status' in run:
print('Run status: {0}'.format(run['status']))
else:
print('Run status: {0}'.format('Not available'))
# Extracts the correct Event Builder machine who processed this run
# Then also the bootstrax state and, in case it was abandoned, the reason
if 'bootstrax' in run:
bootstrax = run['bootstrax']
eb = bootstrax['host'].split('.')[0]
else:
print('Not processed')
return(0)
# Get the EB datum and its status
ebstatus = ""
datum = None
for d in run['data']:
if d['type'] == dtype and eb in d['host']:
datum = d
if 'status' in d:
ebstatus = d['status']
if datum is None:
print('There is no EB datum. No reset is possible')
return(0)
if ebstatus != "":
print('EB status: {0}'.format(ebstatus))
else:
print('EB status: not available')
# Step zero (normally not needed): change the run status to "transferring"
# DB.db.find_one_and_update({'number':number},{'$set':{"status": "transferring"}})
# First action: remove the files stored in datamanager
files = list_file_replicas(number, dtype, hash, "LNGS_USERDISK")
print("Deleting rucio data in datamanager disk. Deleting",len(files),"files")
for file in files:
os.remove(file)
# Second action: remove the LNGS Rucio rule
rucio_rule = rc.GetRule(upload_structure=did, rse='LNGS_USERDISK')
if rucio_rule['exists']:
print("Deleting rucio rule = ", rucio_rule['id'])
rc.DeleteRule(rucio_rule['id'])
# Wait for 1 hour (plus 5 minutes of margin)
delay = 3600+60*5
print("We have to wait for {0} seconds before proceeding to the next step".format(delay))
time.sleep(delay)
else:
print("There is no rule to delete")
# Third action: set the EB status as 'eb_ready_to_upload'
DB.db.find_one_and_update({'_id': run['_id'],'data': {'$elemMatch': datum}},
{'$set': {'data.$.status': 'eb_ready_to_upload'}})
print("EB status changed to eb_ready_to_upload")
# Reload the run
run = DB.db.find_one({'number' : number})
# Gets the status
if 'status' in run:
print('New run status: {0}'.format(run['status']))
else:
print('Ru status: {0}'.format('Not available'))
# Get the EB datum and its status
ebstatus = ""
datum = None
for d in run['data']:
if d['type'] == dtype and eb in d['host']:
datum = d
if 'status' in d:
ebstatus = d['status']
# Prints the eb status as a confirmation of the performed change
if ebstatus != "":
print('New EB status: {0}'.format(ebstatus))
else:
print('New EB status: not available')