def isAtSite(ds, run):
blocks = []
ph = phedex(ds)
# get list of blocks for input dataset directly from DBS3
# documentation: https://cmsweb.cern.ch/dbs/prod/global/DBSReader/
connection = wma.init_connection('cmsweb.cern.ch')
#returns a string which represents a list, so we have to eval
blockDicts = ast.literal_eval(
wma.httpget(connection, wma.DBS3_URL + "blocks?dataset=%s&run_num=%s" % (ds, run)))
for blockDict in blockDicts:
block = blockDict['block_name']
# print "block is: %s"%block
# it's unclear what probing custodiality means; remove this check
#
# for b in filter(lambda b :b.name==block,ph.block):
# for replica in filter(lambda r : r.custodial=='y',b.replica):
# if replica.complete!='y':
# print block,'not complete at custodial site'
# #print block,'not complete at custodial site but ignoring'
# #blocks.append('#'+block.split('#')[-1])
# else:
# print block,'complete at custodial site'
# blocks.append('#'+block.split('#')[-1])
blocks.append('#' + block.split('#')[-1])
if len(blocks) == 0:
print "No possible block for %s in %s" % (run, ds)
return False
else:
print "\n\n\t Block testing succeeded for %s in %s \n\n" % (run, ds)
print blocks
return list(set(blocks))
def isAtSite(ds, run):
blocks=[]
#dbsq1='dbs search --noheader --production --query "find block,block.status where dataset = %s and run = %s"'%(ds,run)
#os.system('curl https://cmsweb.cern.ch/das/cli --insecure > das_client.py')
#os.system('chmod a+x das_client.py')
dbsq1='./das_client.py --limit=0 --query="block dataset=%s run=%s"'%(ds,run)
ph=phedex(ds)
print dbsq1
for line in os.popen(dbsq1):
if line.find("#")==-1: continue
block=line.split()[0]
#status=int(line.split()[1])
#if status!=0:
# print block,'not closed'
# continue
for b in filter(lambda b :b.name==block,ph.block):
for replica in filter(lambda r : r.custodial=='y',b.replica):
if replica.complete!='y':
print block,'not complete at custodial site'
#print block,'not complete at custodial site but ignoring'
#blocks.append('#'+block.split('#')[-1])
else:
print block,'complete at custodial site'
blocks.append('#'+block.split('#')[-1])
if len(blocks)==0:
print "No possible block for %s in %s"%(run,ds)
return False
else:
print "\n\n\t Block testing at succeeded for %s in %s \n\n"%(run,ds)
return list(set(blocks))
def isAtSite(ds, run):
blocks = []
dbsq1 = 'dbs search --noheader --production --query "find block,block.status where dataset = %s and run = %s"' % (
ds,
run,
)
ph = phedex(ds)
print dbsq1
for line in os.popen(dbsq1):
block = line.split()[0]
status = int(line.split()[1])
if status != 0:
print block, "not closed"
continue
for b in filter(lambda b: b.name == block, ph.block):
for replica in filter(lambda r: r.custodial == "y", b.replica):
if replica.complete != "y":
print block, "not complete at custodial site"
else:
blocks.append("#" + block.split("#")[-1])
if len(blocks) == 0:
print "No possible block for %s in %s" % (run, ds)
return False
else:
print "\n\n\t Block testing at succeeded for %s in %s \n\n" % (run, ds)
return list(set(blocks))
def isAtSite(ds, run):
blocks=[]
dbsq1='dbs search --noheader --production --query "find block,block.status where dataset = %s and run = %s"'%(ds,run)
ph=phedex(ds)
print dbsq1
for line in os.popen(dbsq1):
block=line.split()[0]
status=int(line.split()[1])
if status!=0:
print block,'not closed'
continue
for b in filter(lambda b :b.name==block,ph.block):
for replica in filter(lambda r : r.custodial=='y',b.replica):
if replica.complete!='y':
print block,'not complete at custodial site'
else:
blocks.append('#'+block.split('#')[-1])
if len(blocks)==0:
print "No possible block for %s in %s"%(run,ds)
return False
else:
print "\n\n\t Block testing at succeeded for %s in %s \n\n"%(run,ds)
return list(set(blocks))
def isAtSite(ds, run):
blocks = []
ph = phedex(ds)
# get list of blocks for input dataset directly from DBS3
# documentation: https://cmsweb.cern.ch/dbs/prod/global/DBSReader/
connection = wma.init_connection('cmsweb.cern.ch')
#returns a string which represents a list, so we have to eval
blockDicts = ast.literal_eval(
wma.httpget(connection,
wma.DBS3_URL + "blocks?dataset=%s&run_num=%s" % (ds, run)))
for blockDict in blockDicts:
block = blockDict['block_name']
# print "block is: %s"%block
# it's unclear what probing custodiality means; remove this check
#
# for b in filter(lambda b :b.name==block,ph.block):
# for replica in filter(lambda r : r.custodial=='y',b.replica):
# if replica.complete!='y':
# print block,'not complete at custodial site'
# #print block,'not complete at custodial site but ignoring'
# #blocks.append('#'+block.split('#')[-1])
# else:
# print block,'complete at custodial site'
# blocks.append('#'+block.split('#')[-1])
blocks.append('#' + block.split('#')[-1])
if len(blocks) == 0:
print "No possible block for %s in %s" % (run, ds)
return False
else:
print "\n\n\t Block testing succeeded for %s in %s \n\n" % (run, ds)
print blocks
return list(set(blocks))
def parallel_test(arguments, force=False):
DEBUGME = False
global countOld
req, old_useful_info = arguments
try:
if DEBUGME: print "+"
global numberofrequestnameprocessed
numberofrequestnameprocessed += 1
pdmv_request_dict={}
if DEBUGME: print "-"
##faking announced status
if req["request_name"]=='etorassa_EWK-Summer12_DR53X-00089_T1_IT_CNAF_MSS_batch77res_v1__121009_233442_92':
req["status"] = 'announced'
if req["request_name"]=='etorassa_JME-Summer12-00060_batch1_v2__120209_133317_6868':
req["status"] = 'announced'
if req["request_name"]=='jbadillo_TOP-Summer12-00234_00073_v0__140124_154429_3541':
req["status"] = 'announced'
##faking rejected status
if req["request_name"]=='spinoso_SUS-Summer12pLHE-00001_4_v1_STEP0ATCERN_130914_172555_5537':
req["status"] = "rejected"
if not "status" in req:
## this is garbage anyways
return {}
req_status = req["status"]
if req_status in skippable_stati:
print "Rquest in skippable_stati"
return None
# the name
req_name = req["request_name"]
pdmv_request_dict["pdmv_request_name"] = req_name
phedexObj = None
phedexObjInput = None
#allowSkippingOnRandom=0.1
allowSkippingOnRandom = None
if DEBUGME:
print "--"
#transform the workload in a dictionnary for convenience , can also be made an object
dict_from_workload = {}
# check if in the old and in complete_stati, just copy and go on
matching_reqs = filter(lambda oldreq: oldreq['pdmv_request_name'] == req_name, old_useful_info)
skewed = False
deadWood = False
if len(matching_reqs) == 1:
pdmv_request_dict = matching_reqs[0]
if pdmv_request_dict['pdmv_completion_eta_in_DAS'] < 0 and req["status"].startswith('running'):
skewed = True
if req["status"].startswith('running'):
skewed = True
if req["status"] == 'completed':
skewed = True
if 'pdmv_expected_events' in pdmv_request_dict and pdmv_request_dict['pdmv_expected_events'] == -1:
skewed = True
if pdmv_request_dict['pdmv_status_in_DAS'] == '?':
skewed = True
if pdmv_request_dict['pdmv_dataset_name'] == '?':
skewed = True
if pdmv_request_dict['pdmv_dataset_name'] == 'None Yet' and req["status"] not in priority_changable_stati:
skewed = True
if pdmv_request_dict['pdmv_evts_in_DAS'] < 0:
skewed = True
if pdmv_request_dict['pdmv_evts_in_DAS'] == 0 and req["status"] in ['announced']:
skewed = True
if pdmv_request_dict['pdmv_status_in_DAS'] in [None, 'PRODUCTION'] and req["status"] in ['announced', 'normal-archived']:
skewed = True
if pdmv_request_dict["pdmv_status_from_reqmngr"] != req_status and req_status != "normal-archived":
print "CHANGE OF STATUS, do something !"
skewed = True
if 'pdmv_monitor_time' in pdmv_request_dict:
up = time.mktime(time.strptime(pdmv_request_dict['pdmv_monitor_time']))
now = time.mktime(time.gmtime())
deltaUpdate = (now - up) / (60. * 60. * 24.)
##this (with a small number) will create a huge blow to the updating
if deltaUpdate > 10 and countOld <= 100:
print 'too long ago'
skewed = True
countOld += 1
if not 'pdmv_monitor_time' in pdmv_request_dict:
skewed = True
##known bad status
for bad in []:#'JME-Summer12_DR53X-00098','JME-Summer12_DR53X-00097','vlimant_Run2012']:
if bad in req_name:
skewed = True
break
#add that field after the fact
if not 'pdmv_running_sites' in pdmv_request_dict:
pdmv_request_dict['pdmv_running_sites'] = []
##needs to be used by user drives the need to T2/T3 field to be up to date
needsToBeUsed = True
notNeededByUSers = ['GEN-SIM', 'GEN-RAW']
for eachTier in notNeededByUSers:
if pdmv_request_dict["pdmv_dataset_name"].endswith(eachTier):
needsToBeUsed = False
if 'pdmv_status_in_DAS' in pdmv_request_dict and not pdmv_request_dict['pdmv_status_in_DAS'] in ['VALID', 'PRODUCTION']:
needsToBeUsed = False
if not needsToBeUsed:
print "not doing phedex call"
noSites = True
if 'pdmv_at_T2' in pdmv_request_dict and len(pdmv_request_dict['pdmv_at_T2']):
noSites = False
if 'pdmv_at_T3' not in pdmv_request_dict:
pdmv_request_dict['pdmv_at_T3'] = []
if not force:
noSites = False
if noSites and needsToBeUsed and (not 'pdmv_at_T2' in pdmv_request_dict or pdmv_request_dict['pdmv_at_T2'] == []):
if not phedexObj:
phedexObj = phedex(pdmv_request_dict["pdmv_dataset_name"])
pdmv_request_dict['pdmv_at_T2'] = atT2(phedexObj)
elif not 'pdmv_at_T2' in pdmv_request_dict:
pdmv_request_dict['pdmv_at_T2'] = []
if 'pdmv_input_dataset' not in pdmv_request_dict:
if not dict_from_workload:
print "##DEBUG## trying to get dict_from_workload"
dict_from_workload = getDictFromWorkload(req_name)
if not dict_from_workload:
return {}
if 'InputDatasets' in dict_from_workload[req_name] and len(dict_from_workload[req_name]['InputDatasets']):
pdmv_request_dict['pdmv_input_dataset'] = dict_from_workload[req_name]['InputDatasets'][0]
else:
pdmv_request_dict['pdmv_input_dataset'] = ''
if not 'pdmv_custodial_sites' in pdmv_request_dict or pdmv_request_dict['pdmv_custodial_sites']==[]:
if not phedexObj:
phedexObj = phedex(pdmv_request_dict["pdmv_dataset_name"])
pdmv_request_dict['pdmv_custodial_sites'] = custodials(phedexObj)
if not 'pdmv_open_evts_in_DAS' in pdmv_request_dict:
pdmv_request_dict['pdmv_open_evts_in_DAS'] = 0
if 'amaltaro' in req_name and ('type' in req and req['type'] == 'TaskChain'):
skewed = False
#needs to be put by hand once in all workflows
if (pdmv_request_dict['pdmv_status_from_reqmngr'] in complete_stati) and ('pdmv_performance' not in pdmv_request_dict or pdmv_request_dict['pdmv_performance']=={}):
print "skewed with no perf"
from Performances import Performances
pdmv_request_dict['pdmv_performance'] = Performances(req_name)
if (pdmv_request_dict['pdmv_status_from_reqmngr'] in complete_stati) and not skewed and not force:
print "** %s is there already" % req_name
return pdmv_request_dict
print req_name," is skewed ? ",skewed
if allowSkippingOnRandom != None and skewed and random.random()> allowSkippingOnRandom and not force:
return pdmv_request_dict
#if needs to be added to the db since it is not present already
else:
decide = random.random()
if allowSkippingOnRandom != None and decide > allowSkippingOnRandom:
#update only 1 out of 10
print "Skipping new request",req_name,len(matching_reqs)
return {}
if len(matching_reqs)==1:
print "Updating", req_name
else:
print "Processing", req_name
if not 'pdmv_configs' in pdmv_request_dict or pdmv_request_dict['pdmv_configs'] == []:
print "##DEBUG## trying to get dict_from_workload2"
if not dict_from_workload:
dict_from_workload = getDictFromWorkload(req_name)
if not dict_from_workload:
return {}
pdmv_request_dict['pdmv_configs'] = configsFromWorkload(dict_from_workload[req_name])
# assign the date
if not 'pdmv_submission_date' in pdmv_request_dict or not 'pdmv_submission_time' in pdmv_request_dict:
##load it on demand only
print "##DEBUG## trying to get dict_from_workload3"
if not dict_from_workload:
dict_from_workload = getDictFromWorkload(req_name)
if not dict_from_workload:
return {}
pdmv_request_dict["pdmv_submission_date"] = datelist_to_str(
dict_from_workload[req_name]['RequestDate'])
pdmv_request_dict["pdmv_submission_time"] = timelist_to_str(
dict_from_workload[req_name]['RequestDate'])
if DEBUGME:
print "----"
if pdmv_request_dict["pdmv_submission_date"][:2] in ["11", "12"]:
print "Very old request " + req_name
return {}
if DEBUGME:
print "-----"
# the request itself!
pdmv_request_dict["pdmv_request"] = req
# check the status
req_status = req["status"]
pdmv_request_dict["pdmv_status_from_reqmngr"] = req_status
if req_status in priority_changable_stati:
pdmv_request_dict["pdmv_status"] = "ch_prio"
else:
pdmv_request_dict["pdmv_status"] = "fix_prio"
#request type
pdmv_request_dict['pdmv_type'] = req["type"]
if DEBUGME:
print "------"
# number of running days
if pdmv_request_dict["pdmv_status_from_reqmngr"].startswith('running'):
#set only when the requests is in running mode, not anytime after
pdmv_request_dict["pdmv_running_days"] = get_running_days(pdmv_request_dict["pdmv_submission_date"])
elif not "pdmv_running_days" in pdmv_request_dict:
pdmv_request_dict["pdmv_running_days"] = 0
if DEBUGME:
print "-------"
# get the running jobs
pdmv_request_dict["pdmv_all_jobs"] = get_all_jobs(req)
# get the running jobs
pdmv_request_dict["pdmv_running_jobs"] = get_running_jobs(req)
# get the jobs pending in the batch systems
pdmv_request_dict["pdmv_pending_jobs"] = get_pending_jobs(req)
# get Extras --> slow down---------------
raw_expected_evts = None
priority = -1
# assign the campaign and the prepid guessing from the string, then try to get it from the extras
if not dict_from_workload:
print "##DEBUG## trying to get dict_from_workload4"
dict_from_workload = getDictFromWorkload(req_name)
if not dict_from_workload:
return {}
if 'RequestTransition' in dict_from_workload[req_name]:
status_history = []
transitions = dict_from_workload[req_name]['RequestTransition']
for transition in transitions:
status_history.append(transition['Status'])
pdmv_request_dict['pdmv_status_history_from_reqmngr'] = status_history
else:
pdmv_request_dict['pdmv_status_history_from_reqmngr'] = []
if ('PrepID' in dict_from_workload[req_name]) and (dict_from_workload[req_name]['PrepID'] != None):
print "##DEBUG## prepid is not None"
prep_id = dict_from_workload[req_name]['PrepID']
else:
prep_id = 'No-Prepid-Found'
##known comuping fck-ups
if 'RequestName' in dict_from_workload[req_name]:
__computing_fck_ups = {
"alahiff_HIG-Summer12DR53X-02171_00353_v0__141218_204531_2518" : "HIG-Summer12DR53X-02171",
"alahiff_TOP-Summer12DR53X-00276_00354_v0__141218_204512_1227" : "TOP-Summer12DR53X-00276",
"alahiff_TOP-Summer12DR53X-00275_00355_v0__141218_204521_7100" : "TOP-Summer12DR53X-00275",
"alahiff_BTV-Phys14DR-00012_00033_v0_castor_141217_165406_5018" : "BTV-Phys14DR-00012",
"alahiff_B2G-Phys14DR-00058_00084_v0__150105_195309_2573" : "B2G-Phys14DR-00058",
"alahiff_EXO-Phys14DR-00078_00053_v0__141217_165317_4158" : "EXO-Phys14DR-00078",
"alahiff_EXO-Phys14DR-00078_00053_v0__141217_165326_9241" : "EXO-Phys14DR-00078",
"alahiff_BTV-Phys14DR-00011_00033_v0_castor_141217_165355_3898" : "BTV-Phys14DR-00011",
"alahiff_BTV-Phys14DR-00016_00033_v0_castor_141217_165416_6880" : "BTV-Phys14DR-00016",
"alahiff_BTV-Phys14DR-00023_00033_v0_castor_141217_165305_1759" : "BTV-Phys14DR-00023",
"alahiff_HIG-Phys14DR-00042_00080_v0__150105_195258_4618" : "HIG-Phys14DR-00042",
"jen_a_ACDC_HIG-Phys14DR-00042_00080_v0__150113_211610_5968" : "HIG-Phys14DR-00042",
"alahiff_BTV-Phys14DR-00023_00033_v0_castor_141217_165437_3766" : "BTV-Phys14DR-00023",
"alahiff_BTV-Phys14DR-00021_00033_v0_castor_141217_165426_7676" : "BTV-Phys14DR-00021",
"alahiff_HIG-Phys14DR-00041_00081_v0__150105_195249_1950" : "HIG-Phys14DR-00041",
"jen_a_ACDC_HIG-Phys14DR-00041_00081_v0__150113_211059_5006" : "HIG-Phys14DR-00041"
}
if dict_from_workload[req_name]['RequestName'] in __computing_fck_ups:
prep_id = __computing_fck_ups[dict_from_workload[req_name]['RequestName']]
pdmv_request_dict["pdmv_prep_id"] = prep_id
campaign = get_campaign_from_prepid(prep_id)
pdmv_request_dict["pdmv_campaign"] = campaign
req_extras = {}
# Priority
retrievePriority = False
if not 'pdmv_priority' in pdmv_request_dict:
retrievePriority = True
if not 'pdmv_present_priority' in pdmv_request_dict:
retrievePriority = True
if pdmv_request_dict['pdmv_status_from_reqmngr'] in priority_changable_stati + ['acquired', 'running', 'running-open', 'running-closed']:
retrievePriority = True
if retrievePriority:
print "##DEBUG## trying to get dict_from_workload5"
if not dict_from_workload:
dict_from_workload = getDictFromWorkload(req_name)
if not dict_from_workload:
return {}
if 'InitialPriority' in dict_from_workload[req_name]:
pdmv_request_dict["pdmv_priority"] = dict_from_workload[req_name]['InitialPriority']
else:
pdmv_request_dict["pdmv_priority"] = dict_from_workload[req_name]['RequestPriority']
if DEBUGME:
print "----------"
# Present priority
pdmv_request_dict["pdmv_present_priority"] = dict_from_workload[req_name]['RequestPriority']
if DEBUGME:
print "-----------"
#------------------------------------------
# Query yet another service to get the das entry of the prepid
## try and reduce the number of calls to get_dataset_name url
dataset_name = "None Yet"
dataset_list = []
if 'pdmv_dataset_name' in pdmv_request_dict:
dataset_name = pdmv_request_dict["pdmv_dataset_name"]
if 'pdmv_dataset_list' in pdmv_request_dict:
dataset_list = pdmv_request_dict['pdmv_dataset_list']
makedsnquery = False
if (not 'pdmv_dataset_name' in pdmv_request_dict):
makedsnquery = True
if 'pdmv_dataset_name' in pdmv_request_dict and (pdmv_request_dict["pdmv_dataset_name"] in ['?','None Yet'] or 'None-' in pdmv_request_dict["pdmv_dataset_name"] or '24Aug2012' in pdmv_request_dict["pdmv_dataset_name"]):
makedsnquery = True
if 'pdmv_dataset_list' in pdmv_request_dict and not pdmv_request_dict['pdmv_dataset_list']:
makedsnquery = True
if req_status in priority_changable_stati:
makedsnquery = False
if req_status == "assigned":
makedsnquery = True
if force:
makedsnquery = True
if makedsnquery:
dataset_name, dataset_list = get_dataset_name(
dict_from_workload[req_name]['OutputDatasets'])
pdmv_request_dict["pdmv_dataset_name"] = dataset_name
pdmv_request_dict["pdmv_dataset_list"] = dataset_list
deltaUpdate = 0
##remove an old field
if 'pdmv_update_time' in pdmv_request_dict:
pdmv_request_dict.pop('pdmv_update_time')
if 'pdmv_monitor_time' in pdmv_request_dict:
up = time.mktime(time.strptime(pdmv_request_dict['pdmv_monitor_time']))
now = time.mktime(time.gmtime())
deltaUpdate = (now-up) / (60. * 60. * 24.)
else:
pdmv_request_dict["pdmv_monitor_time"] = time.asctime()
if deltaUpdate > 2. or DEBUGME or force or pdmv_request_dict['pdmv_status_from_reqmngr'].startswith('running') or skewed:
#do the expensive procedure rarely or for request which have been update more than 2 days ago
print "\tSumming for", req_name
status,evts,openN = get_status_nevts_from_dbs(pdmv_request_dict["pdmv_dataset_name"])
## aggregate number of events for all output datasets
pdmv_request_dict['pdmv_dataset_statuses'] = {}
for other_ds in pdmv_request_dict['pdmv_dataset_list']:
if other_ds == pdmv_request_dict["pdmv_dataset_name"]:
other_status,other_evts,other_openN = status,evts,openN
else:
other_status,other_evts,other_openN = get_status_nevts_from_dbs( other_ds )
pdmv_request_dict['pdmv_dataset_statuses'][other_ds] = {
'pdmv_status_in_DAS': other_status,
'pdmv_evts_in_DAS': other_evts,
'pdmv_open_evts_in_DAS': other_openN}
if status:
print "\t\tUpdating %s %s %s" % (status, evts, openN)
pdmv_request_dict["pdmv_status_in_DAS"] = status
pdmv_request_dict["pdmv_evts_in_DAS"] = evts
pdmv_request_dict["pdmv_open_evts_in_DAS"] = openN
## add this only when the summing is done
else:
print "\t\tPassed %s %s %s" % (status, evts, openN)
pass
pdmv_request_dict["pdmv_monitor_time"] = time.asctime()
else:
print "\tSkipping summing for",req_name
## move the next line out to go over the "pass" above
if not "pdmv_status_in_DAS" in pdmv_request_dict:
pdmv_request_dict["pdmv_status_in_DAS"] = None
if not "pdmv_evts_in_DAS" in pdmv_request_dict:
pdmv_request_dict["pdmv_evts_in_DAS"] = 0
if not "pdmv_open_evts_in_DAS" in pdmv_request_dict:
pdmv_request_dict["pdmv_open_evts_in_DAS"] = 0
if not 'pdmv_expected_events' in pdmv_request_dict:
pdmv_request_dict['pdmv_expected_events'] = -1
if not 'pdmv_expected_events_per_ds' in pdmv_request_dict:
pdmv_request_dict['pdmv_expected_events_per_ds'] = {}
## why having "expected events" not calculated if the output dataset has no status yet ?
#to prevent stupid update from garbage requests announced or skewed but enrecoverable
if pdmv_request_dict["pdmv_expected_events"] < 0 or (skewed and pdmv_request_dict['pdmv_status_in_DAS'] and not pdmv_request_dict['pdmv_status_in_DAS'] in ['DELETED', 'DEPRECATED'] and pdmv_request_dict["pdmv_expected_events"] < 0) or force:
print "Getting expected events"
print "##DEBUG## trying to get dict_from_workload5.5"
if not dict_from_workload:
dict_from_workload = getDictFromWorkload(req_name)
if not dict_from_workload:
return {}
raw_expected_evts = get_expected_events_withdict(dict_from_workload[req_name])
if DEBUGME:
print "--------"
# get the expected number of evts
expected_evts =- 1
if raw_expected_evts == None or raw_expected_evts == 'None':
expected_evts =- 1
else:
expected_evts = int(raw_expected_evts)
if DEBUGME:
print "---------"
pdmv_request_dict["pdmv_expected_events"] = expected_evts
## get an expected per output dataset if possible
##Reqmgr2 doesn't support this so we set to empty dict for all wfs
pdmv_request_dict['pdmv_expected_events_per_ds'] = {}
completion = 0
if pdmv_request_dict["pdmv_expected_events"] > 0.:
completion = 100*(float(pdmv_request_dict["pdmv_evts_in_DAS"]) + float(pdmv_request_dict["pdmv_open_evts_in_DAS"]))/pdmv_request_dict["pdmv_expected_events"]
pdmv_request_dict["pdmv_completion_in_DAS"] = float("%2.2f" % completion)
if DEBUGME:
print "----------"
pdmv_request_dict["pdmv_completion_eta_in_DAS"] = calc_eta(pdmv_request_dict["pdmv_completion_in_DAS"], pdmv_request_dict["pdmv_running_days"])
if not 'pdmv_running_sites' in pdmv_request_dict:
pdmv_request_dict['pdmv_running_sites'] = []
#find open blocks for running at sites
if pdmv_request_dict['pdmv_status_from_reqmngr'].startswith('running'):
if not phedexObj:
phedexObj = phedex(pdmv_request_dict["pdmv_dataset_name"])
pdmv_request_dict['pdmv_running_sites'] = runningSites(phedexObj)
if not 'pdmv_assigned_sites' in pdmv_request_dict:
pdmv_request_dict['pdmv_assigned_sites'] = []
if not 'pdmv_custodial_sites' in pdmv_request_dict:
pdmv_request_dict['pdmv_custodial_sites'] = []
if (pdmv_request_dict['pdmv_assigned_sites'] == []
or pdmv_request_dict['pdmv_custodial_sites'] == []):
print "##DEBUG## trying to get dict_from_workload6"
if not dict_from_workload:
dict_from_workload = getDictFromWorkload(req_name)
if not dict_from_workload:
return {}
#find out the keys that has constraints
if 'SiteWhitelist' in dict_from_workload[req_name]:
sites = set(dict_from_workload[req_name]['SiteWhitelist'])
else:
sites = []
if pdmv_request_dict['pdmv_status_in_DAS']:
if not phedexObj:
phedexObj = phedex(pdmv_request_dict["pdmv_dataset_name"])
pdmv_request_dict['pdmv_custodial_sites'] = custodials(phedexObj)
##TO-DO works only for 1st entry of site?
if pdmv_request_dict['pdmv_custodial_sites'] == []:
pdmv_request_dict['pdmv_custodial_sites'] = filter(
lambda s : s.startswith('T1_'), sites)
pdmv_request_dict['pdmv_assigned_sites'] = list(sites)
needsToBeUsed = True
notNeededByUSers = ['GEN-SIM', 'GEN-RAW']
for eachTier in notNeededByUSers:
if pdmv_request_dict["pdmv_dataset_name"].endswith(eachTier):
needsToBeUsed = False
noSites = True
if 'pdmv_at_T2' in pdmv_request_dict and len(pdmv_request_dict['pdmv_at_T2']):
noSites = False
if 'pdmv_at_T3' not in pdmv_request_dict:
pdmv_request_dict['pdmv_at_T3'] = []
if noSites and needsToBeUsed and (not 'pdmv_at_T2' in pdmv_request_dict or pdmv_request_dict['pdmv_at_T2']==[]):
if not phedexObj:
phedexObj = phedex(pdmv_request_dict["pdmv_dataset_name"])
pdmv_request_dict['pdmv_at_T2'] = atT2(phedexObj)
elif not 'pdmv_at_T2' in pdmv_request_dict:
pdmv_request_dict['pdmv_at_T2'] = []
if (pdmv_request_dict['pdmv_status_from_reqmngr'] in complete_stati) and ('pdmv_performance' not in pdmv_request_dict or pdmv_request_dict['pdmv_performance']=={}):
print "updating for perf"
from Performances import Performances
pdmv_request_dict['pdmv_performance'] = Performances(req_name)
print "#numebr of processed: %s" % (numberofrequestnameprocessed)
return pdmv_request_dict
except:
import traceback
##something failed lets produce traceback file for future investigations...
trf = '/home/pdmvserv/stats/traceback/traceback_%s.txt' % (req["request_name"])
print req["request_name"],"IS A DEAD FAILING REQUEST"
print trf
tr = open(trf, 'w')
tr.write("dict_from_workload\n")
tr.write(json.dumps(dict_from_workload))
tr.write("\n")
tr.write("pdmv_request_dict\n")
tr.write(json.dumps(pdmv_request_dict))
tr.write("\n")
tr.write("req:\n")
tr.write(json.dumps(req))
tr.write(traceback.format_exc())
tr.close()
return {}
