def __init__(self, sub_directory, care_on_existing=True, clean_on_exit=True):
self.cleanup = clean_on_exit
l_type=locator()
directory = l_type.workLocation()
self.directory = os.path.abspath(directory) + '/' + sub_directory + '/'
self.careOfExistingDirectory = care_on_existing
# check if directory is empty
if not self.directory:
self.logger.error('Data directory is not defined', handler=self.hname)
raise Exception('Data directory is not defined.')
# check if exists (and force)
if os.path.exists(self.directory):
if self.careOfExistingDirectory:
self.logger.error( os.popen('echo %s; ls -f %s'%(self.directory, self.directory)).read())
self.logger.error('Directory ' + self.directory + ' already exists.')
raise Exception('Data directory %s already exists'%(self.directory))
else:
self.logger.log('Directory ' + self.directory + ' already exists.')
else:
self.logger.log('Creating directory :'+self.directory)
# recursively create any needed parents and the dir itself
os.makedirs(self.directory)
def default(self, *vpath, **params):
method = getattr(self, cherrypy.request.method, None)
if not method:
raise cherrypy.HTTPError(405, "Method not implemented.")
if self.access_limit is not None:
self.logger.log('Setting access limit to access_rights.%s (%s)' % (roles[self.access_limit], self.access_limit))
self.authenticator.set_limit(self.access_limit)
elif cherrypy.request.method in self.limit_per_method:
self.authenticator.set_limit(self.limit_per_method[cherrypy.request.method])
else:
raise cherrypy.HTTPError(403, 'You cannot access this page with method %s' % cherrypy.request.method )
user_p = user_pack()
l_type = locator()
if not user_p.get_username():
#meaning we are going public, only allow GET.
#if cherrypy.request.method != 'GET' or not l_type.isDev():
# raise cherrypy.HTTPError(403, 'User credentials were not provided.')
if not 'public' in str(cherrypy.url()):
self.logger.error('From within %s, adfs-login not found: \n %s \n %s' % (self.__class__.__name__, str(cherrypy.request.headers), str(cherrypy.url()) ))
else:
if not self.authenticator.can_access(user_p.get_username()):
raise cherrypy.HTTPError(403, 'You cannot access this page, the limit for the page is {0} ({1})'.format(roles[self.authenticator.get_limit()],
self.authenticator.get_limit()))
# counter for calls
with locker.lock("rest-call-counter"):
self.counter[method.im_class.__name__][method.__name__] += 1
return method(*vpath, **params)
def GET(self, *args):
"""
Goes through invalidation documents and display (/) and announce them to data ops (/announce)
"""
announce = False
clear = False
if len(args) != 0:
if args[0] == 'announce':
announce = True
if args[0] == 'clear':
clear = True
idb = database('invalidations')
r_to_be_rejected = map(invalidation, idb.queries(['status==new', 'type==request']))
ds_to_be_invalidated = map(invalidation, idb.queries(['status==new', 'type==dataset']))
ds_to_be_invalidated = filter(lambda ds : not 'None-' in ds.get_attribute('object'), ds_to_be_invalidated)
l_type = locator()
def add_prepid(invalid, html):
if 'prepid' in invalid.json():
html += 'for request <a href=%srequests?prepid=%s> %s </a>' % (
l_type.baseurl(), invalid.get_attribute('prepid'), invalid.get_attribute('prepid'))
def print_invalidations(invalids):
a_text=''
for invalid in invalids:
a_text += ' %s\n' % (invalid.get_attribute('object'))
return a_text
html = '<html><body>\n'
html += 'Requests to be aborted/rejected <br>\n'
html += '<ul>\n'
for r in r_to_be_rejected:
html += '<li> <a href=%sreqmgr/view/details/%s> %s </a>' % (l_type.cmsweburl(), r.get_attribute('object'), r.get_attribute('object') )
add_prepid(r, html)
html += '</li>\n'
html += '</ul>\n'
html += 'Datasets to be invalidated <br>\n'
html += '<ul>\n'
for ds in ds_to_be_invalidated:
html += '<li> %s ' % ( ds.get_attribute('object') )
add_prepid(ds, html)
html += '</li>\n'
html += '</ul>\n'
html += '<a href=%srestapi/invalidations/inspect/clear> Clear invalidation withouth announcing</a><br>\n'%( l_type.baseurl() )
html += '<a href=%srestapi/invalidations/inspect/announce> Announce invalidations</a><br>\n'%( l_type.baseurl() )
html += '<a href=%srestapi/invalidations/inspect> Back</a><br>\n'%( l_type.baseurl() )
html += '</html></body>\n'
if announce:
announcer = Announcer()
announcer.announce(ds_to_be_invalidated, r_to_be_rejected)
if clear:
__clearer = Clearer()
__clearer.clean()
return html
def notify_batch(self, batch_id, message_notes):
message = message_notes
to_who = [settings.get_value('service_account')]
l_type = locator()
if l_type.isDev():
to_who.append(settings.get_value('hypernews_test'))
else:
to_who.append(settings.get_value('dataops_announce'))
single_batch = batch(self.bdb.get(batch_id))
subject = single_batch.get_subject('[Notification]')
current_message_id = single_batch.get_attribute('message_id')
self.logger.info('current msgID: %s' % current_message_id)
if current_message_id != '':
result = single_batch.notify(subject, message, who=to_who, sender=None, reply_msg_ID=current_message_id)
self.logger.info('result if True : %s' % result)
else:
result = single_batch.notify(subject, message, who=to_who, sender=None)
self.logger.info('result if False : %s' % result)
notification(
subject,
message,
[],
group=notification.BATCHES,
target_role='production_manager',
action_objects=[single_batch.get_attribute('prepid')],
object_type='batches',
base_object=single_batch
)
single_batch.update_history({'action': 'notify', 'step': message})
single_batch.reload()
return {'results': result}
def default(self, *vpath, **params):
method = getattr(self, cherrypy.request.method, None)
if not method:
raise cherrypy.HTTPError(405, "Method not implemented.")
if self.access_limit is not None:
self.logger.log('Setting access limit to access_rights.%s (%s)' % (roles[self.access_limit], self.access_limit))
self.authenticator.set_limit(self.access_limit)
elif cherrypy.request.method in self.limit_per_method:
self.authenticator.set_limit(self.limit_per_method[cherrypy.request.method])
else:
raise cherrypy.HTTPError(403, 'You cannot access this page with method %s' % cherrypy.request.method )
user_p = user_pack()
l_type = locator()
if not user_p.get_username():
#meaning we are going public, only allow GET.
#if cherrypy.request.method != 'GET' or not l_type.isDev():
# raise cherrypy.HTTPError(403, 'User credentials were not provided.')
if not 'public' in str(cherrypy.url()):
self.logger.error('From within %s, adfs-login not found: \n %s \n %s' % (self.__class__.__name__, str(cherrypy.request.headers), str(cherrypy.url()) ))
else:
if not self.authenticator.can_access(user_p.get_username()):
raise cherrypy.HTTPError(403, 'You cannot access this page, the limit for the page is {0} ({1})'.format(roles[self.authenticator.get_limit()],
self.authenticator.get_limit()))
# counter for calls
with locker.lock("rest-call-counter"):
key = method.im_class.__name__ + method.__name__
try:
RESTResource.counter[key] += 1
except KeyError:
RESTResource.counter[key] = 0
return method(*vpath, **params)
def notify_batch(self, batch_id, message_notes):
message = message_notes
to_who = [settings().get_value('service_account')]
l_type = locator()
if l_type.isDev():
to_who.append(settings().get_value('hypernews_test'))
else:
to_who.append(settings().get_value('dataops_announce'))
single_batch = batch(self.bdb.get(batch_id))
subject = single_batch.get_subject('[Notification]')
current_message_id = single_batch.get_attribute('message_id')
self.logger.log('current msgID: %s' % current_message_id)
if current_message_id != '':
result = single_batch.notify(subject,
message,
who=to_who,
sender=None,
reply_msg_ID=current_message_id)
self.logger.log('result if True : %s' % result)
else:
result = single_batch.notify(subject,
message,
who=to_who,
sender=None)
self.logger.log('result if False : %s' % result)
single_batch.update_history({'action': 'notify', 'step': message})
single_batch.reload()
return {'results': result}
def notify_batch(self, batch_id, message_notes):
message = message_notes
to_who = [settings().get_value('service_account')]
l_type = locator()
if l_type.isDev():
to_who.append( settings().get_value('hypernews_test'))
else:
to_who.append( settings().get_value('dataops_announce' ))
single_batch = batch(self.bdb.get(batch_id))
subject = single_batch.get_subject('[Notification]')
current_message_id = single_batch.get_attribute('message_id')
self.logger.log('current msgID: %s' %current_message_id)
if current_message_id != '':
result = single_batch.notify(subject,message,who=to_who,sender=None,reply_msg_ID=current_message_id)
self.logger.log('result if True : %s' %result)
else:
result = single_batch.notify(subject,message,who=to_who,sender=None)
self.logger.log('result if False : %s' %result)
self.update_history({'action':'notify', 'step' : message})
self.reload()
return {'results': result}
def request_join(self, req):
with locker.lock(req.get_attribute('prepid')):
chain = req.get_attribute("member_of_chain")
chain.append(self.get_attribute('_id'))
req.set_attribute("member_of_chain", chain)
loc = locator()
req.notify(
"Request {0} joined chain".format(req.get_attribute('prepid')),
"Request {0} has successfully joined chain {1}\n\n{2}\n".format(
req.get_attribute('prepid'), self.get_attribute('_id'),
"/".join([
loc.baseurl(),
"requests?prepid={0}".format(req.get_attribute('prepid'))
])),
accumulate=True)
req.update_history({
'action': 'join chain',
'step': self.get_attribute('_id')
})
if not req.get_attribute('prepid') in self.get_attribute('chain'):
chain = self.get_attribute('chain')
chain.append(req.get_attribute('prepid'))
self.set_attribute("chain", chain)
self.update_history({
'action': 'add request',
'step': req.get_attribute('prepid')
})
def make_command(self,mcm_r=None):
l_type = locator()
cmd = 'cd %s \n' % ( l_type.workLocation())
if mcm_r:
cmd += mcm_r.make_release()
cmd += 'export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOSTNAME/voms_proxy.cert\n'
cmd += 'export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}\n'
there = ''
if l_type.isDev():
there = '--wmtest --wmtesturl cmsweb-testbed.cern.ch'
cmd += 'wmcontrol.py --url-dict %s/public/restapi/chained_requests/get_dict/%s %s \n'%(l_type.baseurl(), self.prepid, there)
return cmd
def GET(self, *args):
"""
Send a reminder to the production managers for existing opened mccm documents
"""
mdb = database('mccms')
mccms = mdb.queries(['status==new'])
udb = database('users')
block_threshold = 0
if len(args):
block_threshold = int(args[0])
mccms = filter(lambda m: m['block'] <= block_threshold, mccms)
mccms = sorted(mccms, key=lambda m: m['block'])
if len(mccms) == 0:
return dumps({
"results":
True,
"message":
"nothing to remind of at level %s, %s" %
(block_threshold, mccms)
})
l_type = locator()
com = communicator()
subject = 'Gentle reminder on %s tickets to be operated by you' % (
len(mccms))
message = '''\
Dear Production Managers,
please find below the details of %s opened MccM tickets that need to be operated.
''' % (len(mccms))
for mccm in mccms:
message += 'Ticket : %s (block %s)\n' % (mccm['prepid'],
mccm['block'])
message += ' %smccms?prepid=%s \n\n' % (l_type.baseurl(),
mccm['prepid'])
message += '\n'
to_who = [settings().get_value('service_account')]
to_who.extend(
map(lambda u: u['email'],
udb.query(query="role==production_manager", page_num=-1)))
com.sendMail(to_who, subject, message)
return dumps({
"results": True,
"message": map(lambda m: m['prepid'], mccms)
})
def make_command(self):
l_type = locator()
command = 'export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOSTNAME/voms_proxy.cert\n'
command += 'source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh\n'
test_path = ''
test_params = ''
if l_type.isDev():
test_path = '_testful'
test_params = '--wmtest --wmtesturl cmsweb-testbed.cern.ch'
command += 'python /afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol%s/wmapprove.py --workflows %s %s\n' % (
test_path, self.workflows, test_params)
return command
def streaming_function():
mccms_db = database('mccms')
users_db = database('users')
generator_contacts_query = users_db.construct_lucene_query({'role': 'generator_contact'})
generator_contacts = users_db.full_text_search("search", generator_contacts_query, page=-1)
generator_contacts_by_pwg = {}
generator_contacts_emails = set()
for contact in generator_contacts:
for pwg in contact.get('pwg', []):
if pwg not in generator_contacts_by_pwg:
generator_contacts_by_pwg[pwg] = []
generator_contacts_by_pwg[pwg].append(contact.get('email'))
generator_contacts_emails.add(contact.get('email'))
__query = mccms_db.construct_lucene_query({'status': 'new'})
mccms_tickets = mccms_db.full_text_search('search', __query, page=-1)
authors_tickets_dict = dict()
yield '<pre>'
for ticket in mccms_tickets:
yield 'Processing ticket %s\n' % (ticket['prepid'])
mccm_ticket = mccm(json_input=ticket)
pwg = mccm_ticket.get_attribute('pwg')
authors = mccm_ticket.get_actors(what='author_email')
yield '%s worked on %s\n' % (authors, ticket['prepid'])
authors = filter(lambda e: e in generator_contacts_emails, list(set(authors + generator_contacts_by_pwg.get(pwg, []))))
yield '%s will be notified about %s\n' % (authors, ticket['prepid'])
for author_email in authors:
if author_email in generator_contacts_emails:
if author_email not in authors_tickets_dict:
authors_tickets_dict[author_email] = set()
authors_tickets_dict[author_email].add(ticket['prepid'])
subject_template = 'Gentle reminder on %s ticket%s to be operated by you'
message_template = ('Dear GEN Contact,\nPlease find below the details of %s MccM ticket%s in status "new". ' +
'Please present them in next MccM googledoc or cancel tickets if these are not needed anymore.\n\n')
base_url = locator().baseurl()
mail_communicator = communicator()
service_account = settings.get_value('service_account')
for author_email, ticket_prepids in authors_tickets_dict.iteritems():
num_tickets = len(ticket_prepids)
subject = subject_template % (num_tickets, '' if num_tickets == 1 else 's')
message = message_template % (num_tickets, '' if num_tickets == 1 else 's')
for ticket_prepid in ticket_prepids:
message += 'Ticket: %s\n%smccms?prepid=%s\n\n' % (ticket_prepid, base_url, ticket_prepid)
yield '.'
yield '\n'
message += 'You received this email because you are listed as generator contact of physics group(s) of these tickets.\n'
self.logger.info('Email:%s\nSubject: %s\nMessage:%s' % (author_email, subject, message))
mail_communicator.sendMail([author_email, service_account], subject, message)
yield 'Email sent to %s\n' % (author_email)
def make_command(self, mcm_r=None):
l_type = locator()
cmd = 'cd %s \n' % (l_type.workLocation())
if mcm_r:
cmd += mcm_r.make_release()
cmd += 'export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOSTNAME/voms_proxy.cert\n'
there = ''
if l_type.isDev():
there = '--wmtest --wmtesturl cmsweb-testbed.cern.ch'
cmd += 'export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}\n'
cmd += 'wmcontrol.py --dont_approve --url-dict %spublic/restapi/chained_requests/get_dict/%s %s \n' % (
l_type.baseurl(), self.prepid, there)
return cmd
def streaming_function():
mccms_db = database('mccms')
users_db = database('users')
__query = mccms_db.construct_lucene_query({'status': 'new'})
mccms_tickets = mccms_db.full_text_search('search', __query, page=-1)
non_gen_contact_authors = set()
authors_tickets_dict = dict()
emails_prepids = dict()
for ticket in mccms_tickets:
yield '\nProcessing ticket %s' % (ticket['prepid'])
mccm_ticket = mccm(json_input=ticket)
authors = mccm_ticket.get_actors(what='author_email')
for author_email in authors:
if author_email in authors_tickets_dict:
authors_tickets_dict[author_email].append(ticket['prepid'])
elif author_email not in non_gen_contact_authors:
__role_query = users_db.construct_lucene_query({'email': author_email})
result = users_db.full_text_search('search', __role_query, page=-1, include_fields='role,prepid')
time.sleep(0.5) # we don't want to crash DB with a lot of single queries
if result and result[0]['role'] == 'generator_contact':
authors_tickets_dict[author_email] = [ticket['prepid']]
emails_prepids[author_email] = result[0]['prepid']
else:
non_gen_contact_authors.add(author_email)
yield '.'
subject_part1 = 'Gentle reminder on %s '
subject_part2 = ' to be operated by you'
message_part1 = 'Dear GEN Contact, \nPlease find below the details of %s MccM '
message_part2 = ' in status "new". Please present them in next MccM googledoc or cancel tickets if these are not needed anymore.\n\n'
base_url = locator().baseurl()
mail_communicator = communicator()
for author_email, ticket_prepids in authors_tickets_dict.iteritems():
num_tickets = len(ticket_prepids)
full_message = (message_part1 % (num_tickets)) + ('ticket' if num_tickets == 1 else 'tickets') + message_part2
for ticket_prepid in ticket_prepids:
full_message += 'Ticket: %s \n' % (ticket_prepid)
full_message += '%smccms?prepid=%s \n\n' % (base_url, ticket_prepid)
yield '.'
full_message += '\n'
subject = (subject_part1 % (num_tickets)) + ('ticket' if num_tickets == 1 else 'tickets') + subject_part2
notification(
subject,
full_message,
[emails_prepids[author_email]],
group=notification.REMINDERS,
action_objects=ticket_prepids,
object_type='mccms')
mail_communicator.sendMail([author_email], subject, full_message)
yield '\nEmail sent to %s\n' % (author_email)
def request_join(self, req):
with locker.lock(req.get_attribute('prepid')):
chain = req.get_attribute("member_of_chain")
chain.append(self.get_attribute('_id'))
req.set_attribute("member_of_chain", chain)
loc = locator()
req.notify("Request {0} joined chain".format(req.get_attribute('prepid')),
"Request {0} has successfully joined chain {1}\n\n{2}\n".format(req.get_attribute('prepid'),
self.get_attribute('_id'),
"/".join([loc.baseurl(), "requests?prepid={0}".format(req.get_attribute('prepid'))])))
req.update_history({'action': 'join chain', 'step': self.get_attribute('_id')})
if not req.get_attribute('prepid') in self.get_attribute('chain'):
chain = self.get_attribute('chain')
chain.append(req.get_attribute('prepid'))
self.set_attribute("chain", chain)
self.update_history({'action': 'add request', 'step': req.get_attribute('prepid')})
def __init__(self):
self.setup_directories()
self.setup_logger()
self.get_submmited_prepids()
self.batch_retry_timeout = settings.get_value('batch_retry_timeout')
self.check_term_runlimit = settings.get_value('check_term_runlimit')
try:
self.ssh_exec = ssh_executor()
except Exception as e:
self.ssh_exec = None
self.logger.error(str(e) + 'traceback %s ' % traceback.format_exc())
return
if locator().isDev():
self.group = '/dev'
else:
self.group = '/prod'
def get(self, block_threshold=0):
"""
Send a reminder to the production managers for existing opened mccm documents
"""
mdb = database('mccms')
udb = database('users')
__query = mdb.construct_lucene_query({'status': 'new'})
mccms = mdb.full_text_search('search', __query, page=-1)
mccms = filter(lambda m: m['block'] <= block_threshold, mccms)
mccms = sorted(mccms, key=lambda m: m['block'])
if len(mccms) == 0:
return {"results": True, "message": "nothing to remind of at level %s, %s" % (block_threshold, mccms)}
l_type = locator()
com = communicator()
subject = 'Gentle reminder on %s tickets to be operated by you' % (len( mccms))
message = '''\
Dear Production Managers,
please find below the details of %s opened MccM tickets that need to be operated.
''' % (len(mccms))
mccm_prepids = []
for _mccm in mccms:
prepid = _mccm['prepid']
message += 'Ticket : %s (block %s)\n' % (prepid, _mccm['block'])
message += ' %smccms?prepid=%s \n\n' % (l_type.baseurl(), prepid)
mccm_prepids.append(prepid)
message += '\n'
to_who = [settings.get_value('service_account')]
to_who.extend(map(lambda u: u['email'], udb.query(query="role==production_manager", page_num=-1)))
notification(
subject,
message,
[],
group=notification.REMINDERS,
action_objects=mccm_prepids,
object_type='mccms',
target_role='production_manager')
com.sendMail(
to_who,
subject,
message)
return {"results": True, "message": map(lambda m: m['prepid'], mccms)}
def get(self, block_threshold=0):
"""
Send a reminder to the production managers for existing opened mccm documents
"""
mdb = database('mccms')
udb = database('users')
__query = mdb.construct_lucene_query({'status': 'new'})
mccms = mdb.full_text_search('search', __query, page=-1)
mccms = filter(lambda m: m['block'] <= block_threshold, mccms)
mccms = sorted(mccms, key=lambda m: m['block'])
if len(mccms) == 0:
return {"results": True, "message": "nothing to remind of at level %s, %s" % (block_threshold, mccms)}
l_type = locator()
com = communicator()
subject = 'Gentle reminder on %s tickets to be operated by you' % (len( mccms))
message = '''\
Dear Production Managers,
please find below the details of %s opened MccM tickets that need to be operated.
''' % (len(mccms))
mccm_prepids = []
for _mccm in mccms:
prepid = _mccm['prepid']
message += 'Ticket : %s (block %s)\n' % (prepid, _mccm['block'])
message += ' %smccms?prepid=%s \n\n' % (l_type.baseurl(), prepid)
mccm_prepids.append(prepid)
message += '\n'
to_who = [settings.get_value('service_account')]
to_who.extend(map(lambda u: u['email'], udb.query(query="role==production_manager", page_num=-1)))
notification(
subject,
message,
[],
group=notification.REMINDERS,
action_objects=mccm_prepids,
object_type='mccms',
target_role='production_manager')
com.sendMail(
to_who,
subject,
message)
return {"results": True, "message": map(lambda m: m['prepid'], mccms)}
def __init__(self, test_id, test_script):
self.script_for_test = test_script
self.test_id = test_id
self.test_err = os.path.abspath( self.script_for_test + '.err')
self.test_out = os.path.abspath( self.script_for_test + '.out')
locat = locator()
if locat.isDev():
self.group = '/dev'
else:
self.group = '/prod'
self.directory_for_test = os.path.dirname(self.script_for_test)
self.ssh_exec = ssh_executor(self.directory_for_test, self.test_id)
self.log_out = 'Not available'
self.log_err = 'Not available'
self.timeout = settings().get_value('batch_timeout')
def __init__(self, db_name='',url=None, cache=False):
host = os.environ['HOSTNAME']
if url is None:
url = locator().dbLocation()
if not db_name:
raise self.DatabaseNotFoundException(db_name)
self.db_name = db_name
self.cache = cache
if self.db_name in ['campaigns','chained_campaigns']:
## force cache for those.
self.cache=True
try:
self.db = Database(db_name, url=url)
except ValueError as ex:
raise self.DatabaseAccessError(db_name)
self.allowed_operators = ['<=', '<', '>=', '>', '==', '~=']
def __init__(self):
self.setup_directories()
self.setup_logger()
self.get_submmited_prepids()
self.batch_retry_timeout = settings.get_value('batch_retry_timeout')
self.check_term_runlimit = settings.get_value('check_term_runlimit')
try:
self.ssh_exec = ssh_executor()
except Exception as e:
self.ssh_exec = None
self.logger.error(
str(e) + 'traceback %s ' % traceback.format_exc())
return
if locator().isDev():
self.group = '/dev'
else:
self.group = '/prod'
def get(self, pwgs):
"""
Ask for the increase of the role of the current user to the given pwg
"""
# get who's there
user_p = user_pack()
udb = database(self.db_name)
mcm_u = user(udb.get(user_p.get_username()))
# get the requested pwgs
pwgs = pwgs.split(',')
# set the pwgs to the current user
current = mcm_u.get_attribute('pwg')
current = list(set(current + pwgs))
mcm_u.set_attribute('pwg', current)
mcm_u.update_history({'action': 'ask role', 'step': pwgs})
udb.update(mcm_u.json())
# get the production managers emails
__query = udb.construct_lucene_query({'role': 'production_manager'})
production_managers = udb.full_text_search('search', __query, page=-1)
# send a notification to prod manager + service
to_who = map(lambda u: u['email'], production_managers) + [
settings.get_value('service_account')
]
to_who.append(user_p.get_email())
com = communicator()
l_type = locator()
subject = 'Increase role for user %s' % mcm_u.get_attribute('fullname')
message = 'Please increase the role of the user %s to the next level.\n\n%susers?prepid=%s' % (
mcm_u.get_attribute('username'), l_type.baseurl(),
mcm_u.get_attribute('username'))
notification(subject,
message, [],
group=notification.USERS,
action_objects=[mcm_u.get_attribute('prepid')],
object_type='users',
target_role='production_manager')
com.sendMail(to_who, subject, message)
return {
"results":
True,
"message":
"user %s in for %s" % (mcm_u.get_attribute('username'), current)
}
def __init__(self, db_name='', url=None, cache_enabled=False):
host = os.environ['HOSTNAME']
if url is None:
url = locator().dbLocation()
if not db_name:
raise self.DatabaseNotFoundException(db_name)
self.db_name = db_name
self.cache_enabled = cache_enabled
if self.db_name in ['campaigns', 'chained_campaigns']:
## force cache for those.
self.cache_enabled = True
try:
self.db = Database(db_name, url=url)
except ValueError as ex:
raise self.DatabaseAccessError(db_name)
self.allowed_operators = ['<=', '<', '>=', '>', '==', '~=']
def GET(self, *args):
"""
Twiki display of mccm ticket for a given meeting date and /pwg optional
"""
date = args[0]
pwgs = None
if len(args) > 1:
pwgs = args[1].split(',')
mdb = database('mccms')
to_be_shown = [
'prepid', 'notes', 'deadline', 'requests', 'chains', 'repetitions'
]
l_type = locator()
mdocs = mdb.queries(['meeting==%s' % date])
if pwgs:
text = "---++ MccM Tickets for %s : %s \n" % (date,
', '.join(pwgs))
for pwg in pwgs:
mdocs_pwg = filter(lambda m: m['pwg'] == pwg, mdocs)
text += "---+++ Tickets for %s \n" % pwg
text += "[[%smccms?meeting=%s&pwg=%s][link to McM]]\n" % (
l_type.baseurl(), date, pwg)
for t in mdocs_pwg:
text += " * "
for item in to_be_shown:
if item in t:
text += "%s " % t[item]
text += '\n'
else:
text = "---++ MccM Tickets for %s \n<br>" % (date)
text += "[[%smccms?meeting=%s][link to McM]]\n" % (
l_type.baseurl(), date)
for t in mdocs:
text += " * "
for item in to_be_shown:
if item in t:
text += "%s " % t[item]
text += '\n'
return text
def GET(self, *args):
"""
Ask for the increase of the role of the current user to the given pwg
"""
if not args:
return dumps({"results": False, "Message": "not pwg provided"})
## get who's there
user_p = user_pack()
udb = database(self.db_name)
mcm_u = user(udb.get(user_p.get_username()))
## get the requested pwgs
pwgs = args[0].split(',')
#### set the pwgs to the current user
current = mcm_u.get_attribute('pwg')
current = list(set(current + pwgs))
mcm_u.set_attribute('pwg', current)
mcm_u.update_history({'action': 'ask role', 'step': args[0]})
udb.update(mcm_u.json())
## get the production managers emails
production_managers = udb.queries(['role==production_manager'])
### send a notification to prod manager + service
to_who = map(lambda u: u['email'], production_managers) + [
settings().get_value('service_account')
]
to_who.append(user_p.get_email())
com = communicator()
l_type = locator()
com.sendMail(
to_who,
'Increase role for user %s' % mcm_u.get_attribute('fullname'),
'Please increase the role of the user %s to the next level.\n\n%susers?prepid=%s'
% (mcm_u.get_attribute('username'), l_type.baseurl(),
mcm_u.get_attribute('username')))
return dumps({
"results":
True,
"message":
"user %s in for %s" % (mcm_u.get_attribute('username'), current)
})
def sendMail(self,
destination,
subject,
text,
sender=None,
reply_msg_ID=None):
if not isinstance(destination, list):
print "Cannot send email. destination should be a list of strings"
return
msg = MIMEMultipart()
#it could happen that message are send after forking, threading and there's no current user anymore
msg['From'] = sender if sender else '*****@*****.**'
msg['To'] = COMMASPACE.join(destination)
msg['Date'] = formatdate(localtime=True)
new_msg_ID = make_msgid()
msg['Message-ID'] = new_msg_ID
if reply_msg_ID != None:
msg['In-Reply-To'] = reply_msg_ID
msg['References'] = reply_msg_ID
## add a mark on the subjcet automatically
if locator().isDev():
msg['Subject'] = '[McM-dev] ' + subject
else:
msg['Subject'] = '[McM] ' + subject
## add a signature automatically
text += '\n'
text += 'McM Announcing service'
try:
msg.attach(MIMEText(text))
smtpObj = smtplib.SMTP()
smtpObj.connect()
smtpObj.sendmail(sender, destination, msg.as_string())
smtpObj.quit()
return new_msg_ID
except Exception as e:
print "Error: unable to send email", e.__class__
def GET(self, *args):
"""
Send a reminder to the production managers for existing opened mccm documents
"""
mdb = database('mccms')
mccms = mdb.queries(['status==new'])
udb = database('users')
block_threshold = 0
if len(args):
block_threshold = int(args[0])
mccms = filter( lambda m : m['block'] <= block_threshold, mccms)
mccms = sorted( mccms, key = lambda m : m['block'])
if len(mccms)==0:
return dumps({"results": True,"message": "nothing to remind of at level %s, %s"% (block_threshold, mccms)})
l_type = locator()
com = communicator()
subject = 'Gentle reminder on %s tickets to be operated by you' % ( len( mccms))
message = '''\
Dear Production Managers,
please find below the details of %s opened MccM tickets that need to be operated.
''' % (len(mccms))
for mccm in mccms:
message += 'Ticket : %s (block %s)\n'%( mccm['prepid'], mccm['block'] )
message += ' %smccms?prepid=%s \n\n' % (l_type.baseurl(), mccm['prepid'])
message += '\n'
to_who = [settings().get_value('service_account')]
to_who.extend( map( lambda u : u['email'], udb.query(query="role==production_manager", page_num=-1)))
com.sendMail(to_who,
subject,
message)
return dumps({"results" : True, "message" : map( lambda m : m['prepid'], mccms)})
def __init__(self, db_name='', url=None, cache=False):
host = os.environ['HOSTNAME']
if url is None:
url = locator().dbLocation()
#self.logger.log('I chose the url %s'%(url))
if not db_name:
raise self.DatabaseNotFoundException(db_name)
self.db_name = db_name
self.cache = cache
if self.db_name in ['campaigns', 'chained_campaigns']:
## force cache for those.
self.cache = True
try:
self.db = Database(db_name, url=url)
# self.db = Database(db_name, url='http://preptest.cern.ch:5984/')
# self.db = Database(db_name) # for using private DB @localhost:5984
except ValueError as ex:
raise self.DatabaseAccessError(db_name)
self.allowed_operators = ['<=', '<', '>=', '>', '==', '~=']
def __init__(self, db_name='',url=None, cache=False):
host = os.environ['HOSTNAME']
if url == None:
url =locator().dbLocation()
#self.logger.log('I chose the url %s'%(url))
if not db_name:
raise self.DatabaseNotFoundException(db_name)
self.db_name = db_name
self.cache = cache
if self.db_name in ['campaigns','chained_campaigns']:
## force cache for those.
self.cache=True
try:
self.db = Database(db_name, url=url)
# self.db = Database(db_name, url='http://preptest.cern.ch:5984/')
# self.db = Database(db_name) # for using private DB @localhost:5984
except ValueError as ex:
raise self.DatabaseAccessError(db_name)
self.allowed_operators = ['<=', '<', '>=', '>', '==', '~=']
def __init__(self, test_id, test_script, timeout=None):
self.script_for_test = test_script
self.test_id = test_id
self.test_err = os.path.abspath( self.script_for_test + '.err')
self.test_out = os.path.abspath( self.script_for_test + '.out')
locat = locator()
if locat.isDev():
self.group = '/dev'
else:
self.group = '/prod'
self.directory_for_test = os.path.dirname(self.script_for_test)
self.ssh_exec = ssh_executor(self.directory_for_test, self.test_id)
self.log_out = 'Not available'
self.log_err = 'Not available'
self.timeout = int(settings().get_value('batch_timeout'))
self.queue = '8nh'
if timeout:
self.timeout = int(timeout / 60.)
if (self.timeout / 3600. ) > 8.:
self.queue = '1nd' ## fall back to the one day queue at worse
def GET(self, *args):
"""
Twiki display of mccm ticket for a given meeting date and /pwg optional
"""
date = args[0]
pwgs=None
if len(args)>1:
pwgs=args[1].split(',')
mdb = database('mccms')
to_be_shown= ['prepid','notes','deadline','requests','chains','repetitions']
l_type=locator()
mdocs= mdb.queries(['meeting==%s'% date])
if pwgs:
text="---++ MccM Tickets for %s : %s \n"%( date, ', '.join(pwgs) )
for pwg in pwgs:
mdocs_pwg = filter ( lambda m : m['pwg']==pwg, mdocs)
text+="---+++ Tickets for %s \n" %pwg
text+="[[%smccms?meeting=%s&pwg=%s][link to McM]]\n"%( l_type.baseurl(), date, pwg)
for t in mdocs_pwg:
text+=" * "
for item in to_be_shown:
if item in t:
text+="%s "% t[item]
text+='\n'
else:
text="---++ MccM Tickets for %s \n<br>"%( date )
text+="[[%smccms?meeting=%s][link to McM]]\n"%( l_type.baseurl(), date )
for t in mdocs:
text+=" * "
for item in to_be_shown:
if item in t:
text+="%s "% t[item]
text+='\n'
return text
def GET(self, *args):
"""
Ask for the increase of the role of the current user to the given pwg
"""
if not args:
return dumps({"results" : False, "Message" : "not pwg provided"})
## get who's there
user_p = user_pack()
udb = database(self.db_name)
mcm_u = user( udb.get( user_p.get_username()))
## get the requested pwgs
pwgs = args[0].split(',')
#### set the pwgs to the current user
current = mcm_u.get_attribute('pwg')
current = list(set(current+pwgs))
mcm_u.set_attribute('pwg', current)
mcm_u.update_history({'action':'ask role','step' : args[0]})
udb.update(mcm_u.json())
## get the production managers emails
production_managers = udb.queries(['role==production_manager'])
### send a notification to prod manager + service
to_who = map(lambda u: u['email'], production_managers) + [settings().get_value('service_account')]
to_who.append( user_p.get_email() )
com = communicator()
l_type = locator()
com.sendMail( to_who,
'Increase role for user %s' % mcm_u.get_attribute('fullname'),
'Please increase the role of the user %s to the next level.\n\n%susers?prepid=%s' % ( mcm_u.get_attribute('username'),
l_type.baseurl(),
mcm_u.get_attribute('username')
))
return dumps({"results" : True, "message" : "user %s in for %s" %( mcm_u.get_attribute('username'), current)})
def build_location(sub_directory):
l_type = locator()
directory = l_type.workLocation()
return os.path.abspath(directory) + '/' + sub_directory + '/'
def GET(self, *args):
"""
Goes through invalidation documents and display (/) and announce them to data ops (/announce)
"""
announce = False
clear = False
if len(args) != 0:
if args[0] == 'announce':
announce = True
if args[0] == 'clear':
clear = True
idb = database('invalidations')
r_to_be_rejected = map(invalidation, idb.queries(['status==new', 'type==request']))
ds_to_be_invalidated = map(invalidation, idb.queries(['status==new', 'type==dataset']))
ds_to_be_invalidated = filter(lambda ds : not 'None-' in ds.get_attribute('object'), ds_to_be_invalidated)
l_type = locator()
def add_prepid(invalid, html):
if 'prepid' in invalid.json():
html += 'for request <a href=%srequests?prepid=%s> %s </a>' % (
l_type.baseurl(), invalid.get_attribute('prepid'), invalid.get_attribute('prepid'))
def print_invalidations(invalids):
a_text=''
for invalid in invalids:
a_text += ' %s\n' % (invalid.get_attribute('object'))
return a_text
html = '<html><body>\n'
html += 'Requests to be aborted/rejected <br>\n'
html += '<ul>\n'
for r in r_to_be_rejected:
html += '<li> <a href=%sreqmgr/view/details/%s> %s </a>' % (l_type.cmsweburl(), r.get_attribute('object'), r.get_attribute('object') )
add_prepid(r, html)
html += '</li>\n'
html += '</ul>\n'
html += 'Datasets to be invalidated <br>\n'
html += '<ul>\n'
for ds in ds_to_be_invalidated:
html += '<li> %s ' % ( ds.get_attribute('object') )
add_prepid(ds, html)
html += '</li>\n'
html += '</ul>\n'
html += '<a href=%srestapi/invalidations/inspect/clear> Clear invalidation withouth announcing</a><br>\n'%( l_type.baseurl() )
html += '<a href=%srestapi/invalidations/inspect/announce> Announce invalidations</a><br>\n'%( l_type.baseurl() )
html += '<a href=%srestapi/invalidations/inspect> Back</a><br>\n'%( l_type.baseurl() )
html += '</html></body>\n'
if announce and (len(ds_to_be_invalidated)!=0 or len(r_to_be_rejected)!=0):
text = 'Dear Data Operation Team,\n\n'
if len(r_to_be_rejected)!=0:
text += 'please reject or abort the following requests:\n'
text += print_invalidations(r_to_be_rejected)
if len(ds_to_be_invalidated)!=0:
text += '\nPlease invalidate the following datasets:\n'
text += print_invalidations(ds_to_be_invalidated)
text += '\nas a consequence of requests being reset.\n'
com = communicator()
to_who = [settings().get_value('service_account')]
if l_type.isDev():
to_who.append( settings().get_value('hypernews_test'))
else:
to_who.append( settings().get_value('dataops_announce' ))
try:
elem = (r_to_be_rejected + ds_to_be_invalidated)[0]
sender = elem.current_user_email
except IndexError:
sender = None
com.sendMail(to_who,
'Request and Datasets to be Invalidated',
text,
sender)
for to_announce in itertools.chain(r_to_be_rejected, ds_to_be_invalidated):
to_announce.set_announced()
idb.update(to_announce.json())
if clear and (len(ds_to_be_invalidated)!=0 or len(r_to_be_rejected)!=0):
for to_announce in itertools.chain(r_to_be_rejected, ds_to_be_invalidated):
to_announce.set_announced()
idb.update(to_announce.json())
return html
def flow_to_next_step(self, input_dataset='', block_black_list=None, block_white_list=None, check_stats=True, reserve=False):
if not block_white_list: block_white_list = []
if not block_black_list: block_black_list = []
self.logger.log('Flowing chained_request %s to next step...' % (self.get_attribute('_id')))
if not self.get_attribute('chain'):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'chained_request %s has got no root' % (
self.get_attribute('_id')))
# check on the approval of the chained request before all
## let it be flowing regardless
#if self.get_attribute('approval') == 'none':
# raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
# 'The approval of the chained request is none, and therefore flow cannot happen')
#this operation requires to access all sorts of objects
rdb = database('requests')
cdb = database('campaigns')
ccdb = database('chained_campaigns')
crdb = database('chained_requests')
fdb = database('flows')
adb = database('actions')
l_type=locator()
current_step = len(self.get_attribute('chain'))-1 if reserve else self.get_attribute('step')
current_id = self.get_attribute('chain')[current_step]
next_step = current_step + 1
if not rdb.document_exists(current_id):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the request %s does not exist' % current_id)
current_request = request(rdb.get(current_id))
current_campaign = campaign(cdb.get(current_request.get_attribute('member_of_campaign')))
if not ccdb.document_exists(self.get_attribute('member_of_campaign')):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the chain request %s is member of %s that does not exist' % (
self.get_attribute('_id'),
self.get_attribute('member_of_campaign')))
mcm_cc = ccdb.get(self.get_attribute('member_of_campaign'))
if next_step >= len(mcm_cc['campaigns']):
if reserve: return False
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'chained_campaign %s does not allow any further flowing.' % (
self.get_attribute('member_of_campaign')))
if not reserve:
## is the current request in the proper approval
allowed_request_approvals = ['submit']
if current_request.get_attribute('approval') not in allowed_request_approvals:
raise self.NotApprovedException(current_request.get_attribute('prepid'),
current_request.get_attribute('approval'), allowed_request_approvals)
## is the current request in the proper status
allowed_request_statuses = ['submitted', 'done']
if current_request.get_attribute('status') not in allowed_request_statuses:
raise self.NotInProperStateException(current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
original_action_id = self.get_attribute('chain')[0]
original_action_item = self.retrieve_original_action_item(adb, original_action_id)
## what is the campaign to go to next and with which flow
(next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step]
if not fdb.document_exists(flow_name):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow %s does not exist' % flow_name )
mcm_f = flow(fdb.get(flow_name))
if not 'sequences' in mcm_f.get_attribute('request_parameters'):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow %s does not contain sequences information.' % (
flow_name))
if not cdb.document_exists(next_campaign_id):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The next campaign %s does not exist' % next_campaign_id)
next_campaign = campaign(cdb.get(next_campaign_id))
if len(next_campaign.get_attribute('sequences')) != len(mcm_f.get_attribute('request_parameters')['sequences']):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the sequences changes in flow %s are not consistent with the next campaign %s' % (
flow_name, next_campaign_id))
if next_campaign.get_attribute('energy') != current_campaign.get_attribute('energy'):
raise self.EnergyInconsistentException(next_campaign.get_attribute('prepid'))
if next_campaign.get_attribute('type') == 'MCReproc' and (not 'time_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not time per event is specified')
if next_campaign.get_attribute('type') == 'MCReproc' and (not 'size_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not size per event is specified')
## check that it is allowed to flow
allowed_flow_approvals = ['flow', 'submit']
###### cascade of checks
"""
if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow (%s) is not in proper approval state (%s)'%(
mcm_f.get_attribute('prepid'),
mcm_f.get_attribute('approval')))
if not reserve and not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The chained request (%s) is not in the proper approval state (%s)'% (
self.get_attribute('_id'),
self.get_attribute('approval')))
"""
if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals:
if not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing' % (
mcm_f.get_attribute('approval'),
self.get_attribute('approval')))
if next_campaign.get_attribute('status') == 'stopped':
raise self.CampaignStoppedException(next_campaign_id)
#what is going to be the required number of events for the next request
#update the stats to its best
if not reserve:
current_request.get_stats()
next_total_events=current_request.get_attribute('completed_events')
notify_on_fail=True ## to be tuned according to the specific cases
if current_request.get_attribute('completed_events') <= 0:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The number of events completed is negative or null')
else:
allowed_request_statuses = ['done']
## determine if this is a root -> non-root transition to potentially apply staged number
at_a_transition=(current_campaign.get_attribute('root') != 1 and next_campaign.get_attribute('root') == 1)
if ('staged' in original_action_item or 'threshold' in original_action_item) and at_a_transition:
allowed_request_statuses.append('submitted')
##check status
if not current_request.get_attribute('status') in allowed_request_statuses:
raise self.NotInProperStateException(current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
##special check at transition that the statistics is good enough
if at_a_transition:
# at a root -> non-root transition only does the staged/threshold functions !
if 'staged' in original_action_item:
next_total_events = int(original_action_item['staged'])
if 'threshold' in original_action_item:
next_total_events = int(current_request.get_attribute('total_events') * float(original_action_item['threshold'] / 100.))
completed_events_to_pass = next_total_events
else:
## get the original expected events and allow a margin of 5% less statistics
completed_events_to_pass = int(current_request.get_attribute('total_events') * 0.95)
if check_stats and (current_request.get_attribute('completed_events') < completed_events_to_pass):
if notify_on_fail:
current_request.notify('Flowing for %s: not enough statistics'%( current_request.get_attribute('prepid')),
'For this request, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s \n\n %srequests?prepid=%s'%( current_request.get_attribute('completed_events'),
completed_events_to_pass,
l_type.baseurl(),
current_request.get_attribute('prepid')))
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The number of events completed (%s) is not enough for the requirement (%s)'%(current_request.get_attribute('completed_events'), completed_events_to_pass))
## select what is to happened : [create, patch, use]
next_id = None
approach = None
next_request = None
if next_step != len(self.get_attribute('chain')):
#not at the end
next_id = self.get_attribute('chain')[next_step]
if not rdb.document_exists(next_id):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The next request (%s) according to the step (%s) does not exist' % (
next_id, next_step))
next_request = request(rdb.get(next_id))
if next_request.get_attribute('status') == 'new':
#most likely a rewind + resub
approach = 'patch'
else:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % (
next_id, next_step, next_request.get_attribute('status')))
else:
## look in *other* chained campaigns whether you can suck in an existing request
## look up all chained requests that start from the same root request
## remove <pwg>-chain_ and the -serial number, replacing _ with a .
toMatch = '.'.join(self.get_attribute('prepid').split('_')[1:][0:next_step + 1]).split('-')[0]
## make sure they get ordered by prepid
related_crs = sorted(crdb.queries(['root_request==%s' % original_action_id]), key=lambda cr : cr['prepid'])
vetoed_last=[]
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid']==self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute('member_of_campaign'):
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) > next_step:
## one existing request in the very same chained campaign has already something used, make sure it is not going to be used
vetoed_last.append( mcm_cr.get_attribute('chain')[next_step])
continue
else:
continue
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid']==self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute('member_of_campaign'):
continue
truncated = '.'.join(existing_cr['prepid'].split('_')[1:][0:next_step + 1]).split('-')[0]
self.logger.error('to match : %s , this one %s' % ( toMatch, truncated ))
if truncated == toMatch:
#we found a chained request that starts with all same steps
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) <= next_step:
#found one, but it has not enough content either
continue
if mcm_cr.get_attribute('chain')[next_step] in vetoed_last:
continue
next_id = mcm_cr.get_attribute('chain')[next_step]
break
if next_id:
approach = 'use'
else:
approach = 'create'
if approach == 'create':
from rest_api.RequestPrepId import RequestPrepId
next_id = RequestPrepId().next_prepid(current_request.get_attribute('pwg'), next_campaign_id)
next_request = request(rdb.get(next_id))
request.transfer( current_request, next_request)
self.request_join(next_request)
elif approach == 'use':
## there exists a request in another chained campaign that can be re-used here.
# take this one. advance and go on
next_request = request(rdb.get(next_id))
if not reserve:
self.set_attribute('step', next_step)
self.set_attribute('last_status', next_request.get_attribute('status'))
self.update_history({'action': 'flow', 'step': str(next_step)})
self.set_attribute('status', 'processing')
if not self.get_attribute("prepid") in next_request.get_attribute("member_of_chain"):
## register the chain to the next request
self.request_join(next_request)
saved = rdb.update(next_request.json())
if not saved:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Unable to save %s with updated member_of_chains' % next_id)
return True
elif approach == 'patch':
## there exists already a request in the chain (step!=last) and it is usable for the next stage
next_request = request( next_campaign.add_request( rdb.get(next_id)))
### shouldn't this be added ?
#transfer( current_request, next_request)
else:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Unrecognized approach %s' % approach )
#current_request -> next_request
#current_campaign -> next_campaign
##determine whether we have an input dataset for the next request
if len(current_request.get_attribute('reqmgr_name')):
last_wma = current_request.get_attribute('reqmgr_name')[-1]
if 'content' in last_wma and 'pdmv_dataset_name' in last_wma['content']:
input_dataset = last_wma['content']['pdmv_dataset_name']
else:
statsDB = database('stats', url='http://cms-pdmv-stats.cern.ch:5984/')
if statsDB.document_exists(last_wma['name']):
latestStatus = statsDB.get(last_wma['name'])
input_dataset = latestStatus['pdmv_dataset_name']
if input_dataset:
next_request.set_attribute('input_filename', input_dataset)
## set blocks restriction if any
if block_black_list:
next_request.set_attribute('block_black_list', block_black_list)
if block_white_list:
next_request.set_attribute('block_white_list', block_white_list)
## register the flow to the request
next_request.set_attribute('flown_with', flow_name)
##assemble the campaign+flow => request
request.put_together(next_campaign, mcm_f, next_request)
if not reserve:
#already taking stage and threshold into account
next_request.set_attribute('total_events', next_total_events)
next_request.update_history({'action': 'flow', 'step': self.get_attribute('prepid')})
request_saved = rdb.save(next_request.json())
if not request_saved:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Could not save the new request %s' % (
next_request.get_attribute('prepid')))
## inspect priority
self.set_priority(original_action_item['block_number'])
if not reserve:
# sync last status
self.set_attribute('last_status', next_request.get_attribute('status'))
# we can only be processing at this point
self.set_attribute('status', 'processing')
# set to next step
self.set_attribute('step', next_step)
if not reserve:
notification_subject = 'Flow for request %s in %s' % (current_request.get_attribute('prepid'), next_campaign_id)
notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % (
current_request.get_attribute('prepid'),
self.get_attribute('member_of_campaign'),
next_campaign_id,
flow_name,
next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_request.get_attribute('input_filename'),
l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(),
next_request.get_attribute('prepid')
)
current_request.notify(notification_subject, notification_text)
else:
notification_subject = 'Reservation of request {0}'.format(next_request.get_attribute('prepid'))
notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format(
next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_campaign_id,
l_type.baseurl(), current_request.get_attribute('prepid'),
l_type.baseurl(), next_request.get_attribute('prepid'),
)
next_request.notify(notification_subject, notification_text)
return True
def get_command(self, mcm_r, batchNumber, to_execute=False):
command = ''
##JR in order to inject into the testbed instead of the production machine
l_type = locator()
if to_execute:
# set path to proxy certificate
command += 'cd %s\n' % ( l_type.workLocation() )
command += 'export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOSTNAME/voms_proxy.cert\n'
command += mcm_r.make_release()
#command += 'eval `scram runtime -sh`\n'
command += 'source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh\n'
wmagent_type = mcm_r.get_wmagent_type()
command += 'export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}\n'
command += 'wmcontrol.py --release %s' % (mcm_r.get_attribute('cmssw_release'))
command += ' --arch %s' % (mcm_r.get_scram_arch())
command += ' --conditions %s' % (mcm_r.get_attribute('sequences')[0]['conditions'])
command += ' --version %s' % (mcm_r.get_attribute('version'))
if mcm_r.get_attribute('priority') >= 1:
command += ' --priority %s' % (mcm_r.get_attribute("priority"))
command += ' --time-event %s' % (mcm_r.get_attribute('time_event'))
command += ' --size-event %s' % (mcm_r.get_attribute('size_event'))
command += ' --memory %s' % (mcm_r.get_attribute('memory'))
##that type has disappeared
if wmagent_type == 'LHEStepZero':
command += ' --request-type MonteCarlo'
else:
command += ' --request-type %s' % wmagent_type
config_id_from_hashkey = []
## check on the presence of docId ?...
if len(mcm_r.get_attribute('config_id')):
command += ' --step1-docID %s' % ( mcm_r.get_attribute('config_id')[0])
else:
## get the config ID from hash instead of cfg.py
hash_ids = database('configs')
hash_id = mcm_r.configuration_identifier(0)
if hash_ids.document_exists(hash_id):
hash_doc = hash_ids.get(hash_id)
config_cache_id = hash_doc['docid']
command += ' --step1-docID %s' % config_cache_id
config_id_from_hashkey = [config_cache_id]
else:
command += ' --step1-cfg %s_1_cfg.py' % (mcm_r.get_attribute('prepid'))
command += ' --request-id %s' % (mcm_r.get_attribute('prepid'))
if l_type.isDev():
command += ' --wmtest '
command += ' --user pdmvserv '
command += ' --group ppd '
command += ' --batch %s' % batchNumber
processString = mcm_r.get_attribute('process_string')
processingString = mcm_r.get_processing_string(0)
max_forward_eff = mcm_r.get_forward_efficiency()
events_per_lumi = settings().get_value('events_per_lumi')
if wmagent_type == 'MonteCarlo':
# calculate eff dev
command += ' --filter-eff %s' % ( mcm_r.get_efficiency() )
command += ' --events-per-lumi %s' % ( events_per_lumi / max_forward_eff )
command += ' --number-events %s' % (mcm_r.get_attribute('total_events'))
command += ' --primary-dataset %s' % (mcm_r.get_attribute('dataset_name'))
elif wmagent_type == 'MonteCarloFromGEN':
# calculate eff dev
command += ' --filter-eff %s' % ( mcm_r.get_efficiency() )
command += ' --input-ds %s' % (mcm_r.get_attribute('input_dataset'))
command += ' --primary-dataset %s' % (mcm_r.get_attribute('dataset_name'))
if mcm_r.get_attribute('block_white_list'):
command += ' --blocks "' + ','.join(mcm_r.get_attribute('block_white_list')) + '"'
if mcm_r.get_attribute('block_black_list'):
command += ' --blocks_black "' + ','.join(mcm_r.get_attribute('block_black_list')) + '"'
command += ' --number-events %s' % (mcm_r.get_attribute('total_events'))
elif wmagent_type == 'LHEStepZero':
command += ' --number-events %s' % (mcm_r.get_attribute('total_events'))
command += ' --events-per-lumi %s' % ( events_per_lumi / max_forward_eff )
command += ' --primary-dataset %s' % (mcm_r.get_attribute('dataset_name'))
command += ' --filter-eff %s' % ( mcm_r.get_efficiency() )
if mcm_r.get_attribute('mcdb_id') <= 0:
numberOfEventsPerJob = mcm_r.numberOfEventsPerJob()
if not numberOfEventsPerJob:
raise ValueError('Number of events per job could not be retrieved')
command += ' --events-per-job %s' % numberOfEventsPerJob
else:
command += ' --lhe '
if not processString:
processString = ''
elif wmagent_type == 'ReDigi':
command += ' --input-ds %s' % (mcm_r.get_attribute('input_dataset'))
command += ' --primary-dataset %s' % (mcm_r.get_attribute('dataset_name'))
## if PU dataset name is defined : add it
if mcm_r.get_attribute('pileup_dataset_name') and mcm_r.get_attribute('pileup_dataset_name').strip():
command += ' --pileup-ds ' + mcm_r.get_attribute('pileup_dataset_name')
## provide the total number of events requested: by default it is the amount in the input dataset.
# and wmcontrol / wma should understand that we want partial statistics if that number is lower than expected
command += ' --number-events %s' % (mcm_r.get_attribute('total_events'))
# temp ev cont holder
eventcontentlist = mcm_r.get_first_output()
keeps = mcm_r.get_attribute('keep_output')
if not keeps[-1]:
raise ValueError('Is not set to save the output of last task')
for (i, content) in enumerate(eventcontentlist):
if i < 2: #trick to NOT add for step3 and more:
if keeps[i]:
command += ' --keep-step' + str(i + 1) + ' True'
if i > 0:
processingString = mcm_r.get_processing_string(i)
if len(mcm_r.get_attribute('config_id')):
command += ' --step%d-docID %s' % (i + 1, mcm_r.get_attribute('config_id')[i])
else:
hash_id = mcm_r.configuration_identifier(i)
if hash_ids.document_exists(hash_id):
hash_doc = hash_ids.get(hash_id)
config_cache_id = hash_doc['docid']
command += ' --step%d-docID %s' % (i + 1, config_cache_id)
config_id_from_hashkey.append(config_cache_id)
else:
command += ' --step%d-cfg %s_%d_cfg.py' % ( i + 1, mcm_r.get_attribute('prepid'), i + 1)
# set the output of
if i < len(eventcontentlist) - 1:
command += ' --step' + str(i + 1) + '-output ' + content
if mcm_r.get_attribute('block_white_list'):
command += ' --blocks "' + ','.join(mcm_r.get_attribute('block_white_list')) + '"'
if mcm_r.get_attribute('block_black_list'):
command += ' --blocks_black "' + ','.join(mcm_r.get_attribute('block_black_list')) + '"'
if processString:
command += ' --process-string ' + processString
command += ' --processing-string ' + processingString
command += '|| exit $? ;'
command += '\n'
if len(config_id_from_hashkey):
mcm_r.set_attribute('config_id', config_id_from_hashkey)
return command
def announce(self,notes="",user=""):
if self.get_attribute('status')!='new':
return False
if len(self.get_attribute('requests'))==0:
return False
current_notes=self.get_attribute('notes')
if current_notes:
current_notes+='\n'
if notes:
current_notes+=notes
self.set_attribute('notes',current_notes)
total_events=0
content = self.get_attribute('requests')
total_requests=len(content)
rdb =database('requests')
## prepare the announcing message
(campaign,batchNumber)=self.get_attribute('prepid').split('_')[-1].split('-')
subject=self.get_subject()
message=""
message+="Dear Data Operation Team,\n\n"
message+="may you please consider the following batch number %d of %s requests for the campaign %s:\n\n"%(int(batchNumber),total_requests, campaign)
for r in content:
##loose binding of the prepid to the request name, might change later on
if 'pdmv_prep_id' in r['content']:
pid=r['content']['pdmv_prep_id']
else:
pid=r['name'].split('_')[1]
mcm_r = rdb.get(pid)
total_events+=mcm_r['total_events']
message+=" * %s (%s) -> %s\n"%(pid, mcm_r['dataset_name'], r['name'])
message+="\n"
message+="For a total of %s events\n\n"%( re.sub("(\d)(?=(\d{3})+(?!\d))", r"\1,", "%d" % total_events ))
if self.get_attribute('extension'):
message += "This batch is for an extension : {0}\n".format(self.get_attribute('extension'))
if self.get_attribute('version'):
message += "This batch is a resubmission : v{0}\n".format(self.get_attribute('version') + 1)
message+="Link to the batch:\n"
l_type = locator()
message+='%s/batches?prepid=%s \n\n'%(l_type.baseurl(), self.get_attribute('prepid'))
if current_notes:
message+="Additional comments for this batch:\n"+current_notes+'\n'
if self.get_attribute('process_string'):
message+='Please use "%s" in the dataset name.\n' % self.get_attribute('process_string')
self.logger.log('Message send for batch %s'%(self.get_attribute('prepid')))
self.get_current_user_role_level()
to_who = [settings().get_value('service_account')]
if l_type.isDev():
to_who.append( settings().get_value('hypernews_test'))
else:
to_who.append( settings().get_value('dataops_announce' ))
#sender=None
#if self.current_user_level != 3:
# auth = authenticator()
# sender = auth.get_random_product_manager_email()
#current_message_id = self.get_attribute('message_id')
returned_id = self.notify(subject,
message,
who=to_who)#,
#sender=sender)
self.set_attribute('message_id', returned_id)
self.reload()
## toggle the status
### only when we are sure it functions self.set_status()
self.set_status()
return True
def __init__(self):
self.db_name = "invalidations"
self.com = communicator()
self.l_type = locator()
def announce(self,notes="",user=""):
if self.get_attribute('status')!='new':
return False
if len(self.get_attribute('requests'))==0:
return False
current_notes=self.get_attribute('notes')
if current_notes:
current_notes+='\n'
if notes:
current_notes+=notes
self.set_attribute('notes',current_notes)
total_events=0
content = self.get_attribute('requests')
total_requests=len(content)
rdb =database('requests')
## prepare the announcing message
(campaign,batchNumber)=self.get_attribute('prepid').split('_')[-1].split('-')
subject=self.get_subject()
request_messages={}
for r in content:
##loose binding of the prepid to the request name, might change later on
if 'pdmv_prep_id' in r['content']:
pid=r['content']['pdmv_prep_id']
else:
pid=r['name'].split('_')[1]
mcm_r = rdb.get(pid)
total_events+=mcm_r['total_events']
c = mcm_r['member_of_campaign']
if not c in request_messages: request_messages[c]=""
request_messages[c]+=" * %s (%s) -> %s\n"%(pid, mcm_r['dataset_name'], r['name'])
campaigns = sorted(request_messages.keys())
message=""
message+="Dear Data Operation Team,\n\n"
message+="may you please consider the following batch number %d of %s requests for the campaign%s %s:\n\n"%(
int(batchNumber),total_requests,
"s" if len(campaigns)>1 else "",
','.join(campaigns))
for c in campaigns:
message+=request_messages[c]
message+="\n"
message+="For a total of %s events\n\n"%( re.sub("(\d)(?=(\d{3})+(?!\d))", r"\1,", "%d" % total_events ))
if self.get_attribute('extension'):
message += "This batch is for an extension : {0}\n".format(self.get_attribute('extension'))
if self.get_attribute('version'):
message += "This batch is a resubmission : v{0}\n".format(self.get_attribute('version') + 1)
message+="Link to the batch:\n"
l_type = locator()
message+='%s/batches?prepid=%s \n\n'%(l_type.baseurl(), self.get_attribute('prepid'))
if current_notes:
message+="Additional comments for this batch:\n"+current_notes+'\n'
self.logger.log('Message send for batch %s'%(self.get_attribute('prepid')))
self.get_current_user_role_level()
to_who = [settings().get_value('service_account')]
if l_type.isDev():
to_who.append( settings().get_value('hypernews_test'))
else:
to_who.append( settings().get_value('dataops_announce' ))
#sender=None
#if self.current_user_level != 3:
# auth = authenticator()
# sender = auth.get_random_product_manager_email()
#current_message_id = self.get_attribute('message_id')
returned_id = self.notify(subject,
message,
who=to_who)#,
#sender=sender)
self.set_attribute('message_id', returned_id)
self.reload()
## toggle the status
### only when we are sure it functions self.set_status()
self.set_status()
return True
def add_request(self, data=None):
if not data: data = {}
self.logger.log('Adding new request to chained_request %s' % self.get_attribute('_id'))
# import prep-id generator
try:
from rest_api.RequestPrepId import RequestPrepId
except ImportError as ex:
self.logger.error('Could not import prep-id generator class. Reason: %s' % ex, level='critical')
return {}
try:
req = request(json_input=data)
except Exception as ex:
self.logger.error('Could not build request object. Reason: %s' % ex)
return {}
#chain_specific = ['threshold', 'block_number', 'staged']
## JR remove from schema
##this was removed as part of cleaning things up
#if len(self.get_attribute('request_parameters')) > 0:
# changes = self.get_attribute('request_parameters')
# for key in changes:
# if key not in chain_specific:
# req.set_attribute(key, changes[key])
# get the chain and inherit
#req.set_attribute("generators", self.get_attribute("generators"))
#req.set_attribute("total_events", self.get_attribute("total_events")) ## this was taken earlier, with staged number in consideration
req.set_attribute("dataset_name", self.get_attribute("dataset_name"))
req.set_attribute("pwg", self.get_attribute("pwg"))
#JR removed from schema req.set_attribute("priority", self.get_attribute("priority") )
#JR clear the fragment in flowing: always
req.set_attribute('name_of_fragment', '')
req.set_attribute('cvs_tag', '')
req.set_attribute('fragment_tag', '')
req.set_attribute('fragment', '')
req.set_attribute('history', [])
req.set_attribute('reqmgr_name', [])
#JR
#clean the mcdbid in the flown request
#if req.get_attribute('mcdbid')>=0:
# req.set_attribute('mcdbid',0)
# get the new prepid and append it to the chain
prepid = RequestPrepId().generate_prepid(req.get_attribute("pwg"), req.get_attribute('member_of_campaign'))["prepid"]
chain = self.get_attribute("chain")
if not chain or chain is None:
chain = []
flag = False
for pid in chain:
#if req.get_attribute('member_of_campaign') in pid:
if pid.split('-')[1] == req.get_attribute('member_of_campaign'):
flag = True
break
if not flag:
chain.append(prepid)
self.set_attribute("chain", chain)
#self.logger.log('Adding %s to the chain %s'%(prepid,chain))
else:
raise self.CampaignAlreadyInChainException(req.get_attribute('member_of_campaign'))
req.set_attribute('_id', prepid)
req.set_attribute('prepid', prepid)
## JR: add what the request is member of N.B: that breaks down if a digi-reco request has to be member of two chains (R1,R4)
req.set_attribute('member_of_chain', [self.get_attribute('_id')])
## reset the status and approval chain
req.set_status(0)
req.approve(0)
### mode the approval of the new request to the approval of the chained request
if not req.is_root:
self.logger.log('The newly created request %s is not root, the chained request has approval %s' % (
req.get_attribute('prepid'),
self.get_attribute('approval')
))
#if self.get_attribute('approval') == 'approve':
#toggle the request approval to 'approved'?
if self.get_attribute('approval') == 'submit':
req.set_status(to_status='approved')
req.approve(to_approval='submit')
# update history
req.update_history({'action': 'join chain', 'step': self.get_attribute('_id')})
self.update_history({'action': 'add request', 'step': req.get_attribute('_id')})
loc = locator()
req.notify("Request {0} joined chain".format(req.get_attribute('prepid')), "Request {0} has successfuly joined chain {1}\n\tRequest: {2}".format(req.get_attribute('prepid'),
self.get_attribute('_id'),
"/".join([loc.baseurl(), "requests?prepid={0}".format(req.get_attribute('prepid'))])))
# set request approval status to new
#req.approve(0)
return req.json()
def index(self, **args):
db_name='requests'
query=''
query_list=[]
page=0
manual_keys=['db_name','query','page']
if 'db_name' in args:
db_name=args['db_name']
args.pop('db_name')
if 'query' in args:
query=args['query']
args.pop('query')
if 'page' in args:
page=args['page']
args.pop('page')
# retrieve the _design/object document
odb=database(db_name)
design = odb.get('_design/%s'%(db_name))
allowed_key_search = design['views'].keys()
vetoed_keys = []
for (view,f) in design['views'].items():
if 'for(' in f['map'] or 'for (' in f['map']:
vetoed_keys.append( view )
allowed_key_search.sort()
multiple_view=[]
####
## to switch on/off the view creation on the fly
simple_search=(not locator().isDev())
simple_search=False
####
for key in filter (lambda s : '-' not in s, allowed_key_search):
if key in args:
if key in vetoed_keys or simple_search:
query_list.append('%s==%s'%(key,args[key]))
else:
if args[key].isdigit():
multiple_view.append( (key, args[key]) )
else:
multiple_view.append( (key, '"'+args[key]+'"') )
args.pop(key)
if len(multiple_view)>1:
multiple_search = '-'.join( map( lambda p:p[0], multiple_view))
## faster query with multiple keys
if not multiple_search in allowed_key_search:
## try harder to find it
really_not_there=True
m_set = set( map( lambda p:p[0], multiple_view) )
for key in filter (lambda s : '-' in s, allowed_key_search):
## parse all composite view
if set(key.split('-')) == m_set:
#we found one that has the same search in absolute, just the order is different
# then re-order multiple_view so as to map to the existing view
new_multiple_view = []
for sv in key.split('-'):
new_multiple_view.append( filter( lambda e : e[0]==sv, multiple_view) [0] )
multiple_view = new_multiple_view
multiple_search = '-'.join( map( lambda p:p[0], multiple_view))
really_not_there=False
break
if really_not_there:
#tempatively add the view to the design
new_func = "function(doc){ emit([%s], doc._id);}"%( ','.join(map( lambda k: "doc.%s"%(k), map( lambda p:p[0], multiple_view) )))
design['views'] [ multiple_search ] = { "map" : new_func }
saved = odb.update( design )
##### NOTE ####
## the query that will follow will be super slow because the view needs to be re-build
m_query = '%s==[%s]'%(multiple_search,
','.join( map( lambda p:p[1], multiple_view))
)
query_list.append( m_query )
#query_list =[]
elif len(multiple_view)==1:
m_query = '%s==%s'%( multiple_view[0][0], multiple_view[0][1])
query_list.append( m_query )
#revert to simple query for one query only
if len(query_list)==1:
query=query_list[0]
query_list=[]
if len(args):
## check whether the key is actually a member of the object in db and put back the view in the odb design
return simplejson.dumps(args)
#return simplejson.dumps(design['views'].keys())
return self.search(db_name, query, page, query_list)
def __init__(self):
self.db_name = "invalidations"
self.com = communicator()
self.l_type = locator()
def internal_run(self):
if not self.lock.acquire(blocking=False):
return False
try:
req = request(self.request_db.get(self.prepid))
additional_config_ids = {}
cfgs_to_upload = {}
l_type = locator()
dev=''
wmtest=''
if l_type.isDev():
wmtest = '--wmtest'
if req.get_attribute('config_id'): # we already have configuration ids saved in our request
return True
for i in range(len(req.get_attribute('sequences'))):
hash_id = req.configuration_identifier(i)
if self.config_db.document_exists(hash_id): # cached in db
additional_config_ids[i] = self.config_db.get(hash_id)['docid']
else: # has to be setup and uploaded to config cache
cfgs_to_upload[i] = "{0}{1}_{2}_cfg.py".format(req.get_attribute('prepid'), dev, i+1)
if cfgs_to_upload:
with installer(self.prepid, care_on_existing=False) as directory_manager:
command = self.prepare_command([cfgs_to_upload[i] for i in sorted(cfgs_to_upload)], directory_manager.location(), req, wmtest)
_, stdout, stderr = self.ssh_executor.execute(command)
if not stdout and not stderr:
self.logger.error('SSH error for request {0}. Could not retrieve outputs.'.format(self.prepid))
self.logger.inject('SSH error for request {0}. Could not retrieve outputs.'.format(self.prepid), level='error', handler=self.prepid)
req.test_failure('SSH error for request {0}. Could not retrieve outputs.'.format(self.prepid), what='Configuration upload')
return False
output = stdout.read()
error = stderr.read()
if error and not output: # money on the table that it will break
self.logger.error('Error in wmupload: {0}'.format(error))
req.test_failure('Error in wmupload: {0}'.format(error), what='Configuration upload')
return False
cfgs_uploaded = [l for l in output.split("\n") if 'DocID:' in l]
if len(cfgs_to_upload) != len(cfgs_uploaded):
self.logger.error('Problem with uploading the configurations. To upload: {0}, received doc_ids: {1}\nOutput:\n{2}\nError:\n{3}'.format(cfgs_to_upload, cfgs_uploaded, output, error))
self.logger.inject('Problem with uploading the configurations. To upload: {0}, received doc_ids: {1}\nOutput:\n{2}\nError:\n{3}'.format(cfgs_to_upload, cfgs_uploaded, output, error), level='error', handler=self.prepid)
req.test_failure('Problem with uploading the configurations. To upload: {0}, received doc_ids: {1}\nOutput:\n{2}\nError:\n{3}'.format(cfgs_to_upload, cfgs_uploaded, output, error), what='Configuration upload')
return False
for i, line in zip(sorted(cfgs_to_upload), cfgs_uploaded): # filling the config ids for request and config database with uploaded configurations
docid = line.split()[-1]
additional_config_ids[i] = docid
saved = self.config_db.save({"_id": req.configuration_identifier(i),
"docid": docid,
"prepid": self.prepid,
"unique_string": req.unique_string(i)})
if not saved:
self.logger.inject('Could not save the configuration {0}'.format( req.configuration_identifier(i) ), level='warning', handler=self.prepid)
self.logger.inject("Full upload result: {0}".format(output), handler=self.prepid)
sorted_additional_config_ids = [additional_config_ids[i] for i in additional_config_ids]
self.logger.inject("New configs for request {0} : {1}".format(self.prepid, sorted_additional_config_ids), handler=self.prepid)
req.set_attribute('config_id', sorted_additional_config_ids)
self.request_db.save(req.json())
return True
finally:
self.lock.release()
def announce(self, notes="", user=""):
if self.get_attribute('status') != 'new':
return False
if len(self.get_attribute('requests')) == 0:
return False
current_notes = self.get_attribute('notes')
if current_notes:
current_notes += '\n'
if notes:
current_notes += notes
self.set_attribute('notes', current_notes)
total_events = 0
content = self.get_attribute('requests')
total_requests = len(content)
rdb = database('requests')
# prepare the announcing message
(campaign,
batchNumber) = self.get_attribute('prepid').split('_')[-1].split('-')
subject = self.get_subject()
request_messages = {}
for r in content:
# loose binding of the prepid to the request name, might change later on
if 'pdmv_prep_id' in r['content']:
pid = r['content']['pdmv_prep_id']
else:
pid = r['name'].split('_')[1]
mcm_r = rdb.get(pid)
total_events += mcm_r['total_events']
c = mcm_r['member_of_campaign']
if c not in request_messages: request_messages[c] = ""
request_messages[c] += " * %s (%s) -> %s\n" % (
pid, mcm_r['dataset_name'], r['name'])
campaigns = sorted(request_messages.keys())
message = ""
message += "Dear Data Operation Team,\n\n"
message += "may you please consider the following batch number %d of %s requests for the campaign%s %s:\n\n" % (
int(batchNumber), total_requests,
"s" if len(campaigns) > 1 else "", ','.join(campaigns))
for c in campaigns:
message += request_messages[c]
message += "\n"
message += "For a total of %s events\n\n" % (re.sub(
"(\d)(?=(\d{3})+(?!\d))", r"\1,", "%d" % total_events))
if self.get_attribute('extension'):
message += "This batch is for an extension : {0}\n".format(
self.get_attribute('extension'))
if self.get_attribute('version'):
message += "This batch is a resubmission : v{0}\n".format(
self.get_attribute('version') + 1)
message += "Link to the batch:\n"
l_type = locator()
message += '%s/batches?prepid=%s \n\n' % (l_type.baseurl(),
self.get_attribute('prepid'))
if current_notes:
message += "Additional comments for this batch:\n" + current_notes + '\n'
self.logger.info('Message send for batch %s' %
(self.get_attribute('prepid')))
self.get_current_user_role_level()
to_who = [settings.get_value('service_account')]
if l_type.isDev():
to_who.append(settings.get_value('hypernews_test'))
else:
to_who.append(settings.get_value('dataops_announce'))
notification(subject,
message, [],
group=notification.BATCHES,
target_role='production_manager',
action_objects=[self.get_attribute('prepid')],
object_type='batches',
base_object=self)
returned_id = self.notify(subject, message, who=to_who)
self.set_attribute('message_id', returned_id)
self.reload()
# toggle the status
# only when we are sure it functions self.set_status()
self.set_status()
return True
def GET(self, *args):
"""
Goes through invalidation documents and display (/) and announce them to data ops (/announce)
"""
announce = False
clear = False
if len(args) != 0:
if args[0] == 'announce':
announce = True
if args[0] == 'clear':
clear = True
idb = database('invalidations')
r_to_be_rejected = map(invalidation,
idb.queries(['status==new', 'type==request']))
ds_to_be_invalidated = map(
invalidation, idb.queries(['status==new', 'type==dataset']))
ds_to_be_invalidated = filter(
lambda ds: not 'None-' in ds.get_attribute('object'),
ds_to_be_invalidated)
l_type = locator()
def add_prepid(invalid, html):
if 'prepid' in invalid.json():
html += 'for request <a href=%srequests?prepid=%s> %s </a>' % (
l_type.baseurl(), invalid.get_attribute('prepid'),
invalid.get_attribute('prepid'))
def print_invalidations(invalids):
a_text = ''
for invalid in invalids:
a_text += ' %s\n' % (invalid.get_attribute('object'))
return a_text
html = '<html><body>\n'
html += 'Requests to be aborted/rejected <br>\n'
html += '<ul>\n'
for r in r_to_be_rejected:
html += '<li> <a href=%sreqmgr/view/details/%s> %s </a>' % (
l_type.cmsweburl(), r.get_attribute('object'),
r.get_attribute('object'))
add_prepid(r, html)
html += '</li>\n'
html += '</ul>\n'
html += 'Datasets to be invalidated <br>\n'
html += '<ul>\n'
for ds in ds_to_be_invalidated:
html += '<li> %s ' % (ds.get_attribute('object'))
add_prepid(ds, html)
html += '</li>\n'
html += '</ul>\n'
html += '<a href=%srestapi/invalidations/inspect/clear> Clear invalidation withouth announcing</a><br>\n' % (
l_type.baseurl())
html += '<a href=%srestapi/invalidations/inspect/announce> Announce invalidations</a><br>\n' % (
l_type.baseurl())
html += '<a href=%srestapi/invalidations/inspect> Back</a><br>\n' % (
l_type.baseurl())
html += '</html></body>\n'
if announce and (len(ds_to_be_invalidated) != 0
or len(r_to_be_rejected) != 0):
text = 'Dear Data Operation Team,\n\n'
if len(r_to_be_rejected) != 0:
text += 'please reject or abort the following requests:\n'
text += print_invalidations(r_to_be_rejected)
if len(ds_to_be_invalidated) != 0:
text += '\nPlease invalidate the following datasets:\n'
text += print_invalidations(ds_to_be_invalidated)
text += '\nas a consequence of requests being reset.\n'
com = communicator()
to_who = [settings().get_value('service_account')]
if l_type.isDev():
to_who.append(settings().get_value('hypernews_test'))
else:
to_who.append(settings().get_value('dataops_announce'))
try:
elem = (r_to_be_rejected + ds_to_be_invalidated)[0]
sender = elem.current_user_email
except IndexError:
sender = None
com.sendMail(to_who, 'Request and Datasets to be Invalidated',
text, sender)
for to_announce in itertools.chain(r_to_be_rejected,
ds_to_be_invalidated):
to_announce.set_announced()
idb.update(to_announce.json())
if clear and (len(ds_to_be_invalidated) != 0
or len(r_to_be_rejected) != 0):
for to_announce in itertools.chain(r_to_be_rejected,
ds_to_be_invalidated):
to_announce.set_announced()
idb.update(to_announce.json())
return html
def get_command(self, mcm_r, batchNumber, to_execute=False):
command = ''
##JR in order to inject into the testbed instead of the production machine
l_type = locator()
if to_execute:
# set path to proxy certificate
command += 'cd %s\n' % (l_type.workLocation())
command += 'export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOST/voms_proxy.cert\n'
command += mcm_r.make_release()
#command += 'eval `scram runtime -sh`\n'
command += 'source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh\n'
wmagent_type = mcm_r.get_wmagent_type()
command += 'export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}\n'
command += 'wmcontrol.py --release %s' % (
mcm_r.get_attribute('cmssw_release'))
command += ' --arch %s' % (mcm_r.get_scram_arch())
command += ' --conditions %s' % (
mcm_r.get_attribute('sequences')[0]['conditions'])
command += ' --version %s' % (mcm_r.get_attribute('version'))
if mcm_r.get_attribute('priority') >= 1:
command += ' --priority %s' % (mcm_r.get_attribute("priority"))
command += ' --time-event %s' % (mcm_r.get_attribute('time_event'))
command += ' --size-event %s' % (mcm_r.get_attribute('size_event'))
command += ' --memory %s' % (mcm_r.get_attribute('memory'))
##that type has disappeared
if wmagent_type == 'LHEStepZero':
command += ' --request-type MonteCarlo'
else:
command += ' --request-type %s' % wmagent_type
config_id_from_hashkey = []
## check on the presence of docId ?...
if len(mcm_r.get_attribute('config_id')):
command += ' --step1-docID %s' % (
mcm_r.get_attribute('config_id')[0])
else:
## get the config ID from hash instead of cfg.py
hash_ids = database('configs')
hash_id = mcm_r.configuration_identifier(0)
if hash_ids.document_exists(hash_id):
hash_doc = hash_ids.get(hash_id)
config_cache_id = hash_doc['docid']
command += ' --step1-docID %s' % config_cache_id
config_id_from_hashkey = [config_cache_id]
else:
command += ' --step1-cfg %s_1_cfg.py' % (
mcm_r.get_attribute('prepid'))
command += ' --request-id %s' % (mcm_r.get_attribute('prepid'))
if l_type.isDev():
command += ' --wmtest '
command += ' --user pdmvserv '
command += ' --group ppd '
command += ' --batch %s' % batchNumber
processString = mcm_r.get_attribute('process_string')
def efficiency(mcm_r):
feff = mcm_r.get_attribute(
'generator_parameters')[-1]['filter_efficiency']
meff = mcm_r.get_attribute(
'generator_parameters')[-1]['match_efficiency']
return float(feff) * float(meff)
if wmagent_type == 'MonteCarlo':
# calculate eff dev
command += ' --filter-eff %s' % (efficiency(mcm_r))
command += ' --number-events %s' % (
mcm_r.get_attribute('total_events'))
command += ' --primary-dataset %s' % (
mcm_r.get_attribute('dataset_name'))
elif wmagent_type == 'MonteCarloFromGEN':
# calculate eff dev
command += ' --filter-eff %s' % (efficiency(mcm_r))
command += ' --input-ds %s' % (
mcm_r.get_attribute('input_dataset'))
command += ' --primary-dataset %s' % (
mcm_r.get_attribute('dataset_name'))
if mcm_r.get_attribute('block_white_list'):
command += ' --blocks "' + ','.join(
mcm_r.get_attribute('block_white_list')) + '"'
if mcm_r.get_attribute('block_black_list'):
command += ' --blocks_black "' + ','.join(
mcm_r.get_attribute('block_black_list')) + '"'
command += ' --number-events %s' % (
mcm_r.get_attribute('total_events'))
elif wmagent_type == 'LHEStepZero':
command += ' --number-events %s' % (
mcm_r.get_attribute('total_events'))
command += ' --primary-dataset %s' % (
mcm_r.get_attribute('dataset_name'))
if mcm_r.get_attribute('mcdb_id') <= 0:
numberOfEventsPerJob = mcm_r.numberOfEventsPerJob()
if not numberOfEventsPerJob:
raise ValueError(
'Number of events per job could not be retrieved')
command += ' --events-per-job %s' % numberOfEventsPerJob
else:
command += ' --lhe '
if not processString:
processString = ''
processString += 'STEP0ATCERN'
elif wmagent_type == 'ReDigi':
command += ' --input-ds %s' % (
mcm_r.get_attribute('input_dataset'))
command += ' --primary-dataset %s' % (
mcm_r.get_attribute('dataset_name'))
## if PU dataset name is defined : add it
if mcm_r.get_attribute(
'pileup_dataset_name') and mcm_r.get_attribute(
'pileup_dataset_name').strip():
command += ' --pileup-ds ' + mcm_r.get_attribute(
'pileup_dataset_name')
## provide the total number of events requested: by default it is the amount in the input dataset.
# and wmcontrol / wma should understand that we want partial statistics if that number is lower than expected
command += ' --number-events %s' % (
mcm_r.get_attribute('total_events'))
# temp ev cont holder
eventcontentlist = mcm_r.get_first_output()
keeps = mcm_r.get_attribute('keep_output')
if not keeps[-1]:
raise ValueError('Is not set to save the output of last task')
for (i, content) in enumerate(eventcontentlist):
if keeps[i]:
command += ' --keep-step' + str(i + 1) + ' True'
if i > 0:
if len(mcm_r.get_attribute('config_id')):
command += ' --step%d-docID %s' % (
i + 1, mcm_r.get_attribute('config_id')[i])
else:
hash_id = mcm_r.configuration_identifier(i)
if hash_ids.document_exists(hash_id):
hash_doc = hash_ids.get(hash_id)
config_cache_id = hash_doc['docid']
command += ' --step%d-docID %s' % (i + 1,
config_cache_id)
config_id_from_hashkey.append(config_cache_id)
else:
command += ' --step%d-cfg %s_%d_cfg.py' % (
i + 1, mcm_r.get_attribute('prepid'), i + 1)
# set the output of
if i < len(eventcontentlist) - 1:
command += ' --step' + str(i + 1) + '-output ' + content
if mcm_r.get_attribute('block_white_list'):
command += ' --blocks "' + ','.join(
mcm_r.get_attribute('block_white_list')) + '"'
if mcm_r.get_attribute('block_black_list'):
command += ' --blocks_black "' + ','.join(
mcm_r.get_attribute('block_black_list')) + '"'
if processString:
command += ' --process-string ' + processString
command += '|| exit $? ;'
command += '\n'
if len(config_id_from_hashkey):
mcm_r.set_attribute('config_id', config_id_from_hashkey)
return command
def sendMail(self,
destination,
subject,
text,
sender=None,
reply_msg_ID=None,
accumulate=False):
if not isinstance(destination, list):
print "Cannot send email. destination should be a list of strings"
return
destination.sort()
msg = MIMEMultipart()
# it could happen that message are send after forking, threading and there's no current user anymore
msg['From'] = sender if sender else '*****@*****.**'
# add a mark on the subjcet automatically
if locator().isDev():
msg['Subject'] = '[McM-dev] ' + subject
destination = ["*****@*****.**"
] # if -dev send only to service account and sender
if sender:
destination.append(sender)
else:
msg['Subject'] = '[McM] ' + subject
msg['To'] = COMMASPACE.join(destination)
msg['Date'] = formatdate(localtime=True)
new_msg_ID = make_msgid()
msg['Message-ID'] = new_msg_ID
if reply_msg_ID is not None:
msg['In-Reply-To'] = reply_msg_ID
msg['References'] = reply_msg_ID
# accumulate messages prior to sending emails
com__accumulate = settings.get_value('com_accumulate')
force_com_accumulate = settings.get_value('force_com_accumulate')
if force_com_accumulate or (accumulate and com__accumulate):
with locker.lock('accumulating_notifcations'):
# get a subject where the request name is taken out
subject_type = " ".join(
filter(lambda w: w.count('-') != 2,
msg['Subject'].split()))
addressees = msg['To']
sendee = msg['From']
key = (subject_type, sendee, addressees)
if key in self.cache:
self.cache[key]['Text'] += '\n\n'
self.cache[key]['Text'] += text
self.cache[key]['N'] += 1
else:
self.cache[key] = {'Text': text, 'N': 1}
# self.logger.info('Got a message in cache %s'% (self.cache.keys()))
return new_msg_ID
# add a signature automatically
text += '\n\n'
text += 'McM Announcing service'
try:
msg.attach(MIMEText(text))
smtpObj = smtplib.SMTP()
smtpObj.connect()
smtpObj.sendmail(sender, destination, msg.as_string())
smtpObj.quit()
return new_msg_ID
except Exception as e:
print "Error: unable to send email", e.__class__
def GET(self, *args):
"""
Provides the injection command and does the injection.
"""
crn= args[0]
crdb = database('chained_requests')
mcm_cr = chained_request(crdb.get(crn))
rdb = database('requests')
mcm_rs=[]
## upload all config files to config cache, with "configuration economy" already implemented
from tools.locker import locker
from tools.handlers import ConfigMakerAndUploader
for rn in mcm_cr.get_attribute('chain'):
mcm_rs.append( request( rdb.get( rn )))
if self.mode=='inject' and mcm_rs[-1].get_attribute('status') != 'approved':
return dumps({"results" : False, "message" : 'requests %s in in "%s" status, requires "approved"'%( rn, mcm_rs[-1].get_attribute('status'))})
uploader = ConfigMakerAndUploader(prepid=rn, lock = locker.lock(rn))
uploader.run()
mcm_r = mcm_rs[-1]
from rest_api.BatchPrepId import BatchPrepId
batch_name = BatchPrepId().next_batch_id( mcm_cr.get_attribute('member_of_campaign') , create_batch=self.mode=='inject')
from tools.locker import semaphore_events, locker
semaphore_events.increment(batch_name)
from tools.ssh_executor import ssh_executor
from tools.locator import locator
l_type = locator()
with ssh_executor(server = 'pdmvserv-test.cern.ch') as ssh:
cmd='cd /afs/cern.ch/cms/PPD/PdmV/work/McM/dev-submit/\n'
cmd+=mcm_r.make_release()
cmd+='export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOST/voms_proxy.cert\n'
cmd+='export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}\n'
## until we get into production
there='--wmtest --wmtesturl cmsweb-testbed.cern.ch'
cmd+='wmcontrol.py --url-dict %s/public/restapi/chained_requests/get_dict/%s %s \n'%(l_type.baseurl(), crn, there)
if self.mode == 'show':
cherrypy.response.headers['Content-Type'] = 'text/plain'
return cmd
else:
_, stdout, stderr = ssh.execute(cmd)
cherrypy.response.headers['Content-Type'] = 'text/plain'
output = stdout.read()
error = stderr.read()
self.logger.log(output)
self.logger.log(error)
injected_requests = [l.split()[-1] for l in output.split('\n') if
l.startswith('Injected workflow:')]
approved_requests = [l.split()[-1] for l in output.split('\n') if
l.startswith('Approved workflow:')]
if injected_requests and not approved_requests:
return dumps({"results" : False, "message" : "Request %s was injected but could not be approved" % ( injected_requests )})
objects_to_invalidate = [
{"_id": inv_req, "object": inv_req, "type": "request", "status": "new", "prepid": self.prepid}
for inv_req in injected_requests if inv_req not in approved_requests]
if objects_to_invalidate:
return dumps({"results" : False, "message" : "Some requests %s need to be invalidated"})
added_requests = []
for mcm_r in mcm_rs:
added = [{'name': app_req, 'content': {'pdmv_prep_id': mcm_r.get_attribute('prepid')}} for app_req in approved_requests]
added_requests.extend( added )
with locker.lock(batch_name):
bdb = database('batches')
bat = batch(bdb.get(batch_name))
bat.add_requests(added_requests)
bat.update_history({'action': 'updated', 'step': crn})
bat.reload()
for mcm_r in mcm_rs:
mcm_r.set_attribute('reqmgr_name', added_requests)
for mcm_r in mcm_rs:
added = [{'name': app_req, 'content': {'pdmv_prep_id': mcm_r.get_attribute('prepid')}} for app_req in approved_requests]
mcm_r.set_attribute('reqmgr_name', added )
mcm_r.update_history({'action': 'inject','step' : batch_name})
mcm_r.set_attribute('approval', 'submit')
mcm_r.set_status(with_notification=False) ## maybe change to false
mcm_r.reload()
mcm_cr.update_history({'action' : 'inject','step': batch_name})
mcm_cr.set_attribute('step', len(mcm_rs)-1)
mcm_cr.set_attribute('status','processing')
mcm_cr.set_attribute('last_status', mcm_rs[-1].get_attribute('status'))
message=""
for mcm_r in mcm_rs:
message+=mcm_r.textified()
message+="\n\n"
mcm_cr.notify('Injection succeeded for %s'% crn,
message)
mcm_cr.reload()
return dumps({"results" : True, "message" : "request send to batch %s"% batch_name})
def sendMail(self,
destination,
subject,
text,
sender=None,
reply_msg_ID=None,
accumulate=False):
if not isinstance(destination, list):
print "Cannot send email. destination should be a list of strings"
return
destination.sort()
msg = MIMEMultipart()
#it could happen that message are send after forking, threading and there's no current user anymore
msg['From'] = sender if sender else '*****@*****.**'
## add a mark on the subjcet automatically
if locator().isDev():
msg['Subject'] = '[McM-dev] ' + subject
destination = ["*****@*****.**"] # if -dev send only to service account and sender
if sender:
destination.append(sender)
else:
msg['Subject'] = '[McM] ' + subject
msg['To'] = COMMASPACE.join(destination)
msg['Date'] = formatdate(localtime=True)
new_msg_ID = make_msgid()
msg['Message-ID'] = new_msg_ID
if reply_msg_ID is not None:
msg['In-Reply-To'] = reply_msg_ID
msg['References'] = reply_msg_ID
### accumulate messages prior to sending emails
com__accumulate=settings().get_value('com_accumulate')
force_com_accumulate=settings().get_value('force_com_accumulate')
if force_com_accumulate or (accumulate and com__accumulate):
with locker.lock('accumulating_notifcations'):
# get a subject where the request name is taken out
subject_type=" ".join( filter(lambda w : w.count('-')!=2, msg['Subject'].split()) )
addressees = msg['To']
sendee = msg['From']
key = (subject_type, sendee, addressees)
if key in self.cache:
self.cache[key]['Text']+='\n\n'
self.cache[key]['Text']+=text
self.cache[key]['N']+=1
else:
self.cache[key] = {'Text' : text, 'N':1}
#self.logger.log('Got a message in cache %s'% (self.cache.keys()))
return new_msg_ID
## add a signature automatically
text += '\n\n'
text += 'McM Announcing service'
try:
msg.attach(MIMEText(text))
smtpObj = smtplib.SMTP()
smtpObj.connect()
smtpObj.sendmail(sender, destination, msg.as_string())
smtpObj.quit()
return new_msg_ID
except Exception as e:
print "Error: unable to send email", e.__class__
def GET(self, *args):
"""
Provide the taskchain dictionnary for uploading to request manager
"""
from tools.locker import locker
crn= args[0]
crdb = database('chained_requests')
ccdb = database('chained_campaigns')
rdb = database('requests')
mcm_cr = chained_request(crdb.get(crn))
mcm_cc = chained_campaign( ccdb.get( mcm_cr.get_attribute('member_of_campaign')))
from tools.locator import locator
l_type=locator()
wma={
"RequestType" : "TaskChain",
"inputMode" : "couchDB",
"RequestString" : crn.replace(mcm_cc.get_attribute('prepid'), mcm_cc.get_attribute('alias')),
"Group" : "ppd",
"Requestor": "pdmvserv",
"Campaign" : mcm_cc.get_attribute('prepid'),
"OpenRunningTimeout" : 43200,
"TaskChain" : 0,
"ProcessingVersion": 1,
"RequestPriority" : 0,
"PrepID" : crn
}
mcm_rs=[]
for rn in mcm_cr.get_attribute('chain'):
mcm_rs.append( request( rdb.get( rn )))
step=1
last_io=None
for (i,r) in enumerate( mcm_rs ):
steps='step%d'%step
for (si, seq) in enumerate(r.get_attribute('sequences')):
if r.get_attribute('priority') > wma["RequestPriority"]:
wma["RequestPriority"] = r.get_attribute('priority')
wma['Task%d'%step] ={'TaskName' : 'Task%d'%step,
"ConfigCacheID" : r.get_attribute('config_id')[si],
"GlobalTag" : r.get_attribute('sequences')[si]['conditions'],
"CMSSWVersion" : r.get_attribute('cmssw_release'),
"ScramArch": r.get_scram_arch(),
"PrimaryDataset" : r.get_attribute('dataset_name'),
"KeepOutput" : r.get_attribute('keep_output')[si],
"AcquisitionEra" : r.get_attribute('member_of_campaign'),
"ProcessingString" : r.get_processing_string(si),
"ProcessingVersion" : r.get_attribute('version'),
"TimePerEvent" : r.get_attribute("time_event"),
"SizePerEvent" : r.get_attribute('size_event'),
"Memory" : r.get_attribute('memory')
}
if r.get_attribute('pileup_dataset_name'):
wma['Task%d'%step]["MCPileup"] = r.get_attribute('pileup_dataset_name')
for item in ['CMSSWVersion','ScramArch','TimePerEvent','SizePerEvent','GlobalTag','Memory']:
## needed for dictionnary validation in requests manager, but not used explicitely
if item not in wma:
wma[item] = wma['Task%d'%step][item]
if step==1:
wma['Task%d'%step].update({"SplittingAlgo" : "EventBased",
"RequestNumEvents" : r.get_attribute('total_events'),
"Seeding" : "AutomaticSeeding",
### gets wiped out anyways "SplittingArguments" : { "events_per_job" : 123, "events_per_lumi" : 78},
"EventsPerLumi" : 100, ## does not get seen in request manager, yet
"LheInputFiles" : r.get_attribute('mcdb_id')>0
})
else:
wma['Task%d'%step].update({"SplittingAlgo" : "EventAwareLumiBased",
"InputFromOutputModule" : "%soutput"%last_io,
"InputTask" : "Task%d"%(step-1),
### gets wiped out anyways "SplittingArguments" :{}
})
last_io=r.get_attribute('sequences')[si]['eventcontent'][0]
wma['TaskChain']+=1
step+=1
return dumps(wma)
def flow_to_next_step(self,
input_dataset='',
block_black_list=None,
block_white_list=None,
check_stats=True,
reserve=False):
if not block_white_list: block_white_list = []
if not block_black_list: block_black_list = []
self.logger.log('Flowing chained_request %s to next step...' %
(self.get_attribute('_id')))
if not self.get_attribute('chain'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'chained_request %s has got no root' %
(self.get_attribute('_id')))
# check on the approval of the chained request before all
## let it be flowing regardless
#if self.get_attribute('approval') == 'none':
# raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
# 'The approval of the chained request is none, and therefore flow cannot happen')
#this operation requires to access all sorts of objects
rdb = database('requests')
cdb = database('campaigns')
ccdb = database('chained_campaigns')
crdb = database('chained_requests')
fdb = database('flows')
adb = database('actions')
l_type = locator()
current_step = len(self.get_attribute(
'chain')) - 1 if reserve else self.get_attribute('step')
current_id = self.get_attribute('chain')[current_step]
next_step = current_step + 1
if not rdb.document_exists(current_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the request %s does not exist' % current_id)
current_request = request(rdb.get(current_id))
current_campaign = campaign(
cdb.get(current_request.get_attribute('member_of_campaign')))
if not ccdb.document_exists(self.get_attribute('member_of_campaign')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the chain request %s is member of %s that does not exist' %
(self.get_attribute('_id'),
self.get_attribute('member_of_campaign')))
mcm_cc = ccdb.get(self.get_attribute('member_of_campaign'))
if next_step >= len(mcm_cc['campaigns']):
if reserve: return False
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'chained_campaign %s does not allow any further flowing.' %
(self.get_attribute('member_of_campaign')))
if not reserve:
## is the current request in the proper approval
allowed_request_approvals = ['submit']
if current_request.get_attribute(
'approval') not in allowed_request_approvals:
raise self.NotApprovedException(
current_request.get_attribute('prepid'),
current_request.get_attribute('approval'),
allowed_request_approvals)
## is the current request in the proper status
allowed_request_statuses = ['submitted', 'done']
if current_request.get_attribute(
'status') not in allowed_request_statuses:
raise self.NotInProperStateException(
current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
original_action_id = self.get_attribute('chain')[0]
original_action_item = self.retrieve_original_action_item(
adb, original_action_id)
## what is the campaign to go to next and with which flow
(next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step]
if not fdb.document_exists(flow_name):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The flow %s does not exist' % flow_name)
mcm_f = flow(fdb.get(flow_name))
if not 'sequences' in mcm_f.get_attribute('request_parameters'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The flow %s does not contain sequences information.' %
(flow_name))
if not cdb.document_exists(next_campaign_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The next campaign %s does not exist' % next_campaign_id)
next_campaign = campaign(cdb.get(next_campaign_id))
if len(next_campaign.get_attribute('sequences')) != len(
mcm_f.get_attribute('request_parameters')['sequences']):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the sequences changes in flow %s are not consistent with the next campaign %s'
% (flow_name, next_campaign_id))
if next_campaign.get_attribute(
'energy') != current_campaign.get_attribute('energy'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'cannot flow any further. Request {0} has inconsistent energy.'
.format(next_campaign.get_attribute("prepid")))
if not next_campaign.is_release_greater_or_equal_to(
current_campaign.get_attribute('cmssw_release')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute("_id"),
'cannot flow any further. Request {0} has lower release version.'
.format(next_campaign.get_attribute("prepid")))
if next_campaign.get_attribute('type') == 'MCReproc' and (
not 'time_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not time per event is specified'
)
if next_campaign.get_attribute('type') == 'MCReproc' and (
not 'size_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not size per event is specified'
)
## check that it is allowed to flow
allowed_flow_approvals = ['flow', 'submit']
###### cascade of checks
"""
if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow (%s) is not in proper approval state (%s)'%(
mcm_f.get_attribute('prepid'),
mcm_f.get_attribute('approval')))
if not reserve and not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The chained request (%s) is not in the proper approval state (%s)'% (
self.get_attribute('_id'),
self.get_attribute('approval')))
"""
if not reserve and not mcm_f.get_attribute(
'approval') in allowed_flow_approvals:
if not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing'
% (mcm_f.get_attribute('approval'),
self.get_attribute('approval')))
if next_campaign.get_attribute('status') == 'stopped':
raise self.CampaignStoppedException(next_campaign_id)
#what is going to be the required number of events for the next request
#update the stats to its best
if not reserve:
current_request.get_stats()
next_total_events = current_request.get_attribute(
'completed_events')
## get the original expected events and allow a margin of 5% less statistics
statistics_fraction = settings().get_value('statistics_fraction')
current_eff_error = 1. - current_request.get_efficiency_error()
statistics_fraction = min(statistics_fraction, current_eff_error)
completed_events_to_pass = int(
current_request.get_attribute('total_events') *
statistics_fraction)
notify_on_fail = True ## to be tuned according to the specific cases
if current_request.get_attribute('completed_events') <= 0:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The number of events completed is negative or null')
else:
allowed_request_statuses = ['done']
## determine if this is a root -> non-root transition to potentially apply staged number
at_a_transition = (current_campaign.get_attribute('root') != 1
and next_campaign.get_attribute('root')
== 1)
if ('staged' in original_action_item or 'threshold'
in original_action_item) and at_a_transition:
allowed_request_statuses.append('submitted')
##check status
if not current_request.get_attribute(
'status') in allowed_request_statuses:
raise self.NotInProperStateException(
current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
##special check at transition that the statistics is good enough
if at_a_transition:
# at a root -> non-root transition only does the staged/threshold functions !
if 'staged' in original_action_item:
next_total_events = int(original_action_item['staged'])
completed_events_to_pass = next_total_events
if 'threshold' in original_action_item:
next_total_events = int(
current_request.get_attribute('total_events') *
float(original_action_item['threshold'] / 100.))
completed_events_to_pass = next_total_events
if check_stats and (
current_request.get_attribute('completed_events') <
completed_events_to_pass):
if notify_on_fail:
current_request.notify(
'Flowing %s with not enough statistics' %
(current_request.get_attribute('prepid')),
'For the request %s, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s in chain %s \n\n Please report to the operation HN or at the next MccM what action should be taken.\n\n %srequests?prepid=%s\n%schained_requests?contains=%s\n%schained_requests?prepid=%s '
% (current_request.get_attribute('prepid'),
current_request.get_attribute('completed_events'),
completed_events_to_pass,
self.get_attribute('prepid'), l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(), self.get_attribute('prepid')),
accumulate=True)
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The number of events completed (%s) is not enough for the requirement (%s)'
% (current_request.get_attribute('completed_events'),
completed_events_to_pass))
## select what is to happened : [create, patch, use]
next_id = None
approach = None
next_request = None
if next_step != len(self.get_attribute('chain')):
#not at the end
next_id = self.get_attribute('chain')[next_step]
if not rdb.document_exists(next_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The next request (%s) according to the step (%s) does not exist'
% (next_id, next_step))
next_request = request(rdb.get(next_id))
if next_request.get_attribute('status') == 'new':
#most likely a rewind + resub
approach = 'patch'
else:
##this is always the case in chains reserved from existing things: so use the next request
approach = 'use'
#raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
#'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % (
#next_id, next_step, next_request.get_attribute('status')))
else:
## look in *other* chained campaigns whether you can suck in an existing request
## look up all chained requests that start from the same root request
## remove <pwg>-chain_ and the -serial number, replacing _ with a .
toMatch = '.'.join(
self.get_attribute('prepid').split('_')[1:][0:next_step +
1]).split('-')[0]
## make sure they get ordered by prepid
related_crs = sorted(crdb.queries(
['root_request==%s' % original_action_id]),
key=lambda cr: cr['prepid'])
vetoed_last = []
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid'] == self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute(
'member_of_campaign'):
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) > next_step:
## one existing request in the very same chained campaign has already something used, make sure it is not going to be used
vetoed_last.append(
mcm_cr.get_attribute('chain')[next_step])
continue
else:
continue
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid'] == self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute(
'member_of_campaign'):
continue
truncated = '.'.join(
existing_cr['prepid'].split('_')[1:][0:next_step +
1]).split('-')[0]
self.logger.error('to match : %s , this one %s' %
(toMatch, truncated))
if truncated == toMatch:
#we found a chained request that starts with all same steps
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) <= next_step:
#found one, but it has not enough content either
continue
if mcm_cr.get_attribute('chain')[next_step] in vetoed_last:
continue
next_id = mcm_cr.get_attribute('chain')[next_step]
break
if next_id:
approach = 'use'
else:
approach = 'create'
if approach == 'create':
from rest_api.RequestPrepId import RequestPrepId
next_id = RequestPrepId().next_prepid(
current_request.get_attribute('pwg'), next_campaign_id)
next_request = request(rdb.get(next_id))
request.transfer(current_request, next_request)
self.request_join(next_request)
elif approach == 'use':
## there exists a request in another chained campaign that can be re-used here.
# take this one. advance and go on
next_request = request(rdb.get(next_id))
if not reserve:
self.set_attribute('step', next_step)
self.set_attribute('last_status',
next_request.get_attribute('status'))
self.update_history({'action': 'flow', 'step': str(next_id)})
self.set_attribute('status', 'processing')
if not self.get_attribute("prepid") in next_request.get_attribute(
"member_of_chain"):
## register the chain to the next request
self.request_join(next_request)
saved = rdb.update(next_request.json())
if not saved:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Unable to save %s with updated member_of_chains' %
next_id)
return True
elif approach == 'patch':
## there exists already a request in the chain (step!=last) and it is usable for the next stage
next_request = request(next_campaign.add_request(rdb.get(next_id)))
## propagate again some of the fields of the previous request.
request.transfer(current_request, next_request)
else:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Unrecognized approach %s' % approach)
#current_request -> next_request
#current_campaign -> next_campaign
##determine whether we have an input dataset for the next request
if len(current_request.get_attribute('reqmgr_name')):
last_wma = current_request.get_attribute('reqmgr_name')[-1]
if 'content' in last_wma and 'pdmv_dataset_name' in last_wma[
'content']:
input_dataset = last_wma['content']['pdmv_dataset_name']
else:
statsDB = database('stats',
url='http://cms-pdmv-stats.cern.ch:5984/')
if statsDB.document_exists(last_wma['name']):
latestStatus = statsDB.get(last_wma['name'])
input_dataset = latestStatus['pdmv_dataset_name']
if input_dataset:
next_request.set_attribute('input_dataset', input_dataset)
## set blocks restriction if any
if block_black_list:
next_request.set_attribute('block_black_list', block_black_list)
if block_white_list:
next_request.set_attribute('block_white_list', block_white_list)
## register the flow to the request
next_request.set_attribute('flown_with', flow_name)
##assemble the campaign+flow => request
request.put_together(next_campaign, mcm_f, next_request)
if not reserve:
#already taking stage and threshold into account
next_request.set_attribute('total_events', next_total_events)
next_request.update_history({
'action': 'flow',
'step': self.get_attribute('prepid')
})
request_saved = rdb.save(next_request.json())
if not request_saved:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Could not save the new request %s' %
(next_request.get_attribute('prepid')))
## inspect priority
self.set_priority(original_action_item['block_number'])
if not reserve:
# sync last status
self.set_attribute('last_status',
next_request.get_attribute('status'))
# we can only be processing at this point
self.set_attribute('status', 'processing')
# set to next step
self.set_attribute('step', next_step)
self.update_history({
'action': 'flow',
'step': next_request.get_attribute('prepid')
})
if not reserve:
notification_subject = 'Flow for request %s in %s' % (
current_request.get_attribute('prepid'), next_campaign_id)
notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % (
current_request.get_attribute('prepid'),
self.get_attribute('member_of_campaign'), next_campaign_id,
flow_name, next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_request.get_attribute('input_dataset'), l_type.baseurl(),
current_request.get_attribute('prepid'), l_type.baseurl(),
next_request.get_attribute('prepid'))
current_request.notify(notification_subject,
notification_text,
accumulate=True)
else:
notification_subject = 'Reservation of request {0}'.format(
next_request.get_attribute('prepid'))
notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format(
next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_campaign_id,
l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(),
next_request.get_attribute('prepid'),
)
next_request.notify(notification_subject,
notification_text,
accumulate=True)
return True
def build_location(sub_directory):
l_type = locator()
directory = l_type.workLocation()
return os.path.abspath(directory) + '/' + sub_directory + '/'
