def __init__(self, sub_directory, care_on_existing=True, clean_on_exit=True): self.cleanup = clean_on_exit l_type=locator() directory = l_type.workLocation() self.directory = os.path.abspath(directory) + '/' + sub_directory + '/' self.careOfExistingDirectory = care_on_existing # check if directory is empty if not self.directory: self.logger.error('Data directory is not defined', handler=self.hname) raise Exception('Data directory is not defined.') # check if exists (and force) if os.path.exists(self.directory): if self.careOfExistingDirectory: self.logger.error( os.popen('echo %s; ls -f %s'%(self.directory, self.directory)).read()) self.logger.error('Directory ' + self.directory + ' already exists.') raise Exception('Data directory %s already exists'%(self.directory)) else: self.logger.log('Directory ' + self.directory + ' already exists.') else: self.logger.log('Creating directory :'+self.directory) # recursively create any needed parents and the dir itself os.makedirs(self.directory)
def default(self, *vpath, **params): method = getattr(self, cherrypy.request.method, None) if not method: raise cherrypy.HTTPError(405, "Method not implemented.") if self.access_limit is not None: self.logger.log('Setting access limit to access_rights.%s (%s)' % (roles[self.access_limit], self.access_limit)) self.authenticator.set_limit(self.access_limit) elif cherrypy.request.method in self.limit_per_method: self.authenticator.set_limit(self.limit_per_method[cherrypy.request.method]) else: raise cherrypy.HTTPError(403, 'You cannot access this page with method %s' % cherrypy.request.method ) user_p = user_pack() l_type = locator() if not user_p.get_username(): #meaning we are going public, only allow GET. #if cherrypy.request.method != 'GET' or not l_type.isDev(): # raise cherrypy.HTTPError(403, 'User credentials were not provided.') if not 'public' in str(cherrypy.url()): self.logger.error('From within %s, adfs-login not found: \n %s \n %s' % (self.__class__.__name__, str(cherrypy.request.headers), str(cherrypy.url()) )) else: if not self.authenticator.can_access(user_p.get_username()): raise cherrypy.HTTPError(403, 'You cannot access this page, the limit for the page is {0} ({1})'.format(roles[self.authenticator.get_limit()], self.authenticator.get_limit())) # counter for calls with locker.lock("rest-call-counter"): self.counter[method.im_class.__name__][method.__name__] += 1 return method(*vpath, **params)
def GET(self, *args): """ Goes through invalidation documents and display (/) and announce them to data ops (/announce) """ announce = False clear = False if len(args) != 0: if args[0] == 'announce': announce = True if args[0] == 'clear': clear = True idb = database('invalidations') r_to_be_rejected = map(invalidation, idb.queries(['status==new', 'type==request'])) ds_to_be_invalidated = map(invalidation, idb.queries(['status==new', 'type==dataset'])) ds_to_be_invalidated = filter(lambda ds : not 'None-' in ds.get_attribute('object'), ds_to_be_invalidated) l_type = locator() def add_prepid(invalid, html): if 'prepid' in invalid.json(): html += 'for request <a href=%srequests?prepid=%s> %s </a>' % ( l_type.baseurl(), invalid.get_attribute('prepid'), invalid.get_attribute('prepid')) def print_invalidations(invalids): a_text='' for invalid in invalids: a_text += ' %s\n' % (invalid.get_attribute('object')) return a_text html = '<html><body>\n' html += 'Requests to be aborted/rejected <br>\n' html += '<ul>\n' for r in r_to_be_rejected: html += '<li> <a href=%sreqmgr/view/details/%s> %s </a>' % (l_type.cmsweburl(), r.get_attribute('object'), r.get_attribute('object') ) add_prepid(r, html) html += '</li>\n' html += '</ul>\n' html += 'Datasets to be invalidated <br>\n' html += '<ul>\n' for ds in ds_to_be_invalidated: html += '<li> %s ' % ( ds.get_attribute('object') ) add_prepid(ds, html) html += '</li>\n' html += '</ul>\n' html += '<a href=%srestapi/invalidations/inspect/clear> Clear invalidation withouth announcing</a><br>\n'%( l_type.baseurl() ) html += '<a href=%srestapi/invalidations/inspect/announce> Announce invalidations</a><br>\n'%( l_type.baseurl() ) html += '<a href=%srestapi/invalidations/inspect> Back</a><br>\n'%( l_type.baseurl() ) html += '</html></body>\n' if announce: announcer = Announcer() announcer.announce(ds_to_be_invalidated, r_to_be_rejected) if clear: __clearer = Clearer() __clearer.clean() return html
def notify_batch(self, batch_id, message_notes): message = message_notes to_who = [settings.get_value('service_account')] l_type = locator() if l_type.isDev(): to_who.append(settings.get_value('hypernews_test')) else: to_who.append(settings.get_value('dataops_announce')) single_batch = batch(self.bdb.get(batch_id)) subject = single_batch.get_subject('[Notification]') current_message_id = single_batch.get_attribute('message_id') self.logger.info('current msgID: %s' % current_message_id) if current_message_id != '': result = single_batch.notify(subject, message, who=to_who, sender=None, reply_msg_ID=current_message_id) self.logger.info('result if True : %s' % result) else: result = single_batch.notify(subject, message, who=to_who, sender=None) self.logger.info('result if False : %s' % result) notification( subject, message, [], group=notification.BATCHES, target_role='production_manager', action_objects=[single_batch.get_attribute('prepid')], object_type='batches', base_object=single_batch ) single_batch.update_history({'action': 'notify', 'step': message}) single_batch.reload() return {'results': result}
def default(self, *vpath, **params): method = getattr(self, cherrypy.request.method, None) if not method: raise cherrypy.HTTPError(405, "Method not implemented.") if self.access_limit is not None: self.logger.log('Setting access limit to access_rights.%s (%s)' % (roles[self.access_limit], self.access_limit)) self.authenticator.set_limit(self.access_limit) elif cherrypy.request.method in self.limit_per_method: self.authenticator.set_limit(self.limit_per_method[cherrypy.request.method]) else: raise cherrypy.HTTPError(403, 'You cannot access this page with method %s' % cherrypy.request.method ) user_p = user_pack() l_type = locator() if not user_p.get_username(): #meaning we are going public, only allow GET. #if cherrypy.request.method != 'GET' or not l_type.isDev(): # raise cherrypy.HTTPError(403, 'User credentials were not provided.') if not 'public' in str(cherrypy.url()): self.logger.error('From within %s, adfs-login not found: \n %s \n %s' % (self.__class__.__name__, str(cherrypy.request.headers), str(cherrypy.url()) )) else: if not self.authenticator.can_access(user_p.get_username()): raise cherrypy.HTTPError(403, 'You cannot access this page, the limit for the page is {0} ({1})'.format(roles[self.authenticator.get_limit()], self.authenticator.get_limit())) # counter for calls with locker.lock("rest-call-counter"): key = method.im_class.__name__ + method.__name__ try: RESTResource.counter[key] += 1 except KeyError: RESTResource.counter[key] = 0 return method(*vpath, **params)
def notify_batch(self, batch_id, message_notes): message = message_notes to_who = [settings().get_value('service_account')] l_type = locator() if l_type.isDev(): to_who.append(settings().get_value('hypernews_test')) else: to_who.append(settings().get_value('dataops_announce')) single_batch = batch(self.bdb.get(batch_id)) subject = single_batch.get_subject('[Notification]') current_message_id = single_batch.get_attribute('message_id') self.logger.log('current msgID: %s' % current_message_id) if current_message_id != '': result = single_batch.notify(subject, message, who=to_who, sender=None, reply_msg_ID=current_message_id) self.logger.log('result if True : %s' % result) else: result = single_batch.notify(subject, message, who=to_who, sender=None) self.logger.log('result if False : %s' % result) single_batch.update_history({'action': 'notify', 'step': message}) single_batch.reload() return {'results': result}
def notify_batch(self, batch_id, message_notes): message = message_notes to_who = [settings().get_value('service_account')] l_type = locator() if l_type.isDev(): to_who.append( settings().get_value('hypernews_test')) else: to_who.append( settings().get_value('dataops_announce' )) single_batch = batch(self.bdb.get(batch_id)) subject = single_batch.get_subject('[Notification]') current_message_id = single_batch.get_attribute('message_id') self.logger.log('current msgID: %s' %current_message_id) if current_message_id != '': result = single_batch.notify(subject,message,who=to_who,sender=None,reply_msg_ID=current_message_id) self.logger.log('result if True : %s' %result) else: result = single_batch.notify(subject,message,who=to_who,sender=None) self.logger.log('result if False : %s' %result) self.update_history({'action':'notify', 'step' : message}) self.reload() return {'results': result}
def request_join(self, req): with locker.lock(req.get_attribute('prepid')): chain = req.get_attribute("member_of_chain") chain.append(self.get_attribute('_id')) req.set_attribute("member_of_chain", chain) loc = locator() req.notify( "Request {0} joined chain".format(req.get_attribute('prepid')), "Request {0} has successfully joined chain {1}\n\n{2}\n".format( req.get_attribute('prepid'), self.get_attribute('_id'), "/".join([ loc.baseurl(), "requests?prepid={0}".format(req.get_attribute('prepid')) ])), accumulate=True) req.update_history({ 'action': 'join chain', 'step': self.get_attribute('_id') }) if not req.get_attribute('prepid') in self.get_attribute('chain'): chain = self.get_attribute('chain') chain.append(req.get_attribute('prepid')) self.set_attribute("chain", chain) self.update_history({ 'action': 'add request', 'step': req.get_attribute('prepid') })
def make_command(self,mcm_r=None): l_type = locator() cmd = 'cd %s \n' % ( l_type.workLocation()) if mcm_r: cmd += mcm_r.make_release() cmd += 'export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOSTNAME/voms_proxy.cert\n' cmd += 'export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}\n' there = '' if l_type.isDev(): there = '--wmtest --wmtesturl cmsweb-testbed.cern.ch' cmd += 'wmcontrol.py --url-dict %s/public/restapi/chained_requests/get_dict/%s %s \n'%(l_type.baseurl(), self.prepid, there) return cmd
def GET(self, *args): """ Send a reminder to the production managers for existing opened mccm documents """ mdb = database('mccms') mccms = mdb.queries(['status==new']) udb = database('users') block_threshold = 0 if len(args): block_threshold = int(args[0]) mccms = filter(lambda m: m['block'] <= block_threshold, mccms) mccms = sorted(mccms, key=lambda m: m['block']) if len(mccms) == 0: return dumps({ "results": True, "message": "nothing to remind of at level %s, %s" % (block_threshold, mccms) }) l_type = locator() com = communicator() subject = 'Gentle reminder on %s tickets to be operated by you' % ( len(mccms)) message = '''\ Dear Production Managers, please find below the details of %s opened MccM tickets that need to be operated. ''' % (len(mccms)) for mccm in mccms: message += 'Ticket : %s (block %s)\n' % (mccm['prepid'], mccm['block']) message += ' %smccms?prepid=%s \n\n' % (l_type.baseurl(), mccm['prepid']) message += '\n' to_who = [settings().get_value('service_account')] to_who.extend( map(lambda u: u['email'], udb.query(query="role==production_manager", page_num=-1))) com.sendMail(to_who, subject, message) return dumps({ "results": True, "message": map(lambda m: m['prepid'], mccms) })
def make_command(self): l_type = locator() command = 'export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOSTNAME/voms_proxy.cert\n' command += 'source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh\n' test_path = '' test_params = '' if l_type.isDev(): test_path = '_testful' test_params = '--wmtest --wmtesturl cmsweb-testbed.cern.ch' command += 'python /afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol%s/wmapprove.py --workflows %s %s\n' % ( test_path, self.workflows, test_params) return command
def streaming_function(): mccms_db = database('mccms') users_db = database('users') generator_contacts_query = users_db.construct_lucene_query({'role': 'generator_contact'}) generator_contacts = users_db.full_text_search("search", generator_contacts_query, page=-1) generator_contacts_by_pwg = {} generator_contacts_emails = set() for contact in generator_contacts: for pwg in contact.get('pwg', []): if pwg not in generator_contacts_by_pwg: generator_contacts_by_pwg[pwg] = [] generator_contacts_by_pwg[pwg].append(contact.get('email')) generator_contacts_emails.add(contact.get('email')) __query = mccms_db.construct_lucene_query({'status': 'new'}) mccms_tickets = mccms_db.full_text_search('search', __query, page=-1) authors_tickets_dict = dict() yield '<pre>' for ticket in mccms_tickets: yield 'Processing ticket %s\n' % (ticket['prepid']) mccm_ticket = mccm(json_input=ticket) pwg = mccm_ticket.get_attribute('pwg') authors = mccm_ticket.get_actors(what='author_email') yield '%s worked on %s\n' % (authors, ticket['prepid']) authors = filter(lambda e: e in generator_contacts_emails, list(set(authors + generator_contacts_by_pwg.get(pwg, [])))) yield '%s will be notified about %s\n' % (authors, ticket['prepid']) for author_email in authors: if author_email in generator_contacts_emails: if author_email not in authors_tickets_dict: authors_tickets_dict[author_email] = set() authors_tickets_dict[author_email].add(ticket['prepid']) subject_template = 'Gentle reminder on %s ticket%s to be operated by you' message_template = ('Dear GEN Contact,\nPlease find below the details of %s MccM ticket%s in status "new". ' + 'Please present them in next MccM googledoc or cancel tickets if these are not needed anymore.\n\n') base_url = locator().baseurl() mail_communicator = communicator() service_account = settings.get_value('service_account') for author_email, ticket_prepids in authors_tickets_dict.iteritems(): num_tickets = len(ticket_prepids) subject = subject_template % (num_tickets, '' if num_tickets == 1 else 's') message = message_template % (num_tickets, '' if num_tickets == 1 else 's') for ticket_prepid in ticket_prepids: message += 'Ticket: %s\n%smccms?prepid=%s\n\n' % (ticket_prepid, base_url, ticket_prepid) yield '.' yield '\n' message += 'You received this email because you are listed as generator contact of physics group(s) of these tickets.\n' self.logger.info('Email:%s\nSubject: %s\nMessage:%s' % (author_email, subject, message)) mail_communicator.sendMail([author_email, service_account], subject, message) yield 'Email sent to %s\n' % (author_email)
def make_command(self, mcm_r=None): l_type = locator() cmd = 'cd %s \n' % (l_type.workLocation()) if mcm_r: cmd += mcm_r.make_release() cmd += 'export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOSTNAME/voms_proxy.cert\n' there = '' if l_type.isDev(): there = '--wmtest --wmtesturl cmsweb-testbed.cern.ch' cmd += 'export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}\n' cmd += 'wmcontrol.py --dont_approve --url-dict %spublic/restapi/chained_requests/get_dict/%s %s \n' % ( l_type.baseurl(), self.prepid, there) return cmd
def streaming_function(): mccms_db = database('mccms') users_db = database('users') __query = mccms_db.construct_lucene_query({'status': 'new'}) mccms_tickets = mccms_db.full_text_search('search', __query, page=-1) non_gen_contact_authors = set() authors_tickets_dict = dict() emails_prepids = dict() for ticket in mccms_tickets: yield '\nProcessing ticket %s' % (ticket['prepid']) mccm_ticket = mccm(json_input=ticket) authors = mccm_ticket.get_actors(what='author_email') for author_email in authors: if author_email in authors_tickets_dict: authors_tickets_dict[author_email].append(ticket['prepid']) elif author_email not in non_gen_contact_authors: __role_query = users_db.construct_lucene_query({'email': author_email}) result = users_db.full_text_search('search', __role_query, page=-1, include_fields='role,prepid') time.sleep(0.5) # we don't want to crash DB with a lot of single queries if result and result[0]['role'] == 'generator_contact': authors_tickets_dict[author_email] = [ticket['prepid']] emails_prepids[author_email] = result[0]['prepid'] else: non_gen_contact_authors.add(author_email) yield '.' subject_part1 = 'Gentle reminder on %s ' subject_part2 = ' to be operated by you' message_part1 = 'Dear GEN Contact, \nPlease find below the details of %s MccM ' message_part2 = ' in status "new". Please present them in next MccM googledoc or cancel tickets if these are not needed anymore.\n\n' base_url = locator().baseurl() mail_communicator = communicator() for author_email, ticket_prepids in authors_tickets_dict.iteritems(): num_tickets = len(ticket_prepids) full_message = (message_part1 % (num_tickets)) + ('ticket' if num_tickets == 1 else 'tickets') + message_part2 for ticket_prepid in ticket_prepids: full_message += 'Ticket: %s \n' % (ticket_prepid) full_message += '%smccms?prepid=%s \n\n' % (base_url, ticket_prepid) yield '.' full_message += '\n' subject = (subject_part1 % (num_tickets)) + ('ticket' if num_tickets == 1 else 'tickets') + subject_part2 notification( subject, full_message, [emails_prepids[author_email]], group=notification.REMINDERS, action_objects=ticket_prepids, object_type='mccms') mail_communicator.sendMail([author_email], subject, full_message) yield '\nEmail sent to %s\n' % (author_email)
def request_join(self, req): with locker.lock(req.get_attribute('prepid')): chain = req.get_attribute("member_of_chain") chain.append(self.get_attribute('_id')) req.set_attribute("member_of_chain", chain) loc = locator() req.notify("Request {0} joined chain".format(req.get_attribute('prepid')), "Request {0} has successfully joined chain {1}\n\n{2}\n".format(req.get_attribute('prepid'), self.get_attribute('_id'), "/".join([loc.baseurl(), "requests?prepid={0}".format(req.get_attribute('prepid'))]))) req.update_history({'action': 'join chain', 'step': self.get_attribute('_id')}) if not req.get_attribute('prepid') in self.get_attribute('chain'): chain = self.get_attribute('chain') chain.append(req.get_attribute('prepid')) self.set_attribute("chain", chain) self.update_history({'action': 'add request', 'step': req.get_attribute('prepid')})
def __init__(self): self.setup_directories() self.setup_logger() self.get_submmited_prepids() self.batch_retry_timeout = settings.get_value('batch_retry_timeout') self.check_term_runlimit = settings.get_value('check_term_runlimit') try: self.ssh_exec = ssh_executor() except Exception as e: self.ssh_exec = None self.logger.error(str(e) + 'traceback %s ' % traceback.format_exc()) return if locator().isDev(): self.group = '/dev' else: self.group = '/prod'
def get(self, block_threshold=0): """ Send a reminder to the production managers for existing opened mccm documents """ mdb = database('mccms') udb = database('users') __query = mdb.construct_lucene_query({'status': 'new'}) mccms = mdb.full_text_search('search', __query, page=-1) mccms = filter(lambda m: m['block'] <= block_threshold, mccms) mccms = sorted(mccms, key=lambda m: m['block']) if len(mccms) == 0: return {"results": True, "message": "nothing to remind of at level %s, %s" % (block_threshold, mccms)} l_type = locator() com = communicator() subject = 'Gentle reminder on %s tickets to be operated by you' % (len( mccms)) message = '''\ Dear Production Managers, please find below the details of %s opened MccM tickets that need to be operated. ''' % (len(mccms)) mccm_prepids = [] for _mccm in mccms: prepid = _mccm['prepid'] message += 'Ticket : %s (block %s)\n' % (prepid, _mccm['block']) message += ' %smccms?prepid=%s \n\n' % (l_type.baseurl(), prepid) mccm_prepids.append(prepid) message += '\n' to_who = [settings.get_value('service_account')] to_who.extend(map(lambda u: u['email'], udb.query(query="role==production_manager", page_num=-1))) notification( subject, message, [], group=notification.REMINDERS, action_objects=mccm_prepids, object_type='mccms', target_role='production_manager') com.sendMail( to_who, subject, message) return {"results": True, "message": map(lambda m: m['prepid'], mccms)}
def __init__(self, test_id, test_script): self.script_for_test = test_script self.test_id = test_id self.test_err = os.path.abspath( self.script_for_test + '.err') self.test_out = os.path.abspath( self.script_for_test + '.out') locat = locator() if locat.isDev(): self.group = '/dev' else: self.group = '/prod' self.directory_for_test = os.path.dirname(self.script_for_test) self.ssh_exec = ssh_executor(self.directory_for_test, self.test_id) self.log_out = 'Not available' self.log_err = 'Not available' self.timeout = settings().get_value('batch_timeout')
def __init__(self, db_name='',url=None, cache=False): host = os.environ['HOSTNAME'] if url is None: url = locator().dbLocation() if not db_name: raise self.DatabaseNotFoundException(db_name) self.db_name = db_name self.cache = cache if self.db_name in ['campaigns','chained_campaigns']: ## force cache for those. self.cache=True try: self.db = Database(db_name, url=url) except ValueError as ex: raise self.DatabaseAccessError(db_name) self.allowed_operators = ['<=', '<', '>=', '>', '==', '~=']
def __init__(self): self.setup_directories() self.setup_logger() self.get_submmited_prepids() self.batch_retry_timeout = settings.get_value('batch_retry_timeout') self.check_term_runlimit = settings.get_value('check_term_runlimit') try: self.ssh_exec = ssh_executor() except Exception as e: self.ssh_exec = None self.logger.error( str(e) + 'traceback %s ' % traceback.format_exc()) return if locator().isDev(): self.group = '/dev' else: self.group = '/prod'
def get(self, pwgs): """ Ask for the increase of the role of the current user to the given pwg """ # get who's there user_p = user_pack() udb = database(self.db_name) mcm_u = user(udb.get(user_p.get_username())) # get the requested pwgs pwgs = pwgs.split(',') # set the pwgs to the current user current = mcm_u.get_attribute('pwg') current = list(set(current + pwgs)) mcm_u.set_attribute('pwg', current) mcm_u.update_history({'action': 'ask role', 'step': pwgs}) udb.update(mcm_u.json()) # get the production managers emails __query = udb.construct_lucene_query({'role': 'production_manager'}) production_managers = udb.full_text_search('search', __query, page=-1) # send a notification to prod manager + service to_who = map(lambda u: u['email'], production_managers) + [ settings.get_value('service_account') ] to_who.append(user_p.get_email()) com = communicator() l_type = locator() subject = 'Increase role for user %s' % mcm_u.get_attribute('fullname') message = 'Please increase the role of the user %s to the next level.\n\n%susers?prepid=%s' % ( mcm_u.get_attribute('username'), l_type.baseurl(), mcm_u.get_attribute('username')) notification(subject, message, [], group=notification.USERS, action_objects=[mcm_u.get_attribute('prepid')], object_type='users', target_role='production_manager') com.sendMail(to_who, subject, message) return { "results": True, "message": "user %s in for %s" % (mcm_u.get_attribute('username'), current) }
def __init__(self, db_name='', url=None, cache_enabled=False): host = os.environ['HOSTNAME'] if url is None: url = locator().dbLocation() if not db_name: raise self.DatabaseNotFoundException(db_name) self.db_name = db_name self.cache_enabled = cache_enabled if self.db_name in ['campaigns', 'chained_campaigns']: ## force cache for those. self.cache_enabled = True try: self.db = Database(db_name, url=url) except ValueError as ex: raise self.DatabaseAccessError(db_name) self.allowed_operators = ['<=', '<', '>=', '>', '==', '~=']
def GET(self, *args): """ Twiki display of mccm ticket for a given meeting date and /pwg optional """ date = args[0] pwgs = None if len(args) > 1: pwgs = args[1].split(',') mdb = database('mccms') to_be_shown = [ 'prepid', 'notes', 'deadline', 'requests', 'chains', 'repetitions' ] l_type = locator() mdocs = mdb.queries(['meeting==%s' % date]) if pwgs: text = "---++ MccM Tickets for %s : %s \n" % (date, ', '.join(pwgs)) for pwg in pwgs: mdocs_pwg = filter(lambda m: m['pwg'] == pwg, mdocs) text += "---+++ Tickets for %s \n" % pwg text += "[[%smccms?meeting=%s&pwg=%s][link to McM]]\n" % ( l_type.baseurl(), date, pwg) for t in mdocs_pwg: text += " * " for item in to_be_shown: if item in t: text += "%s " % t[item] text += '\n' else: text = "---++ MccM Tickets for %s \n<br>" % (date) text += "[[%smccms?meeting=%s][link to McM]]\n" % ( l_type.baseurl(), date) for t in mdocs: text += " * " for item in to_be_shown: if item in t: text += "%s " % t[item] text += '\n' return text
def GET(self, *args): """ Ask for the increase of the role of the current user to the given pwg """ if not args: return dumps({"results": False, "Message": "not pwg provided"}) ## get who's there user_p = user_pack() udb = database(self.db_name) mcm_u = user(udb.get(user_p.get_username())) ## get the requested pwgs pwgs = args[0].split(',') #### set the pwgs to the current user current = mcm_u.get_attribute('pwg') current = list(set(current + pwgs)) mcm_u.set_attribute('pwg', current) mcm_u.update_history({'action': 'ask role', 'step': args[0]}) udb.update(mcm_u.json()) ## get the production managers emails production_managers = udb.queries(['role==production_manager']) ### send a notification to prod manager + service to_who = map(lambda u: u['email'], production_managers) + [ settings().get_value('service_account') ] to_who.append(user_p.get_email()) com = communicator() l_type = locator() com.sendMail( to_who, 'Increase role for user %s' % mcm_u.get_attribute('fullname'), 'Please increase the role of the user %s to the next level.\n\n%susers?prepid=%s' % (mcm_u.get_attribute('username'), l_type.baseurl(), mcm_u.get_attribute('username'))) return dumps({ "results": True, "message": "user %s in for %s" % (mcm_u.get_attribute('username'), current) })
def sendMail(self, destination, subject, text, sender=None, reply_msg_ID=None): if not isinstance(destination, list): print "Cannot send email. destination should be a list of strings" return msg = MIMEMultipart() #it could happen that message are send after forking, threading and there's no current user anymore msg['From'] = sender if sender else '*****@*****.**' msg['To'] = COMMASPACE.join(destination) msg['Date'] = formatdate(localtime=True) new_msg_ID = make_msgid() msg['Message-ID'] = new_msg_ID if reply_msg_ID != None: msg['In-Reply-To'] = reply_msg_ID msg['References'] = reply_msg_ID ## add a mark on the subjcet automatically if locator().isDev(): msg['Subject'] = '[McM-dev] ' + subject else: msg['Subject'] = '[McM] ' + subject ## add a signature automatically text += '\n' text += 'McM Announcing service' try: msg.attach(MIMEText(text)) smtpObj = smtplib.SMTP() smtpObj.connect() smtpObj.sendmail(sender, destination, msg.as_string()) smtpObj.quit() return new_msg_ID except Exception as e: print "Error: unable to send email", e.__class__
def GET(self, *args): """ Send a reminder to the production managers for existing opened mccm documents """ mdb = database('mccms') mccms = mdb.queries(['status==new']) udb = database('users') block_threshold = 0 if len(args): block_threshold = int(args[0]) mccms = filter( lambda m : m['block'] <= block_threshold, mccms) mccms = sorted( mccms, key = lambda m : m['block']) if len(mccms)==0: return dumps({"results": True,"message": "nothing to remind of at level %s, %s"% (block_threshold, mccms)}) l_type = locator() com = communicator() subject = 'Gentle reminder on %s tickets to be operated by you' % ( len( mccms)) message = '''\ Dear Production Managers, please find below the details of %s opened MccM tickets that need to be operated. ''' % (len(mccms)) for mccm in mccms: message += 'Ticket : %s (block %s)\n'%( mccm['prepid'], mccm['block'] ) message += ' %smccms?prepid=%s \n\n' % (l_type.baseurl(), mccm['prepid']) message += '\n' to_who = [settings().get_value('service_account')] to_who.extend( map( lambda u : u['email'], udb.query(query="role==production_manager", page_num=-1))) com.sendMail(to_who, subject, message) return dumps({"results" : True, "message" : map( lambda m : m['prepid'], mccms)})
def __init__(self, db_name='', url=None, cache=False): host = os.environ['HOSTNAME'] if url is None: url = locator().dbLocation() #self.logger.log('I chose the url %s'%(url)) if not db_name: raise self.DatabaseNotFoundException(db_name) self.db_name = db_name self.cache = cache if self.db_name in ['campaigns', 'chained_campaigns']: ## force cache for those. self.cache = True try: self.db = Database(db_name, url=url) # self.db = Database(db_name, url='http://preptest.cern.ch:5984/') # self.db = Database(db_name) # for using private DB @localhost:5984 except ValueError as ex: raise self.DatabaseAccessError(db_name) self.allowed_operators = ['<=', '<', '>=', '>', '==', '~=']
def __init__(self, db_name='',url=None, cache=False): host = os.environ['HOSTNAME'] if url == None: url =locator().dbLocation() #self.logger.log('I chose the url %s'%(url)) if not db_name: raise self.DatabaseNotFoundException(db_name) self.db_name = db_name self.cache = cache if self.db_name in ['campaigns','chained_campaigns']: ## force cache for those. self.cache=True try: self.db = Database(db_name, url=url) # self.db = Database(db_name, url='http://preptest.cern.ch:5984/') # self.db = Database(db_name) # for using private DB @localhost:5984 except ValueError as ex: raise self.DatabaseAccessError(db_name) self.allowed_operators = ['<=', '<', '>=', '>', '==', '~=']
def __init__(self, test_id, test_script, timeout=None): self.script_for_test = test_script self.test_id = test_id self.test_err = os.path.abspath( self.script_for_test + '.err') self.test_out = os.path.abspath( self.script_for_test + '.out') locat = locator() if locat.isDev(): self.group = '/dev' else: self.group = '/prod' self.directory_for_test = os.path.dirname(self.script_for_test) self.ssh_exec = ssh_executor(self.directory_for_test, self.test_id) self.log_out = 'Not available' self.log_err = 'Not available' self.timeout = int(settings().get_value('batch_timeout')) self.queue = '8nh' if timeout: self.timeout = int(timeout / 60.) if (self.timeout / 3600. ) > 8.: self.queue = '1nd' ## fall back to the one day queue at worse
def GET(self, *args): """ Twiki display of mccm ticket for a given meeting date and /pwg optional """ date = args[0] pwgs=None if len(args)>1: pwgs=args[1].split(',') mdb = database('mccms') to_be_shown= ['prepid','notes','deadline','requests','chains','repetitions'] l_type=locator() mdocs= mdb.queries(['meeting==%s'% date]) if pwgs: text="---++ MccM Tickets for %s : %s \n"%( date, ', '.join(pwgs) ) for pwg in pwgs: mdocs_pwg = filter ( lambda m : m['pwg']==pwg, mdocs) text+="---+++ Tickets for %s \n" %pwg text+="[[%smccms?meeting=%s&pwg=%s][link to McM]]\n"%( l_type.baseurl(), date, pwg) for t in mdocs_pwg: text+=" * " for item in to_be_shown: if item in t: text+="%s "% t[item] text+='\n' else: text="---++ MccM Tickets for %s \n<br>"%( date ) text+="[[%smccms?meeting=%s][link to McM]]\n"%( l_type.baseurl(), date ) for t in mdocs: text+=" * " for item in to_be_shown: if item in t: text+="%s "% t[item] text+='\n' return text
def GET(self, *args): """ Ask for the increase of the role of the current user to the given pwg """ if not args: return dumps({"results" : False, "Message" : "not pwg provided"}) ## get who's there user_p = user_pack() udb = database(self.db_name) mcm_u = user( udb.get( user_p.get_username())) ## get the requested pwgs pwgs = args[0].split(',') #### set the pwgs to the current user current = mcm_u.get_attribute('pwg') current = list(set(current+pwgs)) mcm_u.set_attribute('pwg', current) mcm_u.update_history({'action':'ask role','step' : args[0]}) udb.update(mcm_u.json()) ## get the production managers emails production_managers = udb.queries(['role==production_manager']) ### send a notification to prod manager + service to_who = map(lambda u: u['email'], production_managers) + [settings().get_value('service_account')] to_who.append( user_p.get_email() ) com = communicator() l_type = locator() com.sendMail( to_who, 'Increase role for user %s' % mcm_u.get_attribute('fullname'), 'Please increase the role of the user %s to the next level.\n\n%susers?prepid=%s' % ( mcm_u.get_attribute('username'), l_type.baseurl(), mcm_u.get_attribute('username') )) return dumps({"results" : True, "message" : "user %s in for %s" %( mcm_u.get_attribute('username'), current)})
def build_location(sub_directory): l_type = locator() directory = l_type.workLocation() return os.path.abspath(directory) + '/' + sub_directory + '/'
def GET(self, *args): """ Goes through invalidation documents and display (/) and announce them to data ops (/announce) """ announce = False clear = False if len(args) != 0: if args[0] == 'announce': announce = True if args[0] == 'clear': clear = True idb = database('invalidations') r_to_be_rejected = map(invalidation, idb.queries(['status==new', 'type==request'])) ds_to_be_invalidated = map(invalidation, idb.queries(['status==new', 'type==dataset'])) ds_to_be_invalidated = filter(lambda ds : not 'None-' in ds.get_attribute('object'), ds_to_be_invalidated) l_type = locator() def add_prepid(invalid, html): if 'prepid' in invalid.json(): html += 'for request <a href=%srequests?prepid=%s> %s </a>' % ( l_type.baseurl(), invalid.get_attribute('prepid'), invalid.get_attribute('prepid')) def print_invalidations(invalids): a_text='' for invalid in invalids: a_text += ' %s\n' % (invalid.get_attribute('object')) return a_text html = '<html><body>\n' html += 'Requests to be aborted/rejected <br>\n' html += '<ul>\n' for r in r_to_be_rejected: html += '<li> <a href=%sreqmgr/view/details/%s> %s </a>' % (l_type.cmsweburl(), r.get_attribute('object'), r.get_attribute('object') ) add_prepid(r, html) html += '</li>\n' html += '</ul>\n' html += 'Datasets to be invalidated <br>\n' html += '<ul>\n' for ds in ds_to_be_invalidated: html += '<li> %s ' % ( ds.get_attribute('object') ) add_prepid(ds, html) html += '</li>\n' html += '</ul>\n' html += '<a href=%srestapi/invalidations/inspect/clear> Clear invalidation withouth announcing</a><br>\n'%( l_type.baseurl() ) html += '<a href=%srestapi/invalidations/inspect/announce> Announce invalidations</a><br>\n'%( l_type.baseurl() ) html += '<a href=%srestapi/invalidations/inspect> Back</a><br>\n'%( l_type.baseurl() ) html += '</html></body>\n' if announce and (len(ds_to_be_invalidated)!=0 or len(r_to_be_rejected)!=0): text = 'Dear Data Operation Team,\n\n' if len(r_to_be_rejected)!=0: text += 'please reject or abort the following requests:\n' text += print_invalidations(r_to_be_rejected) if len(ds_to_be_invalidated)!=0: text += '\nPlease invalidate the following datasets:\n' text += print_invalidations(ds_to_be_invalidated) text += '\nas a consequence of requests being reset.\n' com = communicator() to_who = [settings().get_value('service_account')] if l_type.isDev(): to_who.append( settings().get_value('hypernews_test')) else: to_who.append( settings().get_value('dataops_announce' )) try: elem = (r_to_be_rejected + ds_to_be_invalidated)[0] sender = elem.current_user_email except IndexError: sender = None com.sendMail(to_who, 'Request and Datasets to be Invalidated', text, sender) for to_announce in itertools.chain(r_to_be_rejected, ds_to_be_invalidated): to_announce.set_announced() idb.update(to_announce.json()) if clear and (len(ds_to_be_invalidated)!=0 or len(r_to_be_rejected)!=0): for to_announce in itertools.chain(r_to_be_rejected, ds_to_be_invalidated): to_announce.set_announced() idb.update(to_announce.json()) return html
def flow_to_next_step(self, input_dataset='', block_black_list=None, block_white_list=None, check_stats=True, reserve=False): if not block_white_list: block_white_list = [] if not block_black_list: block_black_list = [] self.logger.log('Flowing chained_request %s to next step...' % (self.get_attribute('_id'))) if not self.get_attribute('chain'): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'chained_request %s has got no root' % ( self.get_attribute('_id'))) # check on the approval of the chained request before all ## let it be flowing regardless #if self.get_attribute('approval') == 'none': # raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), # 'The approval of the chained request is none, and therefore flow cannot happen') #this operation requires to access all sorts of objects rdb = database('requests') cdb = database('campaigns') ccdb = database('chained_campaigns') crdb = database('chained_requests') fdb = database('flows') adb = database('actions') l_type=locator() current_step = len(self.get_attribute('chain'))-1 if reserve else self.get_attribute('step') current_id = self.get_attribute('chain')[current_step] next_step = current_step + 1 if not rdb.document_exists(current_id): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'the request %s does not exist' % current_id) current_request = request(rdb.get(current_id)) current_campaign = campaign(cdb.get(current_request.get_attribute('member_of_campaign'))) if not ccdb.document_exists(self.get_attribute('member_of_campaign')): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'the chain request %s is member of %s that does not exist' % ( self.get_attribute('_id'), self.get_attribute('member_of_campaign'))) mcm_cc = ccdb.get(self.get_attribute('member_of_campaign')) if next_step >= len(mcm_cc['campaigns']): if reserve: return False raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'chained_campaign %s does not allow any further flowing.' % ( self.get_attribute('member_of_campaign'))) if not reserve: ## is the current request in the proper approval allowed_request_approvals = ['submit'] if current_request.get_attribute('approval') not in allowed_request_approvals: raise self.NotApprovedException(current_request.get_attribute('prepid'), current_request.get_attribute('approval'), allowed_request_approvals) ## is the current request in the proper status allowed_request_statuses = ['submitted', 'done'] if current_request.get_attribute('status') not in allowed_request_statuses: raise self.NotInProperStateException(current_request.get_attribute('prepid'), current_request.get_attribute('status'), allowed_request_statuses) original_action_id = self.get_attribute('chain')[0] original_action_item = self.retrieve_original_action_item(adb, original_action_id) ## what is the campaign to go to next and with which flow (next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step] if not fdb.document_exists(flow_name): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The flow %s does not exist' % flow_name ) mcm_f = flow(fdb.get(flow_name)) if not 'sequences' in mcm_f.get_attribute('request_parameters'): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The flow %s does not contain sequences information.' % ( flow_name)) if not cdb.document_exists(next_campaign_id): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The next campaign %s does not exist' % next_campaign_id) next_campaign = campaign(cdb.get(next_campaign_id)) if len(next_campaign.get_attribute('sequences')) != len(mcm_f.get_attribute('request_parameters')['sequences']): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'the sequences changes in flow %s are not consistent with the next campaign %s' % ( flow_name, next_campaign_id)) if next_campaign.get_attribute('energy') != current_campaign.get_attribute('energy'): raise self.EnergyInconsistentException(next_campaign.get_attribute('prepid')) if next_campaign.get_attribute('type') == 'MCReproc' and (not 'time_event' in mcm_f.get_attribute('request_parameters')): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'the flow is getting into a MCReproc campaign but not time per event is specified') if next_campaign.get_attribute('type') == 'MCReproc' and (not 'size_event' in mcm_f.get_attribute('request_parameters')): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'the flow is getting into a MCReproc campaign but not size per event is specified') ## check that it is allowed to flow allowed_flow_approvals = ['flow', 'submit'] ###### cascade of checks """ if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The flow (%s) is not in proper approval state (%s)'%( mcm_f.get_attribute('prepid'), mcm_f.get_attribute('approval'))) if not reserve and not self.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The chained request (%s) is not in the proper approval state (%s)'% ( self.get_attribute('_id'), self.get_attribute('approval'))) """ if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals: if not self.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing' % ( mcm_f.get_attribute('approval'), self.get_attribute('approval'))) if next_campaign.get_attribute('status') == 'stopped': raise self.CampaignStoppedException(next_campaign_id) #what is going to be the required number of events for the next request #update the stats to its best if not reserve: current_request.get_stats() next_total_events=current_request.get_attribute('completed_events') notify_on_fail=True ## to be tuned according to the specific cases if current_request.get_attribute('completed_events') <= 0: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The number of events completed is negative or null') else: allowed_request_statuses = ['done'] ## determine if this is a root -> non-root transition to potentially apply staged number at_a_transition=(current_campaign.get_attribute('root') != 1 and next_campaign.get_attribute('root') == 1) if ('staged' in original_action_item or 'threshold' in original_action_item) and at_a_transition: allowed_request_statuses.append('submitted') ##check status if not current_request.get_attribute('status') in allowed_request_statuses: raise self.NotInProperStateException(current_request.get_attribute('prepid'), current_request.get_attribute('status'), allowed_request_statuses) ##special check at transition that the statistics is good enough if at_a_transition: # at a root -> non-root transition only does the staged/threshold functions ! if 'staged' in original_action_item: next_total_events = int(original_action_item['staged']) if 'threshold' in original_action_item: next_total_events = int(current_request.get_attribute('total_events') * float(original_action_item['threshold'] / 100.)) completed_events_to_pass = next_total_events else: ## get the original expected events and allow a margin of 5% less statistics completed_events_to_pass = int(current_request.get_attribute('total_events') * 0.95) if check_stats and (current_request.get_attribute('completed_events') < completed_events_to_pass): if notify_on_fail: current_request.notify('Flowing for %s: not enough statistics'%( current_request.get_attribute('prepid')), 'For this request, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s \n\n %srequests?prepid=%s'%( current_request.get_attribute('completed_events'), completed_events_to_pass, l_type.baseurl(), current_request.get_attribute('prepid'))) raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The number of events completed (%s) is not enough for the requirement (%s)'%(current_request.get_attribute('completed_events'), completed_events_to_pass)) ## select what is to happened : [create, patch, use] next_id = None approach = None next_request = None if next_step != len(self.get_attribute('chain')): #not at the end next_id = self.get_attribute('chain')[next_step] if not rdb.document_exists(next_id): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The next request (%s) according to the step (%s) does not exist' % ( next_id, next_step)) next_request = request(rdb.get(next_id)) if next_request.get_attribute('status') == 'new': #most likely a rewind + resub approach = 'patch' else: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % ( next_id, next_step, next_request.get_attribute('status'))) else: ## look in *other* chained campaigns whether you can suck in an existing request ## look up all chained requests that start from the same root request ## remove <pwg>-chain_ and the -serial number, replacing _ with a . toMatch = '.'.join(self.get_attribute('prepid').split('_')[1:][0:next_step + 1]).split('-')[0] ## make sure they get ordered by prepid related_crs = sorted(crdb.queries(['root_request==%s' % original_action_id]), key=lambda cr : cr['prepid']) vetoed_last=[] for existing_cr in related_crs: ## exclude itself if existing_cr['prepid']==self.get_attribute('prepid'): continue ## prevent from using a request from within the same exact chained_campaigns if existing_cr['member_of_campaign'] == self.get_attribute('member_of_campaign'): mcm_cr = chained_request(crdb.get(existing_cr['prepid'])) if len(mcm_cr.get_attribute('chain')) > next_step: ## one existing request in the very same chained campaign has already something used, make sure it is not going to be used vetoed_last.append( mcm_cr.get_attribute('chain')[next_step]) continue else: continue for existing_cr in related_crs: ## exclude itself if existing_cr['prepid']==self.get_attribute('prepid'): continue ## prevent from using a request from within the same exact chained_campaigns if existing_cr['member_of_campaign'] == self.get_attribute('member_of_campaign'): continue truncated = '.'.join(existing_cr['prepid'].split('_')[1:][0:next_step + 1]).split('-')[0] self.logger.error('to match : %s , this one %s' % ( toMatch, truncated )) if truncated == toMatch: #we found a chained request that starts with all same steps mcm_cr = chained_request(crdb.get(existing_cr['prepid'])) if len(mcm_cr.get_attribute('chain')) <= next_step: #found one, but it has not enough content either continue if mcm_cr.get_attribute('chain')[next_step] in vetoed_last: continue next_id = mcm_cr.get_attribute('chain')[next_step] break if next_id: approach = 'use' else: approach = 'create' if approach == 'create': from rest_api.RequestPrepId import RequestPrepId next_id = RequestPrepId().next_prepid(current_request.get_attribute('pwg'), next_campaign_id) next_request = request(rdb.get(next_id)) request.transfer( current_request, next_request) self.request_join(next_request) elif approach == 'use': ## there exists a request in another chained campaign that can be re-used here. # take this one. advance and go on next_request = request(rdb.get(next_id)) if not reserve: self.set_attribute('step', next_step) self.set_attribute('last_status', next_request.get_attribute('status')) self.update_history({'action': 'flow', 'step': str(next_step)}) self.set_attribute('status', 'processing') if not self.get_attribute("prepid") in next_request.get_attribute("member_of_chain"): ## register the chain to the next request self.request_join(next_request) saved = rdb.update(next_request.json()) if not saved: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'Unable to save %s with updated member_of_chains' % next_id) return True elif approach == 'patch': ## there exists already a request in the chain (step!=last) and it is usable for the next stage next_request = request( next_campaign.add_request( rdb.get(next_id))) ### shouldn't this be added ? #transfer( current_request, next_request) else: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'Unrecognized approach %s' % approach ) #current_request -> next_request #current_campaign -> next_campaign ##determine whether we have an input dataset for the next request if len(current_request.get_attribute('reqmgr_name')): last_wma = current_request.get_attribute('reqmgr_name')[-1] if 'content' in last_wma and 'pdmv_dataset_name' in last_wma['content']: input_dataset = last_wma['content']['pdmv_dataset_name'] else: statsDB = database('stats', url='http://cms-pdmv-stats.cern.ch:5984/') if statsDB.document_exists(last_wma['name']): latestStatus = statsDB.get(last_wma['name']) input_dataset = latestStatus['pdmv_dataset_name'] if input_dataset: next_request.set_attribute('input_filename', input_dataset) ## set blocks restriction if any if block_black_list: next_request.set_attribute('block_black_list', block_black_list) if block_white_list: next_request.set_attribute('block_white_list', block_white_list) ## register the flow to the request next_request.set_attribute('flown_with', flow_name) ##assemble the campaign+flow => request request.put_together(next_campaign, mcm_f, next_request) if not reserve: #already taking stage and threshold into account next_request.set_attribute('total_events', next_total_events) next_request.update_history({'action': 'flow', 'step': self.get_attribute('prepid')}) request_saved = rdb.save(next_request.json()) if not request_saved: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'Could not save the new request %s' % ( next_request.get_attribute('prepid'))) ## inspect priority self.set_priority(original_action_item['block_number']) if not reserve: # sync last status self.set_attribute('last_status', next_request.get_attribute('status')) # we can only be processing at this point self.set_attribute('status', 'processing') # set to next step self.set_attribute('step', next_step) if not reserve: notification_subject = 'Flow for request %s in %s' % (current_request.get_attribute('prepid'), next_campaign_id) notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % ( current_request.get_attribute('prepid'), self.get_attribute('member_of_campaign'), next_campaign_id, flow_name, next_request.get_attribute('prepid'), self.get_attribute('prepid'), next_request.get_attribute('input_filename'), l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), next_request.get_attribute('prepid') ) current_request.notify(notification_subject, notification_text) else: notification_subject = 'Reservation of request {0}'.format(next_request.get_attribute('prepid')) notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format( next_request.get_attribute('prepid'), self.get_attribute('prepid'), next_campaign_id, l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), next_request.get_attribute('prepid'), ) next_request.notify(notification_subject, notification_text) return True
def get_command(self, mcm_r, batchNumber, to_execute=False): command = '' ##JR in order to inject into the testbed instead of the production machine l_type = locator() if to_execute: # set path to proxy certificate command += 'cd %s\n' % ( l_type.workLocation() ) command += 'export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOSTNAME/voms_proxy.cert\n' command += mcm_r.make_release() #command += 'eval `scram runtime -sh`\n' command += 'source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh\n' wmagent_type = mcm_r.get_wmagent_type() command += 'export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}\n' command += 'wmcontrol.py --release %s' % (mcm_r.get_attribute('cmssw_release')) command += ' --arch %s' % (mcm_r.get_scram_arch()) command += ' --conditions %s' % (mcm_r.get_attribute('sequences')[0]['conditions']) command += ' --version %s' % (mcm_r.get_attribute('version')) if mcm_r.get_attribute('priority') >= 1: command += ' --priority %s' % (mcm_r.get_attribute("priority")) command += ' --time-event %s' % (mcm_r.get_attribute('time_event')) command += ' --size-event %s' % (mcm_r.get_attribute('size_event')) command += ' --memory %s' % (mcm_r.get_attribute('memory')) ##that type has disappeared if wmagent_type == 'LHEStepZero': command += ' --request-type MonteCarlo' else: command += ' --request-type %s' % wmagent_type config_id_from_hashkey = [] ## check on the presence of docId ?... if len(mcm_r.get_attribute('config_id')): command += ' --step1-docID %s' % ( mcm_r.get_attribute('config_id')[0]) else: ## get the config ID from hash instead of cfg.py hash_ids = database('configs') hash_id = mcm_r.configuration_identifier(0) if hash_ids.document_exists(hash_id): hash_doc = hash_ids.get(hash_id) config_cache_id = hash_doc['docid'] command += ' --step1-docID %s' % config_cache_id config_id_from_hashkey = [config_cache_id] else: command += ' --step1-cfg %s_1_cfg.py' % (mcm_r.get_attribute('prepid')) command += ' --request-id %s' % (mcm_r.get_attribute('prepid')) if l_type.isDev(): command += ' --wmtest ' command += ' --user pdmvserv ' command += ' --group ppd ' command += ' --batch %s' % batchNumber processString = mcm_r.get_attribute('process_string') processingString = mcm_r.get_processing_string(0) max_forward_eff = mcm_r.get_forward_efficiency() events_per_lumi = settings().get_value('events_per_lumi') if wmagent_type == 'MonteCarlo': # calculate eff dev command += ' --filter-eff %s' % ( mcm_r.get_efficiency() ) command += ' --events-per-lumi %s' % ( events_per_lumi / max_forward_eff ) command += ' --number-events %s' % (mcm_r.get_attribute('total_events')) command += ' --primary-dataset %s' % (mcm_r.get_attribute('dataset_name')) elif wmagent_type == 'MonteCarloFromGEN': # calculate eff dev command += ' --filter-eff %s' % ( mcm_r.get_efficiency() ) command += ' --input-ds %s' % (mcm_r.get_attribute('input_dataset')) command += ' --primary-dataset %s' % (mcm_r.get_attribute('dataset_name')) if mcm_r.get_attribute('block_white_list'): command += ' --blocks "' + ','.join(mcm_r.get_attribute('block_white_list')) + '"' if mcm_r.get_attribute('block_black_list'): command += ' --blocks_black "' + ','.join(mcm_r.get_attribute('block_black_list')) + '"' command += ' --number-events %s' % (mcm_r.get_attribute('total_events')) elif wmagent_type == 'LHEStepZero': command += ' --number-events %s' % (mcm_r.get_attribute('total_events')) command += ' --events-per-lumi %s' % ( events_per_lumi / max_forward_eff ) command += ' --primary-dataset %s' % (mcm_r.get_attribute('dataset_name')) command += ' --filter-eff %s' % ( mcm_r.get_efficiency() ) if mcm_r.get_attribute('mcdb_id') <= 0: numberOfEventsPerJob = mcm_r.numberOfEventsPerJob() if not numberOfEventsPerJob: raise ValueError('Number of events per job could not be retrieved') command += ' --events-per-job %s' % numberOfEventsPerJob else: command += ' --lhe ' if not processString: processString = '' elif wmagent_type == 'ReDigi': command += ' --input-ds %s' % (mcm_r.get_attribute('input_dataset')) command += ' --primary-dataset %s' % (mcm_r.get_attribute('dataset_name')) ## if PU dataset name is defined : add it if mcm_r.get_attribute('pileup_dataset_name') and mcm_r.get_attribute('pileup_dataset_name').strip(): command += ' --pileup-ds ' + mcm_r.get_attribute('pileup_dataset_name') ## provide the total number of events requested: by default it is the amount in the input dataset. # and wmcontrol / wma should understand that we want partial statistics if that number is lower than expected command += ' --number-events %s' % (mcm_r.get_attribute('total_events')) # temp ev cont holder eventcontentlist = mcm_r.get_first_output() keeps = mcm_r.get_attribute('keep_output') if not keeps[-1]: raise ValueError('Is not set to save the output of last task') for (i, content) in enumerate(eventcontentlist): if i < 2: #trick to NOT add for step3 and more: if keeps[i]: command += ' --keep-step' + str(i + 1) + ' True' if i > 0: processingString = mcm_r.get_processing_string(i) if len(mcm_r.get_attribute('config_id')): command += ' --step%d-docID %s' % (i + 1, mcm_r.get_attribute('config_id')[i]) else: hash_id = mcm_r.configuration_identifier(i) if hash_ids.document_exists(hash_id): hash_doc = hash_ids.get(hash_id) config_cache_id = hash_doc['docid'] command += ' --step%d-docID %s' % (i + 1, config_cache_id) config_id_from_hashkey.append(config_cache_id) else: command += ' --step%d-cfg %s_%d_cfg.py' % ( i + 1, mcm_r.get_attribute('prepid'), i + 1) # set the output of if i < len(eventcontentlist) - 1: command += ' --step' + str(i + 1) + '-output ' + content if mcm_r.get_attribute('block_white_list'): command += ' --blocks "' + ','.join(mcm_r.get_attribute('block_white_list')) + '"' if mcm_r.get_attribute('block_black_list'): command += ' --blocks_black "' + ','.join(mcm_r.get_attribute('block_black_list')) + '"' if processString: command += ' --process-string ' + processString command += ' --processing-string ' + processingString command += '|| exit $? ;' command += '\n' if len(config_id_from_hashkey): mcm_r.set_attribute('config_id', config_id_from_hashkey) return command
def announce(self,notes="",user=""): if self.get_attribute('status')!='new': return False if len(self.get_attribute('requests'))==0: return False current_notes=self.get_attribute('notes') if current_notes: current_notes+='\n' if notes: current_notes+=notes self.set_attribute('notes',current_notes) total_events=0 content = self.get_attribute('requests') total_requests=len(content) rdb =database('requests') ## prepare the announcing message (campaign,batchNumber)=self.get_attribute('prepid').split('_')[-1].split('-') subject=self.get_subject() message="" message+="Dear Data Operation Team,\n\n" message+="may you please consider the following batch number %d of %s requests for the campaign %s:\n\n"%(int(batchNumber),total_requests, campaign) for r in content: ##loose binding of the prepid to the request name, might change later on if 'pdmv_prep_id' in r['content']: pid=r['content']['pdmv_prep_id'] else: pid=r['name'].split('_')[1] mcm_r = rdb.get(pid) total_events+=mcm_r['total_events'] message+=" * %s (%s) -> %s\n"%(pid, mcm_r['dataset_name'], r['name']) message+="\n" message+="For a total of %s events\n\n"%( re.sub("(\d)(?=(\d{3})+(?!\d))", r"\1,", "%d" % total_events )) if self.get_attribute('extension'): message += "This batch is for an extension : {0}\n".format(self.get_attribute('extension')) if self.get_attribute('version'): message += "This batch is a resubmission : v{0}\n".format(self.get_attribute('version') + 1) message+="Link to the batch:\n" l_type = locator() message+='%s/batches?prepid=%s \n\n'%(l_type.baseurl(), self.get_attribute('prepid')) if current_notes: message+="Additional comments for this batch:\n"+current_notes+'\n' if self.get_attribute('process_string'): message+='Please use "%s" in the dataset name.\n' % self.get_attribute('process_string') self.logger.log('Message send for batch %s'%(self.get_attribute('prepid'))) self.get_current_user_role_level() to_who = [settings().get_value('service_account')] if l_type.isDev(): to_who.append( settings().get_value('hypernews_test')) else: to_who.append( settings().get_value('dataops_announce' )) #sender=None #if self.current_user_level != 3: # auth = authenticator() # sender = auth.get_random_product_manager_email() #current_message_id = self.get_attribute('message_id') returned_id = self.notify(subject, message, who=to_who)#, #sender=sender) self.set_attribute('message_id', returned_id) self.reload() ## toggle the status ### only when we are sure it functions self.set_status() self.set_status() return True
def __init__(self): self.db_name = "invalidations" self.com = communicator() self.l_type = locator()
def announce(self,notes="",user=""): if self.get_attribute('status')!='new': return False if len(self.get_attribute('requests'))==0: return False current_notes=self.get_attribute('notes') if current_notes: current_notes+='\n' if notes: current_notes+=notes self.set_attribute('notes',current_notes) total_events=0 content = self.get_attribute('requests') total_requests=len(content) rdb =database('requests') ## prepare the announcing message (campaign,batchNumber)=self.get_attribute('prepid').split('_')[-1].split('-') subject=self.get_subject() request_messages={} for r in content: ##loose binding of the prepid to the request name, might change later on if 'pdmv_prep_id' in r['content']: pid=r['content']['pdmv_prep_id'] else: pid=r['name'].split('_')[1] mcm_r = rdb.get(pid) total_events+=mcm_r['total_events'] c = mcm_r['member_of_campaign'] if not c in request_messages: request_messages[c]="" request_messages[c]+=" * %s (%s) -> %s\n"%(pid, mcm_r['dataset_name'], r['name']) campaigns = sorted(request_messages.keys()) message="" message+="Dear Data Operation Team,\n\n" message+="may you please consider the following batch number %d of %s requests for the campaign%s %s:\n\n"%( int(batchNumber),total_requests, "s" if len(campaigns)>1 else "", ','.join(campaigns)) for c in campaigns: message+=request_messages[c] message+="\n" message+="For a total of %s events\n\n"%( re.sub("(\d)(?=(\d{3})+(?!\d))", r"\1,", "%d" % total_events )) if self.get_attribute('extension'): message += "This batch is for an extension : {0}\n".format(self.get_attribute('extension')) if self.get_attribute('version'): message += "This batch is a resubmission : v{0}\n".format(self.get_attribute('version') + 1) message+="Link to the batch:\n" l_type = locator() message+='%s/batches?prepid=%s \n\n'%(l_type.baseurl(), self.get_attribute('prepid')) if current_notes: message+="Additional comments for this batch:\n"+current_notes+'\n' self.logger.log('Message send for batch %s'%(self.get_attribute('prepid'))) self.get_current_user_role_level() to_who = [settings().get_value('service_account')] if l_type.isDev(): to_who.append( settings().get_value('hypernews_test')) else: to_who.append( settings().get_value('dataops_announce' )) #sender=None #if self.current_user_level != 3: # auth = authenticator() # sender = auth.get_random_product_manager_email() #current_message_id = self.get_attribute('message_id') returned_id = self.notify(subject, message, who=to_who)#, #sender=sender) self.set_attribute('message_id', returned_id) self.reload() ## toggle the status ### only when we are sure it functions self.set_status() self.set_status() return True
def add_request(self, data=None): if not data: data = {} self.logger.log('Adding new request to chained_request %s' % self.get_attribute('_id')) # import prep-id generator try: from rest_api.RequestPrepId import RequestPrepId except ImportError as ex: self.logger.error('Could not import prep-id generator class. Reason: %s' % ex, level='critical') return {} try: req = request(json_input=data) except Exception as ex: self.logger.error('Could not build request object. Reason: %s' % ex) return {} #chain_specific = ['threshold', 'block_number', 'staged'] ## JR remove from schema ##this was removed as part of cleaning things up #if len(self.get_attribute('request_parameters')) > 0: # changes = self.get_attribute('request_parameters') # for key in changes: # if key not in chain_specific: # req.set_attribute(key, changes[key]) # get the chain and inherit #req.set_attribute("generators", self.get_attribute("generators")) #req.set_attribute("total_events", self.get_attribute("total_events")) ## this was taken earlier, with staged number in consideration req.set_attribute("dataset_name", self.get_attribute("dataset_name")) req.set_attribute("pwg", self.get_attribute("pwg")) #JR removed from schema req.set_attribute("priority", self.get_attribute("priority") ) #JR clear the fragment in flowing: always req.set_attribute('name_of_fragment', '') req.set_attribute('cvs_tag', '') req.set_attribute('fragment_tag', '') req.set_attribute('fragment', '') req.set_attribute('history', []) req.set_attribute('reqmgr_name', []) #JR #clean the mcdbid in the flown request #if req.get_attribute('mcdbid')>=0: # req.set_attribute('mcdbid',0) # get the new prepid and append it to the chain prepid = RequestPrepId().generate_prepid(req.get_attribute("pwg"), req.get_attribute('member_of_campaign'))["prepid"] chain = self.get_attribute("chain") if not chain or chain is None: chain = [] flag = False for pid in chain: #if req.get_attribute('member_of_campaign') in pid: if pid.split('-')[1] == req.get_attribute('member_of_campaign'): flag = True break if not flag: chain.append(prepid) self.set_attribute("chain", chain) #self.logger.log('Adding %s to the chain %s'%(prepid,chain)) else: raise self.CampaignAlreadyInChainException(req.get_attribute('member_of_campaign')) req.set_attribute('_id', prepid) req.set_attribute('prepid', prepid) ## JR: add what the request is member of N.B: that breaks down if a digi-reco request has to be member of two chains (R1,R4) req.set_attribute('member_of_chain', [self.get_attribute('_id')]) ## reset the status and approval chain req.set_status(0) req.approve(0) ### mode the approval of the new request to the approval of the chained request if not req.is_root: self.logger.log('The newly created request %s is not root, the chained request has approval %s' % ( req.get_attribute('prepid'), self.get_attribute('approval') )) #if self.get_attribute('approval') == 'approve': #toggle the request approval to 'approved'? if self.get_attribute('approval') == 'submit': req.set_status(to_status='approved') req.approve(to_approval='submit') # update history req.update_history({'action': 'join chain', 'step': self.get_attribute('_id')}) self.update_history({'action': 'add request', 'step': req.get_attribute('_id')}) loc = locator() req.notify("Request {0} joined chain".format(req.get_attribute('prepid')), "Request {0} has successfuly joined chain {1}\n\tRequest: {2}".format(req.get_attribute('prepid'), self.get_attribute('_id'), "/".join([loc.baseurl(), "requests?prepid={0}".format(req.get_attribute('prepid'))]))) # set request approval status to new #req.approve(0) return req.json()
def index(self, **args): db_name='requests' query='' query_list=[] page=0 manual_keys=['db_name','query','page'] if 'db_name' in args: db_name=args['db_name'] args.pop('db_name') if 'query' in args: query=args['query'] args.pop('query') if 'page' in args: page=args['page'] args.pop('page') # retrieve the _design/object document odb=database(db_name) design = odb.get('_design/%s'%(db_name)) allowed_key_search = design['views'].keys() vetoed_keys = [] for (view,f) in design['views'].items(): if 'for(' in f['map'] or 'for (' in f['map']: vetoed_keys.append( view ) allowed_key_search.sort() multiple_view=[] #### ## to switch on/off the view creation on the fly simple_search=(not locator().isDev()) simple_search=False #### for key in filter (lambda s : '-' not in s, allowed_key_search): if key in args: if key in vetoed_keys or simple_search: query_list.append('%s==%s'%(key,args[key])) else: if args[key].isdigit(): multiple_view.append( (key, args[key]) ) else: multiple_view.append( (key, '"'+args[key]+'"') ) args.pop(key) if len(multiple_view)>1: multiple_search = '-'.join( map( lambda p:p[0], multiple_view)) ## faster query with multiple keys if not multiple_search in allowed_key_search: ## try harder to find it really_not_there=True m_set = set( map( lambda p:p[0], multiple_view) ) for key in filter (lambda s : '-' in s, allowed_key_search): ## parse all composite view if set(key.split('-')) == m_set: #we found one that has the same search in absolute, just the order is different # then re-order multiple_view so as to map to the existing view new_multiple_view = [] for sv in key.split('-'): new_multiple_view.append( filter( lambda e : e[0]==sv, multiple_view) [0] ) multiple_view = new_multiple_view multiple_search = '-'.join( map( lambda p:p[0], multiple_view)) really_not_there=False break if really_not_there: #tempatively add the view to the design new_func = "function(doc){ emit([%s], doc._id);}"%( ','.join(map( lambda k: "doc.%s"%(k), map( lambda p:p[0], multiple_view) ))) design['views'] [ multiple_search ] = { "map" : new_func } saved = odb.update( design ) ##### NOTE #### ## the query that will follow will be super slow because the view needs to be re-build m_query = '%s==[%s]'%(multiple_search, ','.join( map( lambda p:p[1], multiple_view)) ) query_list.append( m_query ) #query_list =[] elif len(multiple_view)==1: m_query = '%s==%s'%( multiple_view[0][0], multiple_view[0][1]) query_list.append( m_query ) #revert to simple query for one query only if len(query_list)==1: query=query_list[0] query_list=[] if len(args): ## check whether the key is actually a member of the object in db and put back the view in the odb design return simplejson.dumps(args) #return simplejson.dumps(design['views'].keys()) return self.search(db_name, query, page, query_list)
def internal_run(self): if not self.lock.acquire(blocking=False): return False try: req = request(self.request_db.get(self.prepid)) additional_config_ids = {} cfgs_to_upload = {} l_type = locator() dev='' wmtest='' if l_type.isDev(): wmtest = '--wmtest' if req.get_attribute('config_id'): # we already have configuration ids saved in our request return True for i in range(len(req.get_attribute('sequences'))): hash_id = req.configuration_identifier(i) if self.config_db.document_exists(hash_id): # cached in db additional_config_ids[i] = self.config_db.get(hash_id)['docid'] else: # has to be setup and uploaded to config cache cfgs_to_upload[i] = "{0}{1}_{2}_cfg.py".format(req.get_attribute('prepid'), dev, i+1) if cfgs_to_upload: with installer(self.prepid, care_on_existing=False) as directory_manager: command = self.prepare_command([cfgs_to_upload[i] for i in sorted(cfgs_to_upload)], directory_manager.location(), req, wmtest) _, stdout, stderr = self.ssh_executor.execute(command) if not stdout and not stderr: self.logger.error('SSH error for request {0}. Could not retrieve outputs.'.format(self.prepid)) self.logger.inject('SSH error for request {0}. Could not retrieve outputs.'.format(self.prepid), level='error', handler=self.prepid) req.test_failure('SSH error for request {0}. Could not retrieve outputs.'.format(self.prepid), what='Configuration upload') return False output = stdout.read() error = stderr.read() if error and not output: # money on the table that it will break self.logger.error('Error in wmupload: {0}'.format(error)) req.test_failure('Error in wmupload: {0}'.format(error), what='Configuration upload') return False cfgs_uploaded = [l for l in output.split("\n") if 'DocID:' in l] if len(cfgs_to_upload) != len(cfgs_uploaded): self.logger.error('Problem with uploading the configurations. To upload: {0}, received doc_ids: {1}\nOutput:\n{2}\nError:\n{3}'.format(cfgs_to_upload, cfgs_uploaded, output, error)) self.logger.inject('Problem with uploading the configurations. To upload: {0}, received doc_ids: {1}\nOutput:\n{2}\nError:\n{3}'.format(cfgs_to_upload, cfgs_uploaded, output, error), level='error', handler=self.prepid) req.test_failure('Problem with uploading the configurations. To upload: {0}, received doc_ids: {1}\nOutput:\n{2}\nError:\n{3}'.format(cfgs_to_upload, cfgs_uploaded, output, error), what='Configuration upload') return False for i, line in zip(sorted(cfgs_to_upload), cfgs_uploaded): # filling the config ids for request and config database with uploaded configurations docid = line.split()[-1] additional_config_ids[i] = docid saved = self.config_db.save({"_id": req.configuration_identifier(i), "docid": docid, "prepid": self.prepid, "unique_string": req.unique_string(i)}) if not saved: self.logger.inject('Could not save the configuration {0}'.format( req.configuration_identifier(i) ), level='warning', handler=self.prepid) self.logger.inject("Full upload result: {0}".format(output), handler=self.prepid) sorted_additional_config_ids = [additional_config_ids[i] for i in additional_config_ids] self.logger.inject("New configs for request {0} : {1}".format(self.prepid, sorted_additional_config_ids), handler=self.prepid) req.set_attribute('config_id', sorted_additional_config_ids) self.request_db.save(req.json()) return True finally: self.lock.release()
def announce(self, notes="", user=""): if self.get_attribute('status') != 'new': return False if len(self.get_attribute('requests')) == 0: return False current_notes = self.get_attribute('notes') if current_notes: current_notes += '\n' if notes: current_notes += notes self.set_attribute('notes', current_notes) total_events = 0 content = self.get_attribute('requests') total_requests = len(content) rdb = database('requests') # prepare the announcing message (campaign, batchNumber) = self.get_attribute('prepid').split('_')[-1].split('-') subject = self.get_subject() request_messages = {} for r in content: # loose binding of the prepid to the request name, might change later on if 'pdmv_prep_id' in r['content']: pid = r['content']['pdmv_prep_id'] else: pid = r['name'].split('_')[1] mcm_r = rdb.get(pid) total_events += mcm_r['total_events'] c = mcm_r['member_of_campaign'] if c not in request_messages: request_messages[c] = "" request_messages[c] += " * %s (%s) -> %s\n" % ( pid, mcm_r['dataset_name'], r['name']) campaigns = sorted(request_messages.keys()) message = "" message += "Dear Data Operation Team,\n\n" message += "may you please consider the following batch number %d of %s requests for the campaign%s %s:\n\n" % ( int(batchNumber), total_requests, "s" if len(campaigns) > 1 else "", ','.join(campaigns)) for c in campaigns: message += request_messages[c] message += "\n" message += "For a total of %s events\n\n" % (re.sub( "(\d)(?=(\d{3})+(?!\d))", r"\1,", "%d" % total_events)) if self.get_attribute('extension'): message += "This batch is for an extension : {0}\n".format( self.get_attribute('extension')) if self.get_attribute('version'): message += "This batch is a resubmission : v{0}\n".format( self.get_attribute('version') + 1) message += "Link to the batch:\n" l_type = locator() message += '%s/batches?prepid=%s \n\n' % (l_type.baseurl(), self.get_attribute('prepid')) if current_notes: message += "Additional comments for this batch:\n" + current_notes + '\n' self.logger.info('Message send for batch %s' % (self.get_attribute('prepid'))) self.get_current_user_role_level() to_who = [settings.get_value('service_account')] if l_type.isDev(): to_who.append(settings.get_value('hypernews_test')) else: to_who.append(settings.get_value('dataops_announce')) notification(subject, message, [], group=notification.BATCHES, target_role='production_manager', action_objects=[self.get_attribute('prepid')], object_type='batches', base_object=self) returned_id = self.notify(subject, message, who=to_who) self.set_attribute('message_id', returned_id) self.reload() # toggle the status # only when we are sure it functions self.set_status() self.set_status() return True
def GET(self, *args): """ Goes through invalidation documents and display (/) and announce them to data ops (/announce) """ announce = False clear = False if len(args) != 0: if args[0] == 'announce': announce = True if args[0] == 'clear': clear = True idb = database('invalidations') r_to_be_rejected = map(invalidation, idb.queries(['status==new', 'type==request'])) ds_to_be_invalidated = map( invalidation, idb.queries(['status==new', 'type==dataset'])) ds_to_be_invalidated = filter( lambda ds: not 'None-' in ds.get_attribute('object'), ds_to_be_invalidated) l_type = locator() def add_prepid(invalid, html): if 'prepid' in invalid.json(): html += 'for request <a href=%srequests?prepid=%s> %s </a>' % ( l_type.baseurl(), invalid.get_attribute('prepid'), invalid.get_attribute('prepid')) def print_invalidations(invalids): a_text = '' for invalid in invalids: a_text += ' %s\n' % (invalid.get_attribute('object')) return a_text html = '<html><body>\n' html += 'Requests to be aborted/rejected <br>\n' html += '<ul>\n' for r in r_to_be_rejected: html += '<li> <a href=%sreqmgr/view/details/%s> %s </a>' % ( l_type.cmsweburl(), r.get_attribute('object'), r.get_attribute('object')) add_prepid(r, html) html += '</li>\n' html += '</ul>\n' html += 'Datasets to be invalidated <br>\n' html += '<ul>\n' for ds in ds_to_be_invalidated: html += '<li> %s ' % (ds.get_attribute('object')) add_prepid(ds, html) html += '</li>\n' html += '</ul>\n' html += '<a href=%srestapi/invalidations/inspect/clear> Clear invalidation withouth announcing</a><br>\n' % ( l_type.baseurl()) html += '<a href=%srestapi/invalidations/inspect/announce> Announce invalidations</a><br>\n' % ( l_type.baseurl()) html += '<a href=%srestapi/invalidations/inspect> Back</a><br>\n' % ( l_type.baseurl()) html += '</html></body>\n' if announce and (len(ds_to_be_invalidated) != 0 or len(r_to_be_rejected) != 0): text = 'Dear Data Operation Team,\n\n' if len(r_to_be_rejected) != 0: text += 'please reject or abort the following requests:\n' text += print_invalidations(r_to_be_rejected) if len(ds_to_be_invalidated) != 0: text += '\nPlease invalidate the following datasets:\n' text += print_invalidations(ds_to_be_invalidated) text += '\nas a consequence of requests being reset.\n' com = communicator() to_who = [settings().get_value('service_account')] if l_type.isDev(): to_who.append(settings().get_value('hypernews_test')) else: to_who.append(settings().get_value('dataops_announce')) try: elem = (r_to_be_rejected + ds_to_be_invalidated)[0] sender = elem.current_user_email except IndexError: sender = None com.sendMail(to_who, 'Request and Datasets to be Invalidated', text, sender) for to_announce in itertools.chain(r_to_be_rejected, ds_to_be_invalidated): to_announce.set_announced() idb.update(to_announce.json()) if clear and (len(ds_to_be_invalidated) != 0 or len(r_to_be_rejected) != 0): for to_announce in itertools.chain(r_to_be_rejected, ds_to_be_invalidated): to_announce.set_announced() idb.update(to_announce.json()) return html
def get_command(self, mcm_r, batchNumber, to_execute=False): command = '' ##JR in order to inject into the testbed instead of the production machine l_type = locator() if to_execute: # set path to proxy certificate command += 'cd %s\n' % (l_type.workLocation()) command += 'export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOST/voms_proxy.cert\n' command += mcm_r.make_release() #command += 'eval `scram runtime -sh`\n' command += 'source /afs/cern.ch/cms/PPD/PdmV/tools/wmclient/current/etc/wmclient.sh\n' wmagent_type = mcm_r.get_wmagent_type() command += 'export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}\n' command += 'wmcontrol.py --release %s' % ( mcm_r.get_attribute('cmssw_release')) command += ' --arch %s' % (mcm_r.get_scram_arch()) command += ' --conditions %s' % ( mcm_r.get_attribute('sequences')[0]['conditions']) command += ' --version %s' % (mcm_r.get_attribute('version')) if mcm_r.get_attribute('priority') >= 1: command += ' --priority %s' % (mcm_r.get_attribute("priority")) command += ' --time-event %s' % (mcm_r.get_attribute('time_event')) command += ' --size-event %s' % (mcm_r.get_attribute('size_event')) command += ' --memory %s' % (mcm_r.get_attribute('memory')) ##that type has disappeared if wmagent_type == 'LHEStepZero': command += ' --request-type MonteCarlo' else: command += ' --request-type %s' % wmagent_type config_id_from_hashkey = [] ## check on the presence of docId ?... if len(mcm_r.get_attribute('config_id')): command += ' --step1-docID %s' % ( mcm_r.get_attribute('config_id')[0]) else: ## get the config ID from hash instead of cfg.py hash_ids = database('configs') hash_id = mcm_r.configuration_identifier(0) if hash_ids.document_exists(hash_id): hash_doc = hash_ids.get(hash_id) config_cache_id = hash_doc['docid'] command += ' --step1-docID %s' % config_cache_id config_id_from_hashkey = [config_cache_id] else: command += ' --step1-cfg %s_1_cfg.py' % ( mcm_r.get_attribute('prepid')) command += ' --request-id %s' % (mcm_r.get_attribute('prepid')) if l_type.isDev(): command += ' --wmtest ' command += ' --user pdmvserv ' command += ' --group ppd ' command += ' --batch %s' % batchNumber processString = mcm_r.get_attribute('process_string') def efficiency(mcm_r): feff = mcm_r.get_attribute( 'generator_parameters')[-1]['filter_efficiency'] meff = mcm_r.get_attribute( 'generator_parameters')[-1]['match_efficiency'] return float(feff) * float(meff) if wmagent_type == 'MonteCarlo': # calculate eff dev command += ' --filter-eff %s' % (efficiency(mcm_r)) command += ' --number-events %s' % ( mcm_r.get_attribute('total_events')) command += ' --primary-dataset %s' % ( mcm_r.get_attribute('dataset_name')) elif wmagent_type == 'MonteCarloFromGEN': # calculate eff dev command += ' --filter-eff %s' % (efficiency(mcm_r)) command += ' --input-ds %s' % ( mcm_r.get_attribute('input_dataset')) command += ' --primary-dataset %s' % ( mcm_r.get_attribute('dataset_name')) if mcm_r.get_attribute('block_white_list'): command += ' --blocks "' + ','.join( mcm_r.get_attribute('block_white_list')) + '"' if mcm_r.get_attribute('block_black_list'): command += ' --blocks_black "' + ','.join( mcm_r.get_attribute('block_black_list')) + '"' command += ' --number-events %s' % ( mcm_r.get_attribute('total_events')) elif wmagent_type == 'LHEStepZero': command += ' --number-events %s' % ( mcm_r.get_attribute('total_events')) command += ' --primary-dataset %s' % ( mcm_r.get_attribute('dataset_name')) if mcm_r.get_attribute('mcdb_id') <= 0: numberOfEventsPerJob = mcm_r.numberOfEventsPerJob() if not numberOfEventsPerJob: raise ValueError( 'Number of events per job could not be retrieved') command += ' --events-per-job %s' % numberOfEventsPerJob else: command += ' --lhe ' if not processString: processString = '' processString += 'STEP0ATCERN' elif wmagent_type == 'ReDigi': command += ' --input-ds %s' % ( mcm_r.get_attribute('input_dataset')) command += ' --primary-dataset %s' % ( mcm_r.get_attribute('dataset_name')) ## if PU dataset name is defined : add it if mcm_r.get_attribute( 'pileup_dataset_name') and mcm_r.get_attribute( 'pileup_dataset_name').strip(): command += ' --pileup-ds ' + mcm_r.get_attribute( 'pileup_dataset_name') ## provide the total number of events requested: by default it is the amount in the input dataset. # and wmcontrol / wma should understand that we want partial statistics if that number is lower than expected command += ' --number-events %s' % ( mcm_r.get_attribute('total_events')) # temp ev cont holder eventcontentlist = mcm_r.get_first_output() keeps = mcm_r.get_attribute('keep_output') if not keeps[-1]: raise ValueError('Is not set to save the output of last task') for (i, content) in enumerate(eventcontentlist): if keeps[i]: command += ' --keep-step' + str(i + 1) + ' True' if i > 0: if len(mcm_r.get_attribute('config_id')): command += ' --step%d-docID %s' % ( i + 1, mcm_r.get_attribute('config_id')[i]) else: hash_id = mcm_r.configuration_identifier(i) if hash_ids.document_exists(hash_id): hash_doc = hash_ids.get(hash_id) config_cache_id = hash_doc['docid'] command += ' --step%d-docID %s' % (i + 1, config_cache_id) config_id_from_hashkey.append(config_cache_id) else: command += ' --step%d-cfg %s_%d_cfg.py' % ( i + 1, mcm_r.get_attribute('prepid'), i + 1) # set the output of if i < len(eventcontentlist) - 1: command += ' --step' + str(i + 1) + '-output ' + content if mcm_r.get_attribute('block_white_list'): command += ' --blocks "' + ','.join( mcm_r.get_attribute('block_white_list')) + '"' if mcm_r.get_attribute('block_black_list'): command += ' --blocks_black "' + ','.join( mcm_r.get_attribute('block_black_list')) + '"' if processString: command += ' --process-string ' + processString command += '|| exit $? ;' command += '\n' if len(config_id_from_hashkey): mcm_r.set_attribute('config_id', config_id_from_hashkey) return command
def sendMail(self, destination, subject, text, sender=None, reply_msg_ID=None, accumulate=False): if not isinstance(destination, list): print "Cannot send email. destination should be a list of strings" return destination.sort() msg = MIMEMultipart() # it could happen that message are send after forking, threading and there's no current user anymore msg['From'] = sender if sender else '*****@*****.**' # add a mark on the subjcet automatically if locator().isDev(): msg['Subject'] = '[McM-dev] ' + subject destination = ["*****@*****.**" ] # if -dev send only to service account and sender if sender: destination.append(sender) else: msg['Subject'] = '[McM] ' + subject msg['To'] = COMMASPACE.join(destination) msg['Date'] = formatdate(localtime=True) new_msg_ID = make_msgid() msg['Message-ID'] = new_msg_ID if reply_msg_ID is not None: msg['In-Reply-To'] = reply_msg_ID msg['References'] = reply_msg_ID # accumulate messages prior to sending emails com__accumulate = settings.get_value('com_accumulate') force_com_accumulate = settings.get_value('force_com_accumulate') if force_com_accumulate or (accumulate and com__accumulate): with locker.lock('accumulating_notifcations'): # get a subject where the request name is taken out subject_type = " ".join( filter(lambda w: w.count('-') != 2, msg['Subject'].split())) addressees = msg['To'] sendee = msg['From'] key = (subject_type, sendee, addressees) if key in self.cache: self.cache[key]['Text'] += '\n\n' self.cache[key]['Text'] += text self.cache[key]['N'] += 1 else: self.cache[key] = {'Text': text, 'N': 1} # self.logger.info('Got a message in cache %s'% (self.cache.keys())) return new_msg_ID # add a signature automatically text += '\n\n' text += 'McM Announcing service' try: msg.attach(MIMEText(text)) smtpObj = smtplib.SMTP() smtpObj.connect() smtpObj.sendmail(sender, destination, msg.as_string()) smtpObj.quit() return new_msg_ID except Exception as e: print "Error: unable to send email", e.__class__
def GET(self, *args): """ Provides the injection command and does the injection. """ crn= args[0] crdb = database('chained_requests') mcm_cr = chained_request(crdb.get(crn)) rdb = database('requests') mcm_rs=[] ## upload all config files to config cache, with "configuration economy" already implemented from tools.locker import locker from tools.handlers import ConfigMakerAndUploader for rn in mcm_cr.get_attribute('chain'): mcm_rs.append( request( rdb.get( rn ))) if self.mode=='inject' and mcm_rs[-1].get_attribute('status') != 'approved': return dumps({"results" : False, "message" : 'requests %s in in "%s" status, requires "approved"'%( rn, mcm_rs[-1].get_attribute('status'))}) uploader = ConfigMakerAndUploader(prepid=rn, lock = locker.lock(rn)) uploader.run() mcm_r = mcm_rs[-1] from rest_api.BatchPrepId import BatchPrepId batch_name = BatchPrepId().next_batch_id( mcm_cr.get_attribute('member_of_campaign') , create_batch=self.mode=='inject') from tools.locker import semaphore_events, locker semaphore_events.increment(batch_name) from tools.ssh_executor import ssh_executor from tools.locator import locator l_type = locator() with ssh_executor(server = 'pdmvserv-test.cern.ch') as ssh: cmd='cd /afs/cern.ch/cms/PPD/PdmV/work/McM/dev-submit/\n' cmd+=mcm_r.make_release() cmd+='export X509_USER_PROXY=/afs/cern.ch/user/p/pdmvserv/private/$HOST/voms_proxy.cert\n' cmd+='export PATH=/afs/cern.ch/cms/PPD/PdmV/tools/wmcontrol:${PATH}\n' ## until we get into production there='--wmtest --wmtesturl cmsweb-testbed.cern.ch' cmd+='wmcontrol.py --url-dict %s/public/restapi/chained_requests/get_dict/%s %s \n'%(l_type.baseurl(), crn, there) if self.mode == 'show': cherrypy.response.headers['Content-Type'] = 'text/plain' return cmd else: _, stdout, stderr = ssh.execute(cmd) cherrypy.response.headers['Content-Type'] = 'text/plain' output = stdout.read() error = stderr.read() self.logger.log(output) self.logger.log(error) injected_requests = [l.split()[-1] for l in output.split('\n') if l.startswith('Injected workflow:')] approved_requests = [l.split()[-1] for l in output.split('\n') if l.startswith('Approved workflow:')] if injected_requests and not approved_requests: return dumps({"results" : False, "message" : "Request %s was injected but could not be approved" % ( injected_requests )}) objects_to_invalidate = [ {"_id": inv_req, "object": inv_req, "type": "request", "status": "new", "prepid": self.prepid} for inv_req in injected_requests if inv_req not in approved_requests] if objects_to_invalidate: return dumps({"results" : False, "message" : "Some requests %s need to be invalidated"}) added_requests = [] for mcm_r in mcm_rs: added = [{'name': app_req, 'content': {'pdmv_prep_id': mcm_r.get_attribute('prepid')}} for app_req in approved_requests] added_requests.extend( added ) with locker.lock(batch_name): bdb = database('batches') bat = batch(bdb.get(batch_name)) bat.add_requests(added_requests) bat.update_history({'action': 'updated', 'step': crn}) bat.reload() for mcm_r in mcm_rs: mcm_r.set_attribute('reqmgr_name', added_requests) for mcm_r in mcm_rs: added = [{'name': app_req, 'content': {'pdmv_prep_id': mcm_r.get_attribute('prepid')}} for app_req in approved_requests] mcm_r.set_attribute('reqmgr_name', added ) mcm_r.update_history({'action': 'inject','step' : batch_name}) mcm_r.set_attribute('approval', 'submit') mcm_r.set_status(with_notification=False) ## maybe change to false mcm_r.reload() mcm_cr.update_history({'action' : 'inject','step': batch_name}) mcm_cr.set_attribute('step', len(mcm_rs)-1) mcm_cr.set_attribute('status','processing') mcm_cr.set_attribute('last_status', mcm_rs[-1].get_attribute('status')) message="" for mcm_r in mcm_rs: message+=mcm_r.textified() message+="\n\n" mcm_cr.notify('Injection succeeded for %s'% crn, message) mcm_cr.reload() return dumps({"results" : True, "message" : "request send to batch %s"% batch_name})
def sendMail(self, destination, subject, text, sender=None, reply_msg_ID=None, accumulate=False): if not isinstance(destination, list): print "Cannot send email. destination should be a list of strings" return destination.sort() msg = MIMEMultipart() #it could happen that message are send after forking, threading and there's no current user anymore msg['From'] = sender if sender else '*****@*****.**' ## add a mark on the subjcet automatically if locator().isDev(): msg['Subject'] = '[McM-dev] ' + subject destination = ["*****@*****.**"] # if -dev send only to service account and sender if sender: destination.append(sender) else: msg['Subject'] = '[McM] ' + subject msg['To'] = COMMASPACE.join(destination) msg['Date'] = formatdate(localtime=True) new_msg_ID = make_msgid() msg['Message-ID'] = new_msg_ID if reply_msg_ID is not None: msg['In-Reply-To'] = reply_msg_ID msg['References'] = reply_msg_ID ### accumulate messages prior to sending emails com__accumulate=settings().get_value('com_accumulate') force_com_accumulate=settings().get_value('force_com_accumulate') if force_com_accumulate or (accumulate and com__accumulate): with locker.lock('accumulating_notifcations'): # get a subject where the request name is taken out subject_type=" ".join( filter(lambda w : w.count('-')!=2, msg['Subject'].split()) ) addressees = msg['To'] sendee = msg['From'] key = (subject_type, sendee, addressees) if key in self.cache: self.cache[key]['Text']+='\n\n' self.cache[key]['Text']+=text self.cache[key]['N']+=1 else: self.cache[key] = {'Text' : text, 'N':1} #self.logger.log('Got a message in cache %s'% (self.cache.keys())) return new_msg_ID ## add a signature automatically text += '\n\n' text += 'McM Announcing service' try: msg.attach(MIMEText(text)) smtpObj = smtplib.SMTP() smtpObj.connect() smtpObj.sendmail(sender, destination, msg.as_string()) smtpObj.quit() return new_msg_ID except Exception as e: print "Error: unable to send email", e.__class__
def GET(self, *args): """ Provide the taskchain dictionnary for uploading to request manager """ from tools.locker import locker crn= args[0] crdb = database('chained_requests') ccdb = database('chained_campaigns') rdb = database('requests') mcm_cr = chained_request(crdb.get(crn)) mcm_cc = chained_campaign( ccdb.get( mcm_cr.get_attribute('member_of_campaign'))) from tools.locator import locator l_type=locator() wma={ "RequestType" : "TaskChain", "inputMode" : "couchDB", "RequestString" : crn.replace(mcm_cc.get_attribute('prepid'), mcm_cc.get_attribute('alias')), "Group" : "ppd", "Requestor": "pdmvserv", "Campaign" : mcm_cc.get_attribute('prepid'), "OpenRunningTimeout" : 43200, "TaskChain" : 0, "ProcessingVersion": 1, "RequestPriority" : 0, "PrepID" : crn } mcm_rs=[] for rn in mcm_cr.get_attribute('chain'): mcm_rs.append( request( rdb.get( rn ))) step=1 last_io=None for (i,r) in enumerate( mcm_rs ): steps='step%d'%step for (si, seq) in enumerate(r.get_attribute('sequences')): if r.get_attribute('priority') > wma["RequestPriority"]: wma["RequestPriority"] = r.get_attribute('priority') wma['Task%d'%step] ={'TaskName' : 'Task%d'%step, "ConfigCacheID" : r.get_attribute('config_id')[si], "GlobalTag" : r.get_attribute('sequences')[si]['conditions'], "CMSSWVersion" : r.get_attribute('cmssw_release'), "ScramArch": r.get_scram_arch(), "PrimaryDataset" : r.get_attribute('dataset_name'), "KeepOutput" : r.get_attribute('keep_output')[si], "AcquisitionEra" : r.get_attribute('member_of_campaign'), "ProcessingString" : r.get_processing_string(si), "ProcessingVersion" : r.get_attribute('version'), "TimePerEvent" : r.get_attribute("time_event"), "SizePerEvent" : r.get_attribute('size_event'), "Memory" : r.get_attribute('memory') } if r.get_attribute('pileup_dataset_name'): wma['Task%d'%step]["MCPileup"] = r.get_attribute('pileup_dataset_name') for item in ['CMSSWVersion','ScramArch','TimePerEvent','SizePerEvent','GlobalTag','Memory']: ## needed for dictionnary validation in requests manager, but not used explicitely if item not in wma: wma[item] = wma['Task%d'%step][item] if step==1: wma['Task%d'%step].update({"SplittingAlgo" : "EventBased", "RequestNumEvents" : r.get_attribute('total_events'), "Seeding" : "AutomaticSeeding", ### gets wiped out anyways "SplittingArguments" : { "events_per_job" : 123, "events_per_lumi" : 78}, "EventsPerLumi" : 100, ## does not get seen in request manager, yet "LheInputFiles" : r.get_attribute('mcdb_id')>0 }) else: wma['Task%d'%step].update({"SplittingAlgo" : "EventAwareLumiBased", "InputFromOutputModule" : "%soutput"%last_io, "InputTask" : "Task%d"%(step-1), ### gets wiped out anyways "SplittingArguments" :{} }) last_io=r.get_attribute('sequences')[si]['eventcontent'][0] wma['TaskChain']+=1 step+=1 return dumps(wma)
def flow_to_next_step(self, input_dataset='', block_black_list=None, block_white_list=None, check_stats=True, reserve=False): if not block_white_list: block_white_list = [] if not block_black_list: block_black_list = [] self.logger.log('Flowing chained_request %s to next step...' % (self.get_attribute('_id'))) if not self.get_attribute('chain'): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'chained_request %s has got no root' % (self.get_attribute('_id'))) # check on the approval of the chained request before all ## let it be flowing regardless #if self.get_attribute('approval') == 'none': # raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), # 'The approval of the chained request is none, and therefore flow cannot happen') #this operation requires to access all sorts of objects rdb = database('requests') cdb = database('campaigns') ccdb = database('chained_campaigns') crdb = database('chained_requests') fdb = database('flows') adb = database('actions') l_type = locator() current_step = len(self.get_attribute( 'chain')) - 1 if reserve else self.get_attribute('step') current_id = self.get_attribute('chain')[current_step] next_step = current_step + 1 if not rdb.document_exists(current_id): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the request %s does not exist' % current_id) current_request = request(rdb.get(current_id)) current_campaign = campaign( cdb.get(current_request.get_attribute('member_of_campaign'))) if not ccdb.document_exists(self.get_attribute('member_of_campaign')): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the chain request %s is member of %s that does not exist' % (self.get_attribute('_id'), self.get_attribute('member_of_campaign'))) mcm_cc = ccdb.get(self.get_attribute('member_of_campaign')) if next_step >= len(mcm_cc['campaigns']): if reserve: return False raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'chained_campaign %s does not allow any further flowing.' % (self.get_attribute('member_of_campaign'))) if not reserve: ## is the current request in the proper approval allowed_request_approvals = ['submit'] if current_request.get_attribute( 'approval') not in allowed_request_approvals: raise self.NotApprovedException( current_request.get_attribute('prepid'), current_request.get_attribute('approval'), allowed_request_approvals) ## is the current request in the proper status allowed_request_statuses = ['submitted', 'done'] if current_request.get_attribute( 'status') not in allowed_request_statuses: raise self.NotInProperStateException( current_request.get_attribute('prepid'), current_request.get_attribute('status'), allowed_request_statuses) original_action_id = self.get_attribute('chain')[0] original_action_item = self.retrieve_original_action_item( adb, original_action_id) ## what is the campaign to go to next and with which flow (next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step] if not fdb.document_exists(flow_name): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The flow %s does not exist' % flow_name) mcm_f = flow(fdb.get(flow_name)) if not 'sequences' in mcm_f.get_attribute('request_parameters'): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The flow %s does not contain sequences information.' % (flow_name)) if not cdb.document_exists(next_campaign_id): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The next campaign %s does not exist' % next_campaign_id) next_campaign = campaign(cdb.get(next_campaign_id)) if len(next_campaign.get_attribute('sequences')) != len( mcm_f.get_attribute('request_parameters')['sequences']): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the sequences changes in flow %s are not consistent with the next campaign %s' % (flow_name, next_campaign_id)) if next_campaign.get_attribute( 'energy') != current_campaign.get_attribute('energy'): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'cannot flow any further. Request {0} has inconsistent energy.' .format(next_campaign.get_attribute("prepid"))) if not next_campaign.is_release_greater_or_equal_to( current_campaign.get_attribute('cmssw_release')): raise self.ChainedRequestCannotFlowException( self.get_attribute("_id"), 'cannot flow any further. Request {0} has lower release version.' .format(next_campaign.get_attribute("prepid"))) if next_campaign.get_attribute('type') == 'MCReproc' and ( not 'time_event' in mcm_f.get_attribute('request_parameters')): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the flow is getting into a MCReproc campaign but not time per event is specified' ) if next_campaign.get_attribute('type') == 'MCReproc' and ( not 'size_event' in mcm_f.get_attribute('request_parameters')): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the flow is getting into a MCReproc campaign but not size per event is specified' ) ## check that it is allowed to flow allowed_flow_approvals = ['flow', 'submit'] ###### cascade of checks """ if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The flow (%s) is not in proper approval state (%s)'%( mcm_f.get_attribute('prepid'), mcm_f.get_attribute('approval'))) if not reserve and not self.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The chained request (%s) is not in the proper approval state (%s)'% ( self.get_attribute('_id'), self.get_attribute('approval'))) """ if not reserve and not mcm_f.get_attribute( 'approval') in allowed_flow_approvals: if not self.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing' % (mcm_f.get_attribute('approval'), self.get_attribute('approval'))) if next_campaign.get_attribute('status') == 'stopped': raise self.CampaignStoppedException(next_campaign_id) #what is going to be the required number of events for the next request #update the stats to its best if not reserve: current_request.get_stats() next_total_events = current_request.get_attribute( 'completed_events') ## get the original expected events and allow a margin of 5% less statistics statistics_fraction = settings().get_value('statistics_fraction') current_eff_error = 1. - current_request.get_efficiency_error() statistics_fraction = min(statistics_fraction, current_eff_error) completed_events_to_pass = int( current_request.get_attribute('total_events') * statistics_fraction) notify_on_fail = True ## to be tuned according to the specific cases if current_request.get_attribute('completed_events') <= 0: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The number of events completed is negative or null') else: allowed_request_statuses = ['done'] ## determine if this is a root -> non-root transition to potentially apply staged number at_a_transition = (current_campaign.get_attribute('root') != 1 and next_campaign.get_attribute('root') == 1) if ('staged' in original_action_item or 'threshold' in original_action_item) and at_a_transition: allowed_request_statuses.append('submitted') ##check status if not current_request.get_attribute( 'status') in allowed_request_statuses: raise self.NotInProperStateException( current_request.get_attribute('prepid'), current_request.get_attribute('status'), allowed_request_statuses) ##special check at transition that the statistics is good enough if at_a_transition: # at a root -> non-root transition only does the staged/threshold functions ! if 'staged' in original_action_item: next_total_events = int(original_action_item['staged']) completed_events_to_pass = next_total_events if 'threshold' in original_action_item: next_total_events = int( current_request.get_attribute('total_events') * float(original_action_item['threshold'] / 100.)) completed_events_to_pass = next_total_events if check_stats and ( current_request.get_attribute('completed_events') < completed_events_to_pass): if notify_on_fail: current_request.notify( 'Flowing %s with not enough statistics' % (current_request.get_attribute('prepid')), 'For the request %s, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s in chain %s \n\n Please report to the operation HN or at the next MccM what action should be taken.\n\n %srequests?prepid=%s\n%schained_requests?contains=%s\n%schained_requests?prepid=%s ' % (current_request.get_attribute('prepid'), current_request.get_attribute('completed_events'), completed_events_to_pass, self.get_attribute('prepid'), l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), self.get_attribute('prepid')), accumulate=True) raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The number of events completed (%s) is not enough for the requirement (%s)' % (current_request.get_attribute('completed_events'), completed_events_to_pass)) ## select what is to happened : [create, patch, use] next_id = None approach = None next_request = None if next_step != len(self.get_attribute('chain')): #not at the end next_id = self.get_attribute('chain')[next_step] if not rdb.document_exists(next_id): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The next request (%s) according to the step (%s) does not exist' % (next_id, next_step)) next_request = request(rdb.get(next_id)) if next_request.get_attribute('status') == 'new': #most likely a rewind + resub approach = 'patch' else: ##this is always the case in chains reserved from existing things: so use the next request approach = 'use' #raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), #'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % ( #next_id, next_step, next_request.get_attribute('status'))) else: ## look in *other* chained campaigns whether you can suck in an existing request ## look up all chained requests that start from the same root request ## remove <pwg>-chain_ and the -serial number, replacing _ with a . toMatch = '.'.join( self.get_attribute('prepid').split('_')[1:][0:next_step + 1]).split('-')[0] ## make sure they get ordered by prepid related_crs = sorted(crdb.queries( ['root_request==%s' % original_action_id]), key=lambda cr: cr['prepid']) vetoed_last = [] for existing_cr in related_crs: ## exclude itself if existing_cr['prepid'] == self.get_attribute('prepid'): continue ## prevent from using a request from within the same exact chained_campaigns if existing_cr['member_of_campaign'] == self.get_attribute( 'member_of_campaign'): mcm_cr = chained_request(crdb.get(existing_cr['prepid'])) if len(mcm_cr.get_attribute('chain')) > next_step: ## one existing request in the very same chained campaign has already something used, make sure it is not going to be used vetoed_last.append( mcm_cr.get_attribute('chain')[next_step]) continue else: continue for existing_cr in related_crs: ## exclude itself if existing_cr['prepid'] == self.get_attribute('prepid'): continue ## prevent from using a request from within the same exact chained_campaigns if existing_cr['member_of_campaign'] == self.get_attribute( 'member_of_campaign'): continue truncated = '.'.join( existing_cr['prepid'].split('_')[1:][0:next_step + 1]).split('-')[0] self.logger.error('to match : %s , this one %s' % (toMatch, truncated)) if truncated == toMatch: #we found a chained request that starts with all same steps mcm_cr = chained_request(crdb.get(existing_cr['prepid'])) if len(mcm_cr.get_attribute('chain')) <= next_step: #found one, but it has not enough content either continue if mcm_cr.get_attribute('chain')[next_step] in vetoed_last: continue next_id = mcm_cr.get_attribute('chain')[next_step] break if next_id: approach = 'use' else: approach = 'create' if approach == 'create': from rest_api.RequestPrepId import RequestPrepId next_id = RequestPrepId().next_prepid( current_request.get_attribute('pwg'), next_campaign_id) next_request = request(rdb.get(next_id)) request.transfer(current_request, next_request) self.request_join(next_request) elif approach == 'use': ## there exists a request in another chained campaign that can be re-used here. # take this one. advance and go on next_request = request(rdb.get(next_id)) if not reserve: self.set_attribute('step', next_step) self.set_attribute('last_status', next_request.get_attribute('status')) self.update_history({'action': 'flow', 'step': str(next_id)}) self.set_attribute('status', 'processing') if not self.get_attribute("prepid") in next_request.get_attribute( "member_of_chain"): ## register the chain to the next request self.request_join(next_request) saved = rdb.update(next_request.json()) if not saved: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Unable to save %s with updated member_of_chains' % next_id) return True elif approach == 'patch': ## there exists already a request in the chain (step!=last) and it is usable for the next stage next_request = request(next_campaign.add_request(rdb.get(next_id))) ## propagate again some of the fields of the previous request. request.transfer(current_request, next_request) else: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Unrecognized approach %s' % approach) #current_request -> next_request #current_campaign -> next_campaign ##determine whether we have an input dataset for the next request if len(current_request.get_attribute('reqmgr_name')): last_wma = current_request.get_attribute('reqmgr_name')[-1] if 'content' in last_wma and 'pdmv_dataset_name' in last_wma[ 'content']: input_dataset = last_wma['content']['pdmv_dataset_name'] else: statsDB = database('stats', url='http://cms-pdmv-stats.cern.ch:5984/') if statsDB.document_exists(last_wma['name']): latestStatus = statsDB.get(last_wma['name']) input_dataset = latestStatus['pdmv_dataset_name'] if input_dataset: next_request.set_attribute('input_dataset', input_dataset) ## set blocks restriction if any if block_black_list: next_request.set_attribute('block_black_list', block_black_list) if block_white_list: next_request.set_attribute('block_white_list', block_white_list) ## register the flow to the request next_request.set_attribute('flown_with', flow_name) ##assemble the campaign+flow => request request.put_together(next_campaign, mcm_f, next_request) if not reserve: #already taking stage and threshold into account next_request.set_attribute('total_events', next_total_events) next_request.update_history({ 'action': 'flow', 'step': self.get_attribute('prepid') }) request_saved = rdb.save(next_request.json()) if not request_saved: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Could not save the new request %s' % (next_request.get_attribute('prepid'))) ## inspect priority self.set_priority(original_action_item['block_number']) if not reserve: # sync last status self.set_attribute('last_status', next_request.get_attribute('status')) # we can only be processing at this point self.set_attribute('status', 'processing') # set to next step self.set_attribute('step', next_step) self.update_history({ 'action': 'flow', 'step': next_request.get_attribute('prepid') }) if not reserve: notification_subject = 'Flow for request %s in %s' % ( current_request.get_attribute('prepid'), next_campaign_id) notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % ( current_request.get_attribute('prepid'), self.get_attribute('member_of_campaign'), next_campaign_id, flow_name, next_request.get_attribute('prepid'), self.get_attribute('prepid'), next_request.get_attribute('input_dataset'), l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), next_request.get_attribute('prepid')) current_request.notify(notification_subject, notification_text, accumulate=True) else: notification_subject = 'Reservation of request {0}'.format( next_request.get_attribute('prepid')) notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format( next_request.get_attribute('prepid'), self.get_attribute('prepid'), next_campaign_id, l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), next_request.get_attribute('prepid'), ) next_request.notify(notification_subject, notification_text, accumulate=True) return True