def get(self, message_number=0): """ Trigger the accumulated communications from McM, optionally above /N messages """ com = communicator() res = com.flush(message_number) return {'results': True, 'subject' : res}
def get(self, message_number=0): """ Trigger the accumulated communications from McM, optionally above /N messages """ com = communicator() res = com.flush(message_number) return {'results': True, 'subject': res}
def notify(self, body): db = database('users') data = threaded_loads(body) list_of_mails = [x["value"] for x in db.raw_query('pwg-mail', {'key': data["pwg"]})] com = communicator() com.sendMail(list_of_mails, data["subject"], data["content"], user_pack().get_email()) return {'results': True, 'message': 'Sent message to {0}'.format(list_of_mails)}
def GET(self, *args): """ Trigger the accumulated communications from McM, optionally above /N messages """ N=0 if len(args): N=args[0] com = communicator() res=com.flush(N) return dumps({'results':True, 'subject' : res})
def GET(self, *args): """ Trigger the accumulated communications from McM, optionally above /N messages """ N = 0 if len(args): N = args[0] com = communicator() res = com.flush(N) return dumps({'results': True, 'subject': res})
def streaming_function(): mccms_db = database('mccms') users_db = database('users') generator_contacts_query = users_db.construct_lucene_query({'role': 'generator_contact'}) generator_contacts = users_db.full_text_search("search", generator_contacts_query, page=-1) generator_contacts_by_pwg = {} generator_contacts_emails = set() for contact in generator_contacts: for pwg in contact.get('pwg', []): if pwg not in generator_contacts_by_pwg: generator_contacts_by_pwg[pwg] = [] generator_contacts_by_pwg[pwg].append(contact.get('email')) generator_contacts_emails.add(contact.get('email')) __query = mccms_db.construct_lucene_query({'status': 'new'}) mccms_tickets = mccms_db.full_text_search('search', __query, page=-1) authors_tickets_dict = dict() yield '<pre>' for ticket in mccms_tickets: yield 'Processing ticket %s\n' % (ticket['prepid']) mccm_ticket = mccm(json_input=ticket) pwg = mccm_ticket.get_attribute('pwg') authors = mccm_ticket.get_actors(what='author_email') yield '%s worked on %s\n' % (authors, ticket['prepid']) authors = filter(lambda e: e in generator_contacts_emails, list(set(authors + generator_contacts_by_pwg.get(pwg, [])))) yield '%s will be notified about %s\n' % (authors, ticket['prepid']) for author_email in authors: if author_email in generator_contacts_emails: if author_email not in authors_tickets_dict: authors_tickets_dict[author_email] = set() authors_tickets_dict[author_email].add(ticket['prepid']) subject_template = 'Gentle reminder on %s ticket%s to be operated by you' message_template = ('Dear GEN Contact,\nPlease find below the details of %s MccM ticket%s in status "new". ' + 'Please present them in next MccM googledoc or cancel tickets if these are not needed anymore.\n\n') base_url = locator().baseurl() mail_communicator = communicator() service_account = settings.get_value('service_account') for author_email, ticket_prepids in authors_tickets_dict.iteritems(): num_tickets = len(ticket_prepids) subject = subject_template % (num_tickets, '' if num_tickets == 1 else 's') message = message_template % (num_tickets, '' if num_tickets == 1 else 's') for ticket_prepid in ticket_prepids: message += 'Ticket: %s\n%smccms?prepid=%s\n\n' % (ticket_prepid, base_url, ticket_prepid) yield '.' yield '\n' message += 'You received this email because you are listed as generator contact of physics group(s) of these tickets.\n' self.logger.info('Email:%s\nSubject: %s\nMessage:%s' % (author_email, subject, message)) mail_communicator.sendMail([author_email, service_account], subject, message) yield 'Email sent to %s\n' % (author_email)
def GET(self, *args): """ Send a reminder to the production managers for existing opened mccm documents """ mdb = database('mccms') mccms = mdb.queries(['status==new']) udb = database('users') block_threshold = 0 if len(args): block_threshold = int(args[0]) mccms = filter(lambda m: m['block'] <= block_threshold, mccms) mccms = sorted(mccms, key=lambda m: m['block']) if len(mccms) == 0: return dumps({ "results": True, "message": "nothing to remind of at level %s, %s" % (block_threshold, mccms) }) l_type = locator() com = communicator() subject = 'Gentle reminder on %s tickets to be operated by you' % ( len(mccms)) message = '''\ Dear Production Managers, please find below the details of %s opened MccM tickets that need to be operated. ''' % (len(mccms)) for mccm in mccms: message += 'Ticket : %s (block %s)\n' % (mccm['prepid'], mccm['block']) message += ' %smccms?prepid=%s \n\n' % (l_type.baseurl(), mccm['prepid']) message += '\n' to_who = [settings().get_value('service_account')] to_who.extend( map(lambda u: u['email'], udb.query(query="role==production_manager", page_num=-1))) com.sendMail(to_who, subject, message) return dumps({ "results": True, "message": map(lambda m: m['prepid'], mccms) })
def notify(self, body): db = database('users') data = loads(body) list_of_mails = [ x["value"] for x in db.raw_query('pwg-mail', {'key': data["pwg"]}) ] com = communicator() com.sendMail(list_of_mails, data["subject"], data["content"], user_pack().get_email()) return { 'results': True, 'message': 'Sent message to {0}'.format(list_of_mails) }
def streaming_function(): mccms_db = database('mccms') users_db = database('users') __query = mccms_db.construct_lucene_query({'status': 'new'}) mccms_tickets = mccms_db.full_text_search('search', __query, page=-1) non_gen_contact_authors = set() authors_tickets_dict = dict() emails_prepids = dict() for ticket in mccms_tickets: yield '\nProcessing ticket %s' % (ticket['prepid']) mccm_ticket = mccm(json_input=ticket) authors = mccm_ticket.get_actors(what='author_email') for author_email in authors: if author_email in authors_tickets_dict: authors_tickets_dict[author_email].append(ticket['prepid']) elif author_email not in non_gen_contact_authors: __role_query = users_db.construct_lucene_query({'email': author_email}) result = users_db.full_text_search('search', __role_query, page=-1, include_fields='role,prepid') time.sleep(0.5) # we don't want to crash DB with a lot of single queries if result and result[0]['role'] == 'generator_contact': authors_tickets_dict[author_email] = [ticket['prepid']] emails_prepids[author_email] = result[0]['prepid'] else: non_gen_contact_authors.add(author_email) yield '.' subject_part1 = 'Gentle reminder on %s ' subject_part2 = ' to be operated by you' message_part1 = 'Dear GEN Contact, \nPlease find below the details of %s MccM ' message_part2 = ' in status "new". Please present them in next MccM googledoc or cancel tickets if these are not needed anymore.\n\n' base_url = locator().baseurl() mail_communicator = communicator() for author_email, ticket_prepids in authors_tickets_dict.iteritems(): num_tickets = len(ticket_prepids) full_message = (message_part1 % (num_tickets)) + ('ticket' if num_tickets == 1 else 'tickets') + message_part2 for ticket_prepid in ticket_prepids: full_message += 'Ticket: %s \n' % (ticket_prepid) full_message += '%smccms?prepid=%s \n\n' % (base_url, ticket_prepid) yield '.' full_message += '\n' subject = (subject_part1 % (num_tickets)) + ('ticket' if num_tickets == 1 else 'tickets') + subject_part2 notification( subject, full_message, [emails_prepids[author_email]], group=notification.REMINDERS, action_objects=ticket_prepids, object_type='mccms') mail_communicator.sendMail([author_email], subject, full_message) yield '\nEmail sent to %s\n' % (author_email)
def send_email_failure(self, output, error): com = communicator() users_db = database('users') query = users_db.construct_lucene_query({'role': 'production_manager'}) production_managers = users_db.full_text_search('search', query, page=-1) subject = "There was an error while trying to approve workflows" text = "Workflows: %s\nOutput:\n%s\nError output: \n%s" % ( self.workflows, output, error) notification(subject, text, [], group=notification.REQUEST_OPERATIONS, target_role="production_manager") com.sendMail(map(lambda u: u['email'], production_managers), subject, text)
def get(self, block_threshold=0): """ Send a reminder to the production managers for existing opened mccm documents """ mdb = database('mccms') udb = database('users') __query = mdb.construct_lucene_query({'status': 'new'}) mccms = mdb.full_text_search('search', __query, page=-1) mccms = filter(lambda m: m['block'] <= block_threshold, mccms) mccms = sorted(mccms, key=lambda m: m['block']) if len(mccms) == 0: return {"results": True, "message": "nothing to remind of at level %s, %s" % (block_threshold, mccms)} l_type = locator() com = communicator() subject = 'Gentle reminder on %s tickets to be operated by you' % (len( mccms)) message = '''\ Dear Production Managers, please find below the details of %s opened MccM tickets that need to be operated. ''' % (len(mccms)) mccm_prepids = [] for _mccm in mccms: prepid = _mccm['prepid'] message += 'Ticket : %s (block %s)\n' % (prepid, _mccm['block']) message += ' %smccms?prepid=%s \n\n' % (l_type.baseurl(), prepid) mccm_prepids.append(prepid) message += '\n' to_who = [settings.get_value('service_account')] to_who.extend(map(lambda u: u['email'], udb.query(query="role==production_manager", page_num=-1))) notification( subject, message, [], group=notification.REMINDERS, action_objects=mccm_prepids, object_type='mccms', target_role='production_manager') com.sendMail( to_who, subject, message) return {"results": True, "message": map(lambda m: m['prepid'], mccms)}
def get(self, block_threshold=0): """ Send a reminder to the production managers for existing opened mccm documents """ mdb = database('mccms') udb = database('users') __query = mdb.construct_lucene_query({'status': 'new'}) mccms = mdb.full_text_search('search', __query, page=-1) mccms = filter(lambda m: m['block'] <= block_threshold, mccms) mccms = sorted(mccms, key=lambda m: m['block']) if len(mccms) == 0: return {"results": True, "message": "nothing to remind of at level %s, %s" % (block_threshold, mccms)} l_type = locator() com = communicator() subject = 'Gentle reminder on %s tickets to be operated by you' % (len( mccms)) message = '''\ Dear Production Managers, please find below the details of %s opened MccM tickets that need to be operated. ''' % (len(mccms)) mccm_prepids = [] for _mccm in mccms: prepid = _mccm['prepid'] message += 'Ticket : %s (block %s)\n' % (prepid, _mccm['block']) message += ' %smccms?prepid=%s \n\n' % (l_type.baseurl(), prepid) mccm_prepids.append(prepid) message += '\n' to_who = [settings.get_value('service_account')] to_who.extend(map(lambda u: u['email'], udb.query(query="role==production_manager", page_num=-1))) notification( subject, message, [], group=notification.REMINDERS, action_objects=mccm_prepids, object_type='mccms', target_role='production_manager') com.sendMail( to_who, subject, message) return {"results": True, "message": map(lambda m: m['prepid'], mccms)}
def get(self, pwgs): """ Ask for the increase of the role of the current user to the given pwg """ # get who's there user_p = user_pack() udb = database(self.db_name) mcm_u = user(udb.get(user_p.get_username())) # get the requested pwgs pwgs = pwgs.split(',') # set the pwgs to the current user current = mcm_u.get_attribute('pwg') current = list(set(current + pwgs)) mcm_u.set_attribute('pwg', current) mcm_u.update_history({'action': 'ask role', 'step': pwgs}) udb.update(mcm_u.json()) # get the production managers emails __query = udb.construct_lucene_query({'role': 'production_manager'}) production_managers = udb.full_text_search('search', __query, page=-1) # send a notification to prod manager + service to_who = map(lambda u: u['email'], production_managers) + [ settings.get_value('service_account') ] to_who.append(user_p.get_email()) com = communicator() l_type = locator() subject = 'Increase role for user %s' % mcm_u.get_attribute('fullname') message = 'Please increase the role of the user %s to the next level.\n\n%susers?prepid=%s' % ( mcm_u.get_attribute('username'), l_type.baseurl(), mcm_u.get_attribute('username')) notification(subject, message, [], group=notification.USERS, action_objects=[mcm_u.get_attribute('prepid')], object_type='users', target_role='production_manager') com.sendMail(to_who, subject, message) return { "results": True, "message": "user %s in for %s" % (mcm_u.get_attribute('username'), current) }
def GET(self, *args): """ Ask for the increase of the role of the current user to the given pwg """ if not args: return dumps({"results": False, "Message": "not pwg provided"}) ## get who's there user_p = user_pack() udb = database(self.db_name) mcm_u = user(udb.get(user_p.get_username())) ## get the requested pwgs pwgs = args[0].split(',') #### set the pwgs to the current user current = mcm_u.get_attribute('pwg') current = list(set(current + pwgs)) mcm_u.set_attribute('pwg', current) mcm_u.update_history({'action': 'ask role', 'step': args[0]}) udb.update(mcm_u.json()) ## get the production managers emails production_managers = udb.queries(['role==production_manager']) ### send a notification to prod manager + service to_who = map(lambda u: u['email'], production_managers) + [ settings().get_value('service_account') ] to_who.append(user_p.get_email()) com = communicator() l_type = locator() com.sendMail( to_who, 'Increase role for user %s' % mcm_u.get_attribute('fullname'), 'Please increase the role of the user %s to the next level.\n\n%susers?prepid=%s' % (mcm_u.get_attribute('username'), l_type.baseurl(), mcm_u.get_attribute('username'))) return dumps({ "results": True, "message": "user %s in for %s" % (mcm_u.get_attribute('username'), current) })
def GET(self, *args): """ Send a reminder to the production managers for existing opened mccm documents """ mdb = database('mccms') mccms = mdb.queries(['status==new']) udb = database('users') block_threshold = 0 if len(args): block_threshold = int(args[0]) mccms = filter( lambda m : m['block'] <= block_threshold, mccms) mccms = sorted( mccms, key = lambda m : m['block']) if len(mccms)==0: return dumps({"results": True,"message": "nothing to remind of at level %s, %s"% (block_threshold, mccms)}) l_type = locator() com = communicator() subject = 'Gentle reminder on %s tickets to be operated by you' % ( len( mccms)) message = '''\ Dear Production Managers, please find below the details of %s opened MccM tickets that need to be operated. ''' % (len(mccms)) for mccm in mccms: message += 'Ticket : %s (block %s)\n'%( mccm['prepid'], mccm['block'] ) message += ' %smccms?prepid=%s \n\n' % (l_type.baseurl(), mccm['prepid']) message += '\n' to_who = [settings().get_value('service_account')] to_who.extend( map( lambda u : u['email'], udb.query(query="role==production_manager", page_num=-1))) com.sendMail(to_who, subject, message) return dumps({"results" : True, "message" : map( lambda m : m['prepid'], mccms)})
def GET(self, *args): """ Ask for the increase of the role of the current user to the given pwg """ if not args: return dumps({"results" : False, "Message" : "not pwg provided"}) ## get who's there user_p = user_pack() udb = database(self.db_name) mcm_u = user( udb.get( user_p.get_username())) ## get the requested pwgs pwgs = args[0].split(',') #### set the pwgs to the current user current = mcm_u.get_attribute('pwg') current = list(set(current+pwgs)) mcm_u.set_attribute('pwg', current) mcm_u.update_history({'action':'ask role','step' : args[0]}) udb.update(mcm_u.json()) ## get the production managers emails production_managers = udb.queries(['role==production_manager']) ### send a notification to prod manager + service to_who = map(lambda u: u['email'], production_managers) + [settings().get_value('service_account')] to_who.append( user_p.get_email() ) com = communicator() l_type = locator() com.sendMail( to_who, 'Increase role for user %s' % mcm_u.get_attribute('fullname'), 'Please increase the role of the user %s to the next level.\n\n%susers?prepid=%s' % ( mcm_u.get_attribute('username'), l_type.baseurl(), mcm_u.get_attribute('username') )) return dumps({"results" : True, "message" : "user %s in for %s" %( mcm_u.get_attribute('username'), current)})
def __init__(self): self.db_name = "invalidations" self.com = communicator() self.l_type = locator()
self.logger.info(message) notification(subject, message, [], group=notification.REQUEST_OPERATIONS, action_objects=[request_prepid], object_type='requests', base_object=mcm_request) mcm_request.notify(subject, message) return validations_count < max_validations def main(self): # First we check if some of the submitted jobs already finished to process them. if self.ssh_exec is None: sys.exit(-1) self.monitor_submmited_jobs() if not self.is_condor_working: # Sometimes htcondor is not working so we stop everything return self.new_jobs = self.get_new_request_prepids( ) + self.get_new_chain_prepids() prepids_to_submit = self.get_prepids_to_submit() self.submit_jobs(prepids_to_submit) self.save_jobs() if __name__ == "__main__": validation_handler = ValidationHandler() validation_handler.main() com = communicator() com.flush(0)
def setup(self): self.com = communicator()
def __init__(self): self.db_name = "invalidations" self.com = communicator() self.l_type = locator()
def at_flask_exit(*args): RESTResource.counter.close() com = communicator() com.flush(0)
def stop(): logfactory.log(".mcm_rest_counter persistence closing") RESTResource.counter.close() logfactory.log("Flushing communications") com = communicator() com.flush(0)
def GET(self, *args): """ Goes through invalidation documents and display (/) and announce them to data ops (/announce) """ announce = False clear = False if len(args) != 0: if args[0] == 'announce': announce = True if args[0] == 'clear': clear = True idb = database('invalidations') r_to_be_rejected = map(invalidation, idb.queries(['status==new', 'type==request'])) ds_to_be_invalidated = map( invalidation, idb.queries(['status==new', 'type==dataset'])) ds_to_be_invalidated = filter( lambda ds: not 'None-' in ds.get_attribute('object'), ds_to_be_invalidated) l_type = locator() def add_prepid(invalid, html): if 'prepid' in invalid.json(): html += 'for request <a href=%srequests?prepid=%s> %s </a>' % ( l_type.baseurl(), invalid.get_attribute('prepid'), invalid.get_attribute('prepid')) def print_invalidations(invalids): a_text = '' for invalid in invalids: a_text += ' %s\n' % (invalid.get_attribute('object')) return a_text html = '<html><body>\n' html += 'Requests to be aborted/rejected <br>\n' html += '<ul>\n' for r in r_to_be_rejected: html += '<li> <a href=%sreqmgr/view/details/%s> %s </a>' % ( l_type.cmsweburl(), r.get_attribute('object'), r.get_attribute('object')) add_prepid(r, html) html += '</li>\n' html += '</ul>\n' html += 'Datasets to be invalidated <br>\n' html += '<ul>\n' for ds in ds_to_be_invalidated: html += '<li> %s ' % (ds.get_attribute('object')) add_prepid(ds, html) html += '</li>\n' html += '</ul>\n' html += '<a href=%srestapi/invalidations/inspect/clear> Clear invalidation withouth announcing</a><br>\n' % ( l_type.baseurl()) html += '<a href=%srestapi/invalidations/inspect/announce> Announce invalidations</a><br>\n' % ( l_type.baseurl()) html += '<a href=%srestapi/invalidations/inspect> Back</a><br>\n' % ( l_type.baseurl()) html += '</html></body>\n' if announce and (len(ds_to_be_invalidated) != 0 or len(r_to_be_rejected) != 0): text = 'Dear Data Operation Team,\n\n' if len(r_to_be_rejected) != 0: text += 'please reject or abort the following requests:\n' text += print_invalidations(r_to_be_rejected) if len(ds_to_be_invalidated) != 0: text += '\nPlease invalidate the following datasets:\n' text += print_invalidations(ds_to_be_invalidated) text += '\nas a consequence of requests being reset.\n' com = communicator() to_who = [settings().get_value('service_account')] if l_type.isDev(): to_who.append(settings().get_value('hypernews_test')) else: to_who.append(settings().get_value('dataops_announce')) try: elem = (r_to_be_rejected + ds_to_be_invalidated)[0] sender = elem.current_user_email except IndexError: sender = None com.sendMail(to_who, 'Request and Datasets to be Invalidated', text, sender) for to_announce in itertools.chain(r_to_be_rejected, ds_to_be_invalidated): to_announce.set_announced() idb.update(to_announce.json()) if clear and (len(ds_to_be_invalidated) != 0 or len(r_to_be_rejected) != 0): for to_announce in itertools.chain(r_to_be_rejected, ds_to_be_invalidated): to_announce.set_announced() idb.update(to_announce.json()) return html
def stop(): logfactory.log(".mcm_rest_counter persistence closing") RESTResource.counter.close() logfactory.log("Flushing communications") com = communicator() com.flush(0)
def setup(self): self.com = communicator()
def GET(self, *args): """ Goes through invalidation documents and display (/) and announce them to data ops (/announce) """ announce = False clear = False if len(args) != 0: if args[0] == 'announce': announce = True if args[0] == 'clear': clear = True idb = database('invalidations') r_to_be_rejected = map(invalidation, idb.queries(['status==new', 'type==request'])) ds_to_be_invalidated = map(invalidation, idb.queries(['status==new', 'type==dataset'])) ds_to_be_invalidated = filter(lambda ds : not 'None-' in ds.get_attribute('object'), ds_to_be_invalidated) l_type = locator() def add_prepid(invalid, html): if 'prepid' in invalid.json(): html += 'for request <a href=%srequests?prepid=%s> %s </a>' % ( l_type.baseurl(), invalid.get_attribute('prepid'), invalid.get_attribute('prepid')) def print_invalidations(invalids): a_text='' for invalid in invalids: a_text += ' %s\n' % (invalid.get_attribute('object')) return a_text html = '<html><body>\n' html += 'Requests to be aborted/rejected <br>\n' html += '<ul>\n' for r in r_to_be_rejected: html += '<li> <a href=%sreqmgr/view/details/%s> %s </a>' % (l_type.cmsweburl(), r.get_attribute('object'), r.get_attribute('object') ) add_prepid(r, html) html += '</li>\n' html += '</ul>\n' html += 'Datasets to be invalidated <br>\n' html += '<ul>\n' for ds in ds_to_be_invalidated: html += '<li> %s ' % ( ds.get_attribute('object') ) add_prepid(ds, html) html += '</li>\n' html += '</ul>\n' html += '<a href=%srestapi/invalidations/inspect/clear> Clear invalidation withouth announcing</a><br>\n'%( l_type.baseurl() ) html += '<a href=%srestapi/invalidations/inspect/announce> Announce invalidations</a><br>\n'%( l_type.baseurl() ) html += '<a href=%srestapi/invalidations/inspect> Back</a><br>\n'%( l_type.baseurl() ) html += '</html></body>\n' if announce and (len(ds_to_be_invalidated)!=0 or len(r_to_be_rejected)!=0): text = 'Dear Data Operation Team,\n\n' if len(r_to_be_rejected)!=0: text += 'please reject or abort the following requests:\n' text += print_invalidations(r_to_be_rejected) if len(ds_to_be_invalidated)!=0: text += '\nPlease invalidate the following datasets:\n' text += print_invalidations(ds_to_be_invalidated) text += '\nas a consequence of requests being reset.\n' com = communicator() to_who = [settings().get_value('service_account')] if l_type.isDev(): to_who.append( settings().get_value('hypernews_test')) else: to_who.append( settings().get_value('dataops_announce' )) try: elem = (r_to_be_rejected + ds_to_be_invalidated)[0] sender = elem.current_user_email except IndexError: sender = None com.sendMail(to_who, 'Request and Datasets to be Invalidated', text, sender) for to_announce in itertools.chain(r_to_be_rejected, ds_to_be_invalidated): to_announce.set_announced() idb.update(to_announce.json()) if clear and (len(ds_to_be_invalidated)!=0 or len(r_to_be_rejected)!=0): for to_announce in itertools.chain(r_to_be_rejected, ds_to_be_invalidated): to_announce.set_announced() idb.update(to_announce.json()) return html
notification( subject, message, [], group=notification.REQUEST_OPERATIONS, action_objects=[request_prepid], object_type='requests', base_object=mcm_request ) mcm_request.notify(subject, message) return validations_count < max_validations def main(self): # First we check if some of the submitted jobs already finished to process them. if self.ssh_exec is None: sys.exit(-1) self.monitor_submmited_jobs() if not self.is_condor_working: # Sometimes htcondor is not working so we stop everything return self.new_jobs = self.get_new_request_prepids() + self.get_new_chain_prepids() prepids_to_submit = self.get_prepids_to_submit() self.submit_jobs(prepids_to_submit) self.save_jobs() if __name__ == "__main__": validation_handler = ValidationHandler() validation_handler.main() com = communicator() com.flush(0)