def put(self, id): ''' PUT /activity/<id> This method allows to update an event status for example to re-run after an error { status: <new_status> } ''' # User has to be authenticated current_user = self.get_current_user() if not current_user: self.userError("not authenticated") return # TODO: User's right to see an event # User that has send the event OR PI of the authority OR Admin # TODO: id must be a valid UUID if not id: self.userError("no event id provided") return try: data = escape.json_decode(self.request.body) except json.decoder.JSONDecodeError as e: logger.error(self.request.body) logger.exception("malformed request") self.set_status(400) self.finish( json.dumps({ "return": { "status": "error", "messages": "malformed request" } })) if not 'status' in data: self.userError("status of event is required") return try: activity = yield r.table('activity').get(id).run(self.dbconnection) event = Event(activity) event.previous_status = event.status event.status = data['status'] result = yield dispatch(self.dbconnection, event) self.set_status(200) self.finish(json.dumps({"result": result}, cls=myJSONEncoder)) except Exception as e: logger.exception("error in PUT activity") self.set_status(500) self.finish( json.dumps( {"return": { "status": "error", "messages": e.message }})) return
def confirmEmails(qConfirmEmails): """ Process Event and send an Email to the user to confirm his/her email """ # db connection is shared between threads dbconnection = connect() while True: try: event = Event(qConfirmEmails.get()) except Exception as e: logger.exception(e) logger.error("Problem with event: {}".format(e)) continue else: if event.notify: # We try to send the email only once event.notify = False dispatch(dbconnection, event) try: # Recipients # Status did NOT changed if event.status == event.previous_status: logger.warning("TODO: send specific emails with messages") recipients = set() url = s.web['url'] if s.web['port'] and s.web['port'] != 80: url = url +':'+ s.web['port'] # Look for the user email in the Event if event.object.type == ObjectType.USER: recipients.add(User(event.data)) elif event.object.type == ObjectType.AUTHORITY: for user in event.data['users']: recipients.add(User(user)) else: for user in event.data['pi_users']: recipients.add(User(user)) url = url+'/confirm/'+event.id subject, template = build_subject_and_template('confirm', event) buttonLabel = "Confirm Email" sendEmail(event, recipients, subject, template, url, buttonLabel) except Exception as e: import traceback traceback.print_exc() msg = "Error in event {} while trying to send a confirmation email: {}".format(event.id, e) logger.error(msg) event.logWarning(msg) continue finally: dispatch(dbconnection, event)
def add_pi_users(self, data, object, object_type): events = [] pi_users = [] # check if the users in the request are in the object for data_pi in data['pi_users']: if data_pi not in object['pi_users']: if isinstance(data_pi, dict): data_pi = data_pi['id'] pi_users.append(data_pi) if len(pi_users)>0: # create event add user to object pis try: event = Event({ 'action': EventAction.ADD, 'user': self.current_user['id'], 'object': {'type': object_type, 'id': object['id']}, 'data': {'type': DataType.PI, 'values': pi_users} }) except Exception as e: # TODO: we should log here # log.error("Can't create request....") logger.error("Can't create event in add_pi_users") logger.exception(e) else: events.append(event) return events
def remove_pi_users(self, data, object, object_type): events = [] pi_users = [] # check if the users in the object are int the delete request for object_pi in object['pi_users']: if object_pi not in data['pi_users']: if isinstance(object_pi, dict): object_pi = object_pi['id'] pi_users.append(object_pi) if len(pi_users)>0: # dispatch event remove pi from object try: event = Event({ 'action': EventAction.REMOVE, 'user': self.current_user['id'], 'object': { 'type': object_type, 'id': object['id'] }, 'data': { 'type': DataType.PI, 'values': pi_users } }) except Exception as e: # TODO: we should log here # log.error("Can't create request....") logger.error("Can't create event in add_pi_users") logger.exception(e) else: events.append(event) return events
def post(self, id=None, o=None): """ POST /leases { testbed: string, slice_id: string, start_time: int, end_time: int, duration: int } or list of leases [{ testbed: string, slice_id: string, start_time: int, end_time: int, duration: int }] :return: """ if not self.get_current_user(): self.userError('permission denied user not logged in') return if not self.request.body: self.userError("empty request") return try: data = escape.json_decode(self.request.body) except json.decoder.JSONDecodeError as e: self.userError("malformed request", e.msg) return evData = [] try: if isinstance(data, list): for l in data: d = yield self.processLease(l) evData.append(d) else: d = yield self.processLease(data) evData.append(d) event = Event({ 'action': EventAction.CREATE, 'user': self.get_current_user()['id'], 'object': { 'type': ObjectType.LEASE, 'id': None, }, 'data': evData }) res = yield dispatch(self.dbconnection, event) result = res['generated_keys'] except Exception as e: self.userError(e) self.write( json.dumps( { "result": "success", "events": result, "error": None, "debug": None, }, cls=myJSONEncoder))
def run(): """ """ signal.signal(signal.SIGINT, receive_signal) signal.signal(signal.SIGTERM, receive_signal) signal.signal(signal.SIGHUP, receive_signal) qEvents = Queue() threads = [] for y in range(1): t = threading.Thread(target=manageEvents, args=(qEvents,)) t.daemon = True threads.append(t) t.start() context = zmq.Context() socket = context.socket(zmq.SUB) socket.setsockopt_string(zmq.SUBSCRIBE, 'activity') socket.connect("tcp://localhost:6002") logger.info("[activity] Collecting updates from ZMQ bus for activity") while True: logger.debug("[activity]Change in activity feed") topic, zmqmessage = socket.recv_multipart() activity = pickle.loads(zmqmessage) logger.debug("[activity]{0}: {1}".format(topic, activity)) try: event = Event(activity['new_val']) logger.debug("[activity] Adding event %s to Events queue" % (event.id)) qEvents.put(event) #for debbuging purposes except Exception as e: import traceback traceback.print_exc() logger.exception(e) if 'new_val' in activity and 'id' in activity['new_val']: logger.error("[activity] Problem with event: {}".format(activity['new_val']['id'])) continue logger.critical("Service activity stopped") # waits for the thread to finish for x in threads: x.join()
def delete(self, id, o=None): """ DELETE /slices/<id> :return: """ if not self.get_current_user(): self.userError('permission denied user not logged in') return try: # Check if the user has the right to delete a slice s = yield r.table('slices').get(id).run(self.dbconnection) u = yield r.table('users').get(self.get_current_user()['id']).run( self.dbconnection) # Check if the user isAdmin admin = self.isAdmin() if not admin: if not self.get_current_user()['id'] in s['users'] and s[ 'authority'] not in u['pi_authorities']: self.userError("your user has no rights on slice: %s" % id) return except Exception: self.userError("not authenticated or project not specified") return try: event = Event({ 'action': EventAction.DELETE, 'user': self.get_current_user()['id'], 'object': { 'type': ObjectType.SLICE, 'id': id, } }) except AttributeError as e: self.userError("Can't create request", e) return except Exception as e: self.userError("Can't create request", e) return else: result = yield dispatch(self.dbconnection, event) self.write( json.dumps( { "result": "success", "events": result['generated_keys'], "error": None, "debug": None, }, cls=myJSONEncoder))
def get(self, hashing): msg = '' new_hashing = '' # If user not authenticated # compare the hash of the url with the hash in the users table #if not self.get_current_user(): if not self.get_current_user(): try: # Find the Event based on the hashing cursor = yield r.table('activity').filter( lambda ev: ev["data"]["hashing"] == hashing).run( self.application.dbconnection) while (yield cursor.fetch_next()): ev = yield cursor.next() event = Event(ev) #msg = "hashing=%s" % hashing #msg='compare hash user=%s' % user # Update the hashing of the user # For security reason, link sent by email can be used only once # Updating the hashing with a new one to perfom the Update Query onSubmit #feed = yield r.table('users').filter({"hashing": hashing}).update({'hashing':'yyy'}).run(self.application.dbconnection) new_hashing = str(uuid.uuid4()) event.data["hashing"] = new_hashing result = yield dispatch(self.application.dbconnection, event) except Exception as e: #import traceback #traceback.print_exc() logger.error(e) msg = "This link is not valid, please generate a new one." self.render(self.application.templates + "/password_forgot.html", message=msg) return self.render(self.application.templates + "/password.html", message=msg, new_hashing=new_hashing)
def delete(self, id, o=None): """ DELETE /projects/<id> :return: """ # Check if the user is PI of the Project try: p = yield r.table('projects').get(id).run(self.dbconnection) u = yield r.table('users').get(self.current_user['id']).run( self.dbconnection) # Check if the user isAdmin admin = self.isAdmin() if not admin: if not self.current_user['id'] in p['pi_users'] and p[ 'authority'] not in u['pi_authorities']: self.userError("your user has no rights on project: %s" % id) return except Exception: self.userError("not authenticated or project not specified") return try: event = Event({ 'action': EventAction.DELETE, 'user': self.current_user['id'], 'object': { 'type': ObjectType.PROJECT, 'id': id, } }) except AttributeError as e: self.userError("Can't create request", e) return except Exception as e: self.userError("Can't create request", e) return else: result = yield dispatch(self.dbconnection, event) self.write( json.dumps( { "result": "success", "events": result["generated_keys"], "error": None, "debug": None }, cls=myJSONEncoder))
def delete(self, id, o=None): """ DELETE /authorities/<id> :return: """ try: # Check if the user has the right to delete an authority, PI of an upper authority a = yield r.table('authorities').get(id).run(self.dbconnection) if not a: self.userError("this authority %s does not exist" % id) return # Check if the user isAdmin admin = self.isAdmin() if not admin: if self.current_user['id'] not in a['pi_users']: self.userError("your user has no rights on authority: %s" % id) return except Exception: import traceback traceback.print_exc() self.userError("not authenticated") return try: event = Event({ 'action': EventAction.DELETE, 'user': self.current_user['id'], 'object': { 'type': ObjectType.AUTHORITY, 'id': id, } }) except AttributeError as e: self.userError("Can't create request", e) return except Exception as e: self.userError("Can't create request", e) return else: result = yield dispatch(self.dbconnection, event) self.write( json.dumps( { "result": "success", "events": result["generated_keys"], "error": None, "debug": None }, cls=myJSONEncoder))
def delete(self, id, o=None): """ DELETE /leases/<id> :return: """ if not self.get_current_user(): self.userError('permission denied user not logged in') return # Check if the user is a member of the slice try: u = yield r.table('users').get(self.get_current_user()['id']).run( self.dbconnection) p = yield r.table('leases').get(id).run(self.dbconnection) if p['slice_id'] not in u['slices']: self.userError("your user is not a member of this slice: %s" % p['slice_id']) except Exception: self.userError("not authenticated or slice not specified") return try: event = Event({ 'action': EventAction.DELETE, 'user': self.get_current_user()['id'], 'object': { 'type': ObjectType.LEASE, 'id': id, } }) except AttributeError as e: self.userError("Can't create request", e) return except Exception as e: self.userError("Can't create request", e) return else: result = yield dispatch(self.dbconnection, event) self.write( json.dumps( { "result": "success", "events": result['generated_keys'], "error": None, "debug": None, }, cls=myJSONEncoder))
def handle_unhandled_activities(dbconnection): """ Process events that were not watched while Server process was not running :param dbconnection: :return: """ unhandled_events = events(dbconnection, status="NEW") for event in unhandled_events: try: event = Event(change['new_val']) except: logger.error("[myslice-router] Unhandled message was not correct event object {}".format(change)) else: channels = filter_channels(event) for channel in channels: sock.send_multipart([channel, pickle.dumps(change)])
def update_slice(self, data, slice): # update slice data only if it has changed # TODO: check what we can change # Update slice properties try: event = Event({ 'action': EventAction.UPDATE, 'user': self.get_current_user()['id'], 'object': { 'type': ObjectType.SLICE, 'id': slice['id'] }, 'data': data }) except Exception as e: # TODO: we should log here log.error("Can't create event") log.errpr(e) return False else: return event
def get(self, id): """ GET /confirm/id it allows to confirm an email address using the event id :return: """ try: ev = yield r.table('activity').get(id).run(self.application.dbconnection) if len(ev) != 1: raise ValueError("event id is not valid") event = Event(ev) event.setPending() dispatch(self.application.dbconnection, event) event.logInfo("Event is pending, a manager will validate your request") self.finish(json.dumps({"result": ["your email is confirmed"]}, cls=myJSONEncoder)) except Exception as e: self.userError("This link is not valid") return
def get(self, id): msg = "" try: ev = yield r.table('activity').get(id).run(self.application.dbconnection) if ev is None: raise ValueError("event id is not valid") event = Event(ev) event.setPending() event.logInfo("Event is pending, a manager will validate your request") msg = "Your email is confirmed. " msg += "A request has been sent to a manager. " msg += "You will be informed as soon as your account will be validated." # dispatch the updated event dispatch(self.application.dbconnection, event) except Exception as e: import traceback traceback.print_exc() msg = "This link is not valid" self.render(self.application.templates + "/confirm.html", message=msg)
def run(): """ """ signal.signal(signal.SIGINT, receive_signal) signal.signal(signal.SIGTERM, receive_signal) signal.signal(signal.SIGHUP, receive_signal) # db connection is shared between threads qAuthorityEvents = Queue() lock = threading.Lock() threads = [] for y in range(1): t = threading.Thread(target=manageAuthoritiesEvents, args=(lock, qAuthorityEvents)) t.daemon = True threads.append(t) t.start() if config.services['authorities']['sync']: for y in range(1): t = threading.Thread(target=syncAuthorities, args=(lock, )) t.daemon = True threads.append(t) t.start() context = zmq.Context() socket = context.socket(zmq.SUB) socket.setsockopt_string(zmq.SUBSCRIBE, 'authorities') socket.connect("tcp://localhost:6002") logger.info("[authorities] Collecting updates from ZMQ bus for activity") while True: logger.debug("[authorities]Change in authorities feed") topic, zmqmessage = socket.recv_multipart() activity = pickle.loads(zmqmessage) logger.debug("[authorities]{0}: {1}".format(topic, activity)) try: event = Event(activity['new_val']) logger.debug("[authorities] Add event %s to %s queue" % (event.id, event.object.type)) qAuthorityEvents.put(event) except Exception as e: import traceback traceback.print_exc() logger.exception(e) if 'new_val' in activity and 'id' in activity['new_val']: logger.error("[authorities] Problem with event: {}".format( activity['new_val']['id'])) else: logger.error( "[authorities] Event is malformed: {}".format(activity)) continue logger.critical("[authorities] Service authorities stopped") for x in threads: x.join()
def delete(self, id, o=None): """ DELETE /users/<id> :return: """ # A user has the right to delete his own account # Check if the current user is PI of the authority of the user # Or an upper authority current_user = self.get_current_user() if not current_user: self.userError("not authenticated") return try: # user id from DB user = None cursor = yield r.table('users') \ .filter({'id': id}) \ .run(self.dbconnection) while (yield cursor.fetch_next()): user = yield cursor.next() if not user: self.userError("this user %s does NOT exist" % id) return if current_user['id']!=id: # Check if the user isAdmin admin = self.isAdmin() a = yield r.table('authorities').get(user['authority']).run(self.dbconnection) if not admin: if a and current_user['id'] not in a['pi_users']: self.userError("your user has no rights on: %s" % id) return except Exception as e: self.userError(e) return try: event = Event({ 'action': EventAction.DELETE, 'user': current_user['id'], 'object': { 'type': ObjectType.USER, 'id': id, }, 'data': {'authority': user['authority']} }) except AttributeError as e: self.userError("Can't create request", e) return except Exception as e: self.userError("Can't create request", e) return else: result = yield dispatch(self.dbconnection, event) self.write(json.dumps( { "result": "success", "events": result["generated_keys"], "error": None, "debug": None }, cls=myJSONEncoder))
def run(): """ A Process that will manage Projects and Slices """ signal.signal(signal.SIGINT, receive_signal) signal.signal(signal.SIGTERM, receive_signal) signal.signal(signal.SIGHUP, receive_signal) # db connection is shared between threads qProjects = Queue() qSlices = Queue() lockProjects = threading.Lock() # threads threads = [] # projects manager for y in range(1): t = threading.Thread(target=manageProjects, args=(lockProjects, qProjects)) t.daemon = True threads.append(t) t.start() # projects sync if config.services['experiments']['sync']: for y in range(1): t = threading.Thread(target=syncProjects, args=(lockProjects, )) t.daemon = True threads.append(t) t.start() # slices manager for y in range(1): t = threading.Thread(target=manageSlices, args=(qSlices, )) t.daemon = True threads.append(t) t.start() # slices sync if config.services['experiments']['sync']: for y in range(1): t = threading.Thread(target=syncSlices) t.daemon = True threads.append(t) t.start() context = zmq.Context() socket = context.socket(zmq.SUB) socket.setsockopt_string(zmq.SUBSCRIBE, 'experiments') socket.connect("tcp://localhost:6002") logger.info("[emails] Collecting updates from ZMQ bus for activity") while True: logger.debug("[emails]Change in emails feed") topic, zmqmessage = socket.recv_multipart() activity = pickle.loads(zmqmessage) logger.debug("[emails]{0}: {1}".format(topic, activity)) try: # if activity['new_val']['status'] in ["WAITING","APPROVED"]: event = Event(activity['new_val']) if event.object.type == ObjectType.PROJECT: logger.debug("Add event %s to %s queue" % (event.id, event.object.type)) qProjects.put(event) if event.object.type == ObjectType.SLICE: logger.debug("Add event %s to %s queue" % (event.id, event.object.type)) qSlices.put(event) except Exception as e: import traceback traceback.print_exc() logger.exception(e) if 'new_val' in activity and 'id' in activity['new_val']: logger.error("Problem with event: {}".format( activity['new_val']['id'])) continue logger.critical("Service experiments stopped") # waits for the thread to finish for x in threads: x.join()
def events_run(qSliceEvents): """ Process the slice after approval """ logger.info("Worker slices events starting") while True: try: event = Event(qSliceEvents.get()) except Exception as e: logger.error("Problem with event: {}".format(e)) event.logError(str(e)) event.setError() db.dispatch(dbconnection, event) continue else: logger.info("Processing event {} from user {}".format(event.id, event.user)) with lock: try: event.setRunning() isSuccess = False u = User(db.get(dbconnection, table='users', id=event.user)) user_setup = UserSetup(u, myslicelibsetup.endpoints) if event.creatingObject(): sli = Slice(event.data) # XXX To be checked sli.id = event.object.id # Add all Project PIs to the Slice project = db.get(dbconnection, table='projects', id=sli.project) for us in project['pi_users']: us = User(db.get(dbconnection, table='users', id=us)) sli.addUser(us) if 'users' in event.data and 'geni_users' not in event.data: for u_id in event.data['users']: u = User(db.get(dbconnection, table='users', id=u_id)) sli.addUser(u) # Take into account the Resources on Create if 'resources' in event.data: for resource in event.data['resources']: if not isinstance(resource, dict): res = db.get(dbconnection, table='resources', id=resource) if not res: raise Exception("Resource %s doesn't exist" % resource) resource = res sli.addResource(Resource(resource)) # expiration_date = Renew at AMs isSuccess = sli.save(dbconnection, user_setup) if event.updatingObject(): logger.debug("Update users / resources of Slice %s" % event.object.id) sli = Slice(db.get(dbconnection, table='slices', id=event.object.id)) ## adding users sli = add_users(event.data, sli) ## removing users sli = remove_users(event.data, sli) ## adding resources sli = add_resources(event.data, sli) ## removing resources sli = remove_resources(event.data, sli) isSuccess = sli.save(dbconnection, user_setup) if event.deletingObject(): sli = Slice(db.get(dbconnection, table='slices', id=event.object.id)) if not sli: raise Exception("Slice doesn't exist") isSuccess = sli.delete(dbconnection, user_setup) if event.addingObject(): sli = Slice(db.get(dbconnection, table='slices', id=event.object.id)) if event.data.type == DataType.USER: for val in event.data.values: if isinstance(val, dict): val = val['id'] u = User(db.get(dbconnection, table='users', id=val)) sli.addUser(u) isSuccess = sli.save(dbconnection, user_setup) if event.data['type'] == DataType.RESOURCE: for val in event.data.values: if isinstance(val, dict): val = val['id'] res = Resource(db.get(dbconnection, table='resources', id=val)) sli.addResource(res) isSuccess = sli.save(dbconnection, user_setup) if event.removingObject(): sli = Slice(db.get(dbconnection, table='slices', id=event.object.id)) if event.data.type == DataType.USER: for val in event.data['values']: if isinstance(val, dict): val = val['id'] u = User(db.get(dbconnection, table='users', id=val)) sli.removeUser(u) isSuccess = sli.save(dbconnection, user_setup) if event.data.type == DataType.RESOURCE: for val in event.data.values: if isinstance(val, dict): val = val['id'] r = Resource(db.get(dbconnection, table='resources', id=val)) sli.removeResource(r) isSuccess = sli.save(dbconnection, user_setup) except SliceException as e: # CREATE, UPDATE, DELETE # Calls toward Registry # If an AM sends an Error it is not blocking if event.creatingObject() or event.updatingObject() or event.deletingObject(): logger.debug("DEBUG services worker slices") for err in e.stack: logger.debug(err) if err['type'] == 'Reg': event.setError() break else: # XXX TO BE REFINED event.setSuccess() #event.setWarning() # TODO: # if ALL AMs have failed -> Error # if One AM succeeded -> Warning else: # XXX TO BE REFINED logger.debug("DEBUG services worker slices") for err in e.stack: logger.debug(err) event.setWarning() #event.setError() continue except Exception as e: import traceback traceback.print_exc() logger.error("Problem with event: {}".format(e)) event.logError(str(e)) event.setError() continue else: if isSuccess: event.setSuccess() else: event.setError() db.dispatch(dbconnection, event)
def post(self): """ POST /authorities { 'name': 'Test Authority', 'shortname': 'test_authority' } :return: """ if not self.request.body: self.userError("empty request") return try: data = escape.json_decode(self.request.body) except json.decoder.JSONDecodeError as e: self.userError("Malformed request", e) return except Exception as e: self.userError("Malformed request", e) return if data.get('authority', None) is None: data['authority'] = self.get_root_auth() if not data['authority']: self.userError("authority must be specified") return if data.get('name', None) is None: self.userError("Authority name must be specified") return if data.get('shortname', None) is None: self.userError("Authority shortname must be specified") return if not self.get_current_user() and 'users' not in data: self.userError("users of the new Authority must be specified") return # if new users are specified to be added to the authority if len(data.get('users', [])) > 0: for u in data['users']: if not isinstance(u, dict): self.userError( "New user properties under a new authority must be sent as dict" ) return if u.get('first_name', None) is None: self.userError("User first_name must be specified") return if u.get('last_name', None) is None: self.userError("User last_name must be specified") return if u.get('password', None) is None: self.userError("User password must be specified") return if len(u['password']) < 8: self.userError("Password must be at least 8 characters") return # password must be encrypted before storing into DB u['password'] = crypt_password(u['password']) if u.get('email', None) is None: self.userError("User email must be specified") return if not self.isEmail(u['email']): self.userError("Wrong Email address") return if u.get('terms', False) is False: self.userError("User must accept terms and conditions") return # if pi_users are specified to manage the authority if len(data.get('pi_users', [])) > 0: for u in data['pi_users']: # if pi_user is a New User we need at least his/her email if isinstance(u, dict): if u.get('email', None) is None: self.userError( "email of a new pi_user must be specified") return if not self.isEmail(u['email']): self.userError("Wrong Email address") return if not any([ user['email'] == u['email'] for user in data['users'] ]): self.userError( "email %s of pi_user is not among new users" % u['email']) return if not self.get_current_user(): # authority registration for new user current_user_id = None else: # admin create user directly current_user_id = self.get_current_user()['id'] try: event = Event({ 'action': EventAction.CREATE, 'user': current_user_id, 'object': { 'type': ObjectType.AUTHORITY, 'id': None }, 'data': data }) except Exception as e: self.userError("Can't create request", e.message) return else: result = yield dispatch(self.dbconnection, event) self.write( json.dumps( { "result": "success", "events": result["generated_keys"], "error": None, "debug": None }, cls=myJSONEncoder))
def events_run(lock, qLeasesEvents): """ Process the leases events """ logger.info("Worker leases events starting") # db connection is shared between threads dbconnection = connect() while True: try: event = Event(qLeasesEvents.get()) except Exception as e: logger.error("Problem with event: {}".format(e)) event.logError("Error in worker leases: {}".format(e)) event.setError() db.dispatch(dbconnection, event) continue else: logger.info("Processing event from user {}".format(event.user)) try: event.setRunning() event.logInfo("Event is running") logger.debug("Event %s is running" % event.id) isSuccess = False u = User(db.get(dbconnection, table='users', id=event.user)) user_setup = UserSetup(u, myslicelibsetup.endpoints) if event.creatingObject(): leases = [] if isinstance(event.data, list): for l in event.data: slice_id = l['slice_id'] leases.append(Lease(l)) else: slice_id = event.data['slice_id'] leases.append(Lease(event.data)) sli = Slice( db.get(dbconnection, table='slices', id=slice_id)) if not sli: raise Exception("Slice doesn't exist") for lease in leases: for val in lease.resources: r = db.get(dbconnection, table='resources', id=val) # Add resource only if it exists in DB if r is not None: r = Resource(r) sli.addResource(r) else: r = Resource({'id': val}) lease.removeResource(r) if len(lease.resources) > 0: sli.addLease(lease) else: raise Exception("Invalid resources") isSuccess = sli.save(dbconnection, user_setup) if event.deletingObject(): lease = Lease( db.get(dbconnection, table='leases', id=event.object.id)) if not lease: raise Exception("Lease doesn't exist") sli = Slice( db.get(dbconnection, table='slices', id=event.data['slice_id'])) if not sli: raise Exception("Slice doesn't exist") for val in event.data['resources']: r = Resource( db.get(dbconnection, table='resources', id=val)) # Remove resource only if it exists in DB if r: sli.removeResource(r) else: r = Resource({'id': val}) lease.removeResource(r) sli.removeLease(lease) isSuccess = sli.save(dbconnection, user_setup) except SliceException as e: # CREATE, DELETE # If at least one of the AMs replies with success, it's ok # If all AMs have failed -> Error for err in e.stack: event.logError("Error in worker leases: {}".format(err)) logger.error("Error in worker leases: {}".format(err)) # XXX TO BE REFINED event.setError() continue except SliceWarningException as e: for err in e.stack: event.logWarning(str(err)) logger.warning(str(err)) event.setWarning() continue except Exception as e: import traceback traceback.print_exc() logger.error("Problem with event {}: {}".format(event.id, e)) event.logError("Error in worker leases: {}".format(e)) event.setError() continue else: if isSuccess: event.setSuccess() event.logInfo("Event success") logger.debug("Event %s Success" % event.id) else: logger.error("Error event {}: action failed".format( event.id)) event.setError() event.logError("Error in worker leases: action failed") db.dispatch(dbconnection, event)
def put(self): """ PUT /profile :return: """ try: data = escape.json_decode(self.request.body) except json.decoder.JSONDecodeError as e: self.userError("malformed request", e) return except Exception as e: self.userError("malformed request", e) return # filter fields try: for key in data: if key not in ['first_name', 'last_name', 'bio', 'url', 'generate_keys']: raise KeyError except Exception as e: self.userError("malformed request", e) return user_id = self.get_current_user()['id'] try: event = Event({ 'action': EventAction.UPDATE, 'user': user_id , 'object': { 'type': ObjectType.USER, 'id': user_id , }, 'data': data }) except Exception as e: self.userError("problem with request", e) return else: activity = yield dispatch(self.dbconnection, event) feed = yield changes(self.dbconnection, table='activity', status=['ERROR', 'SUCCESS'], id = activity['generated_keys'][0] ) while (yield feed.fetch_next()): result = yield feed.next() if result['new_val']['status'] == 'SUCCESS': feed = yield r.table('users') \ .pluck(self.fields['profile']) \ .filter({'id': user_id}) \ .merge(lambda user: { 'authority': r.table('authorities').get(user['authority']) \ .pluck(self.fields_short['authorities']) \ .default({'id': user['authority']}) }) \ .run(self.dbconnection) yield feed.fetch_next() profile = yield feed.next() self.finish(json.dumps({"result": profile}, cls=myJSONEncoder)) else: self.userError("updated failed", result['new_val']) return
def emails_run(qEmails): """ Process Requests and send Emails accordingly """ # db connection is shared between threads dbconnection = connect() while True: try: event = Event(qEmails.get()) except Exception as e: logger.error("Problem with event: {}".format(e)) continue else: if event.notify: # We try to send the email only once event.notify = False dispatch(dbconnection, event) # Recipients # TODO: Send specific emails # Status did NOT changed # Comments about an event with a message if event.status == event.previous_status: logger.warning("TODO: send specific emails with messages") recipients = set() url = s.web['url'] if s.web['port'] and s.web['port'] != 80: url = url +':'+ s.web['port'] buttonLabel = "View details" if event.object.type == ObjectType.PASSWORD: recipients.add(User(event.object.id)) url = url+'/password/'+event.data['hashing'] subject, template = build_subject_and_template('password', event) buttonLabel = "Change password" else: if event.isPending(): # Find the authority of the event object # Then according the authority, put the pi_emails in pis_email try: authority_id = event.data['authority'] except KeyError: msg = 'Authority id not specified ({})'.format(event.id) logger.error(msg) event.logWarning('Authority not specified in event {}, email not sent'.format(event.id)) pass else: authority = Authority(db.get(dbconnection, table='authorities', id=authority_id)) if not authority: # get admin users users = db.get(dbconnection, table='users') for u in users: user = User(u) if user.isAdmin(): logger.debug("user %s is admin" % user.id) recipients.add(user) else: for pi_id in authority.pi_users: recipients.add(User(pi_id)) if not recipients: msg = 'Emails cannot be sent because no one is the PI of {}'.format(event.object.id) logger.error(msg) event.logWarning('No recipients could be found for event {}, email not sent'.format(event.id)) subject, template = build_subject_and_template('request', event) buttonLabel = "Approve / Deny" url = url + '/activity' else: if event.user: recipients.add(User(event.user)) else: # Look for the user email in the Event if event.object.type == ObjectType.USER: recipients.add(User({'email':event.data['email'], 'first_name':event.data['first_name'], 'last_name':event.data['last_name']})) elif event.object.type == ObjectType.AUTHORITY: for user in event.data['users']: recipients.add(User(user)) else: for user in event.data['pi_users']: recipients.add(User(user)) if event.isSuccess(): subject, template = build_subject_and_template('approve', event) elif event.isDenied(): subject, template = build_subject_and_template('deny', event) try: sendEmail(event, recipients, subject, template, url, buttonLabel) except Exception as e: import traceback traceback.print_exc() msg = "Error in event {} while trying to send an email: {} {}".format(event.id, e, traceback.print_exc()) logger.error(msg) event.logWarning(msg) continue finally: dispatch(dbconnection, event)
def events_run(lock, qProjectEvents): """ Process the authority after approval """ logger.info("Worker authorities events starting") # db connection is shared between threads dbconnection = connect() while True: try: event = Event(qProjectEvents.get()) except Exception as e: logger.error("Problem with event: {}".format(e)) event.logError(str(e)) event.setError() db.dispatch(dbconnection, event) continue else: logger.info("Processing event: {} from user {}".format(event.id, event.user)) with lock: try: if event.isApproved(): user_id = event.manager else: user_id = event.user u = User(db.get(dbconnection, table='users', id=user_id)) # Registry Only user_setup = UserSetup(u, myslicelibsetup.registry_endpoints) event.setRunning() event.logInfo("Event is running") logger.debug("Event %s is running" % event.id) isSuccess = False if event.creatingObject() or event.updatingObject(): logger.info("creating or updating the object project {}".format(event.data)) proj = Project(event.data) proj.id = event.object.id # XXX CASES TO BE CHECKED if event.user not in proj.pi_users: pi = User(db.get(dbconnection, table='users', id=event.user)) proj.addPi(pi) isSuccess = proj.save(dbconnection, user_setup) else: p = db.get(dbconnection, table='projects', id=event.object.id) if not p: raise Exception("Project doesn't exist") proj = Project(p) if event.deletingObject(): logger.info("deleting the object project {}".format(event.object.id)) isSuccess = proj.delete(dbconnection, user_setup) if event.addingObject(): logger.info("adding data to the object project {}".format(event.object.id)) if event.data.type == DataType.USER: logger.info("Project only supports PI at the moment, need new feature in SFA Reg") if event.data.type == DataType.PI or event.data.type == DataType.USER: for val in event.data.values: pi = User(db.get(dbconnection, table='users', id=val)) proj.addPi(pi) isSuccess = proj.save(dbconnection, user_setup) if event.removingObject(): logger.info("removing data from the object project {}".format(event.object.id)) if event.data.type == DataType.USER: logger.info("Project only supports PI at the moment, need new feature in SFA Reg") if event.data.type == DataType.PI or event.data.type == DataType.USER: for val in event.data.values: pi = User(db.get(dbconnection, table='users', id=val)) proj.removePi(pi) isSuccess = proj.save(dbconnection, user_setup) except Exception as e: import traceback traceback.print_exc() logger.error("Problem with event {}: {}".format(event.id,e)) event.logError("Error in worker projects: {}, traceback: {}".format(e,traceback.print_exc())) event.setError() continue else: if isSuccess: event.setSuccess() event.logInfo("Event success") logger.debug("Event %s Success" % event.id) else: logger.error("Error event {}: action failed".format(event.id)) event.setError() event.logError("Error in worker projects: action failed") finally: db.dispatch(dbconnection, event)
def post(self, hashing=None): """ POST /password[/<hashing>] :return: """ if not self.request.body: self.userError("empty request") return try: data = escape.json_decode(self.request.body) except json.decoder.JSONDecodeError as e: self.userError("Malformed request", e) return except Exception as e: self.userError("Malformed request", e) return try: # NEW Event UPDATE PASSWORD if not hashing: if not 'email' in data: self.userError("Email not specified") return if not 'hashing' in data: data['hashing'] = str(uuid.uuid4()) user = None cursor = yield r.table('users').filter({ 'email': data['email'] }).run(self.dbconnection) while (yield cursor.fetch_next()): user = yield cursor.next() if not user: self.userError("no user found or permission denied") return event = Event({ 'action': EventAction.CREATE, 'user': None, 'object': { 'type': ObjectType.PASSWORD, 'id': user['id'] }, 'data': data }) # APPROVE Event UPDATE PASSWORD else: if not 'password' in data: self.userError("Password not specified") return # Find the Event based on the hashing cursor = yield r.table('activity').filter( lambda ev: ev["data"]["hashing"] == hashing).run( self.application.dbconnection) while (yield cursor.fetch_next()): ev = yield cursor.next() event = Event(ev) user = None cursor = yield r.table('users').filter({ 'email': event.data['email'] }).run(self.dbconnection) while (yield cursor.fetch_next()): user = yield cursor.next() if not user: self.userError("user not found or permission denied") return #user = User(user) # Crypt password new_password = crypt_password(data['password']) event.data['password'] = new_password event.setApproved() except Exception as e: self.userError("Can't create request", e) return else: result = yield dispatch(self.dbconnection, event) self.write( json.dumps({ "result": "success", "error": None, "debug": None }, cls=myJSONEncoder))
def run(): """ """ signal.signal(signal.SIGINT, receive_signal) signal.signal(signal.SIGTERM, receive_signal) signal.signal(signal.SIGHUP, receive_signal) threads = [] qEmails = Queue() for y in range(1): t = threading.Thread(target=manageEmails, args=(qEmails, )) t.daemon = True threads.append(t) t.start() qConfirmEmails = Queue() for y in range(1): t = threading.Thread(target=confirmEmails, args=(qConfirmEmails, )) t.daemon = True threads.append(t) t.start() context = zmq.Context() socket = context.socket(zmq.SUB) socket.setsockopt_string(zmq.SUBSCRIBE, 'emails') socket.connect("tcp://localhost:6002") logger.info("[emails] Collecting updates from ZMQ bus for activity") while True: logger.debug("[emails]Change in emails feed") topic, zmqmessage = socket.recv_multipart() activity = pickle.loads(zmqmessage) logger.debug("[emails]{0}: {1}".format(topic, activity)) try: event = Event(activity['new_val']) except Exception as e: logger.error("Problem with event: {}".format(e)) continue else: if event.isConfirm() and event.notify: logger.debug("Add event %s to Confirm Email queue" % (event.id)) qConfirmEmails.put(event) elif event.isPending() and event.notify: logger.debug("Add event %s to Email queue" % (event.id)) qEmails.put(event) elif event.isDenied() and event.notify: logger.info("event {} is denied".format(event.id)) logger.debug("Add event %s to Email queue" % (event.id)) qEmails.put(event) elif event.isSuccess() and event.notify: logger.debug("Add event %s to Email queue" % (event.id)) qEmails.put(event) logger.critical("Service emails stopped") # waits for the thread to finish for x in threads: x.join()
def put(self, id=None, o=None): """ PUT /authorities/<id> { authority object } :return: """ try: # Check if the user has the right to Update an authority a = yield r.table('authorities').get(id).run(self.dbconnection) if not a: self.userError("this authority %s does not exist" % id) return # only admin or pi? if self.current_user['id'] not in a[ 'pi_users'] and not self.isAdmin(): self.userError("your user has no rights on authority: %s" % id) return except Exception: import traceback traceback.print_exc() self.userError("not authenticated ") return events = [] response = [] current_user = self.get_current_user() if not current_user: self.userError('permission denied') return if not self.request.body: self.userError("empty request") return try: data = escape.json_decode(self.request.body) except json.decoder.JSONDecodeError as e: self.userError("malformed request", e.message) return # authority id from DB cursor = yield r.table('authorities') \ .filter({'id': id}) \ .run(self.dbconnection) while (yield cursor.fetch_next()): authority = yield cursor.next() # handle authority as dict if "authority" in data and type(data["authority"]) is dict: data["authority"] = data["authority"]["id"] # Slices should be under a project (Legacy slices under an authority) # handle slices as dict if "slices" in data and type(data["slices"]) is dict: data["slices"] = data["slices"]["id"] # Users belong to an Authority, it can NOT be changed # handled by POST /users & DELETE /users/<id> if "users" in data and type(data["users"]) is dict: data["users"] = data["users"]["id"] # Update authority data try: event = Event({ 'action': EventAction.UPDATE, 'user': current_user['id'], 'object': { 'type': ObjectType.AUTHORITY, 'id': id }, 'data': data }) except Exception as e: self.userError("Can't create request", e.message) return else: result = yield dispatch(self.dbconnection, event) response = response + result["generated_keys"] # handle pi_user as dict if all(isinstance(n, dict) for n in data['pi_users']): data['pi_users'] = [x['id'] for x in data['pi_users']] e = self.update_authority(data, authority) if e: events.append(e) # adding pi users events += self.add_pi_users(data, authority, ObjectType.AUTHORITY) # removing pi users events += self.remove_pi_users(data, authority, ObjectType.AUTHORITY) for e in events: result = yield dispatch(self.dbconnection, e) response = response + result['generated_keys'] ## # projects # This is handled by the POST /projects and DELETE /projects/<id> calls ## # users # This is handled by the POST /users and DELETE /users/<id> calls self.write( json.dumps( { "result": "success", "events": response, "error": None, "debug": None }, cls=myJSONEncoder))
def run(): """ """ signal.signal(signal.SIGINT, receive_signal) signal.signal(signal.SIGTERM, receive_signal) signal.signal(signal.SIGHUP, receive_signal) qUserEvents = Queue() qPasswordEvents = Queue() lock = threading.Lock() threads = [] for y in range(1): t = threading.Thread(target=manageUsersEvents, args=(lock, qUserEvents)) t.daemon = True threads.append(t) t.start() for y in range(1): t = threading.Thread(target=managePasswordEvents, args=(lock, qPasswordEvents)) t.daemon = True threads.append(t) t.start() if config.services['users']['sync']: for y in range(1): t = threading.Thread(target=syncUsers, args=(lock, )) t.daemon = True threads.append(t) t.start() context = zmq.Context() socket = context.socket(zmq.SUB) socket.setsockopt_string(zmq.SUBSCRIBE, 'users') socket.connect("tcp://localhost:6002") logger.info("[emails] Collecting updates from ZMQ bus for activity") while True: logger.debug("[users] Change in emails feed") topic, zmqmessage = socket.recv_multipart() activity = pickle.loads(zmqmessage) logger.debug("[users] {0}: {1}".format(topic, activity)) try: event = Event(activity['new_val']) if event.object.type == ObjectType.USER: logger.debug("Add event %s to %s queue" % (event.id, event.object.type)) qUserEvents.put(event) if event.object.type == ObjectType.PASSWORD: logger.debug("Add event %s to %s queue" % (event.id, event.object.type)) qPasswordEvents.put(event) except Exception as e: import traceback traceback.print_exc() logger.exception(e) if 'new_val' in activity and 'id' in activity['new_val']: logger.error("Problem with event: {}".format( activity['new_val']['id'])) continue logger.critical("Service users stopped") for x in threads: x.join()
def run(q): """ Manages newly created events """ logger.info("Worker activity events starting") # db connection is shared between threads dbconnection = connect() while True: try: event = Event(q.get()) except Exception as e: import traceback traceback.print_exc() logger.error("Problem with event: {}".format(e)) event.logError("Error in worker activity: {}".format(e)) event.setError() dispatch(dbconnection, event) continue else: logger.debug("%s - Manage event".format(event.id)) # TODO: if event.creatingObject() # Check if the event.object.id already exists or not # if it exists -> add a numer to the id & hrn to make it unique if event.object.type == ObjectType.PASSWORD: try: event.setPending() event.logInfo("Event is pending, please check your email") logger.debug("Event %s is pending" % event.id) except Exception as e: logger.error( "Error in processing Event (1): {}".format(event)) logger.error("Error while processing: {}".format(e)) event.logError(str(e)) continue # Register a new object for a new user # id should be generated into the web to avoid duplicates elif event.isNew() and event.object.type in [ ObjectType.USER, ObjectType.AUTHORITY ] and event.user is None and event.creatingObject(): try: # The user must confirm his/her email logger.debug("Event Type: {}".format(type(event))) event.setConfirm() logger.debug("Event %s is confirm" % event.id) event.logInfo( "Event is expecting your confirmation, please check your email" ) except Exception as e: logger.error( "Error in processing Event (2): {}".format(event)) logger.error("Error while processing: {}".format(e)) event.logError(str(e)) continue else: try: logger.debug("%s - get user %s" % (event.id, event.user)) db_user = db.get(dbconnection, table='users', id=event.user) if db_user: user = User(db_user) logger.debug("%s - Does user %s has privilege?" % (event.id, event.user)) if user.has_privilege(event): logger.debug("%s - setWaiting" % event.id) event.logInfo("Event waiting to be processed") event.setWaiting() else: logger.debug("%s - setPending" % event.id) event.logInfo( "your user has no rights on %s, event pending until approved" % event.user) event.setPending() else: event.setError() logger.error("User {} not found in event {}".format( event.user, event.id)) event.logError("User %s not found" % event.user) # raising an Exception here, blocks the REST API #raise Exception("User %s not found" % event.user) except Exception as e: logger.error("Error processing Event") logger.error(event) import traceback logger.error(traceback.print_exc()) traceback.print_exc() event.setError() event.logError(str(e)) logger.error("Unable to fetch the user {} from db".format( event.user)) logger.exception(e) continue # dispatch the updated event dispatch(dbconnection, event)
def events_run(lock, qPasswordEvents): """ Process the user after approval """ logger.info("Worker password events starting") dbconnection = connect() while True: try: event = Event(qPasswordEvents.get()) except Exception as e: logger.error("Problem with event: {}".format(e)) event.logError(str(e)) event.setError() db.dispatch(dbconnection, event) continue else: logger.info("Processing password event from user {}".format( event.user)) event.setRunning() event.logInfo("Event is running") logger.debug("Event %s is running" % event.id) ## # Creating a new password for user if event.creatingObject(): logger.info("Creating password {}".format(event.object.id)) try: cursor = db.users(dbconnection, email=event.data['email']) u = cursor.next() u['password'] = event.data['password'] db.users(dbconnection, u, u['id']) except Exception as e: logger.error( "Problem updating password of user: {} - {}".format( event.object.id, e)) event.logError(str(e)) event.setError() continue else: event.setSuccess() event.logInfo("Event success") logger.debug("Event %s Success" % event.id) ## # we then dispatch the event db.dispatch(dbconnection, event)