def put(self): values = json.loads(request.values['values']) if 'timestamp' in request.values: timestamp = parse_json_datetime(request.values['timestamp']) # check for drift delta = datetime.datetime.utcnow() - timestamp drift = delta.total_seconds() # print 'drift', drift if abs(drift) > 30: # get current controller correction # fix(later): support user updates as well? auth = request.authorization key = find_key_fast(auth.password) # key is provided as HTTP basic auth password if key and key.access_as_controller_id: controller_id = key.access_as_controller_id controller_status = ControllerStatus.query.filter(ControllerStatus.id == controller_id).one() attributes = json.loads(controller_status.attributes) correction = attributes.get('timestamp_correction', 0) # if stored correction is reasonable, use it; otherwise store new correction if abs(correction - drift) > 100: correction = drift attributes['timestamp_correction'] = drift controller_status.attributes = json.dumps(attributes) db.session.commit() # print 'storing new correction (%.2f)' % correction else: pass # print 'applying previous correction (%.2f)' % correction timestamp += datetime.timedelta(seconds=correction) else: timestamp = datetime.datetime.utcnow() # for now, assume all sequences in same folder items = list(values.items()) if items: items = sorted(items) # sort by keys so we can re-use folder lookup and permission check between items in same folder folder_resource = None folder_name = None for (full_name, value) in items: item_folder_name = full_name.rsplit('/', 1)[0] if item_folder_name != folder_name: # if this folder doesn't match the folder resource record we have folder_name = item_folder_name folder_resource = find_resource(folder_name) if folder_resource and access_level(folder_resource.query_permissions()) < ACCESS_LEVEL_WRITE: folder_resource = None # don't have write access if folder_resource: seq_name = full_name.rsplit('/', 1)[1] try: resource = ( Resource.query .filter(Resource.parent_id == folder_resource.id, Resource.name == seq_name, not_(Resource.deleted)) .one() ) update_sequence_value(resource, full_name, timestamp, str(value), emit_message=True) # fix(later): revisit emit_message except NoResultFound: pass db.session.commit()
def worker_log(worker_name, message): print(worker_name + ': ' + message) name = '/system/worker/log' log_resource = find_resource(name) if log_resource: update_sequence_value(log_resource, name, datetime.datetime.utcnow(), str(worker_name + ': ' + message)) # convert unicode to plain string else: print('worker log (%s) missing' % name)
def on_message(client, userdata, msg): # pylint: disable=unused-argument # print('MQTT: %s %s' % (msg.topic, msg.payload.decode())) message_struct = json.loads(msg.payload.decode()) message_type = message_struct['type'] if message_type == 'update_sequence': controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: parameters = message_struct['parameters'] seq_name = parameters['sequence'] if not seq_name.startswith('/'): # handle relative sequence names resource = Resource.query.filter(Resource.id == controller.id).one() # this is ok for now since .. doesn't have special meaning in resource path (no way to escape controller folder) seq_name = resource.path() + '/' + seq_name timestamp = parameters.get('timestamp', '') # fix(soon): need to convert to datetime if not timestamp: timestamp = datetime.datetime.utcnow() value = parameters['value'] if 'encoded' in parameters: value = base64.b64decode(value) # remove this; require clients to use REST POST for images resource = find_resource(seq_name) if not resource: return system_attributes = json.loads(resource.system_attributes) if resource.system_attributes else None if system_attributes and system_attributes['data_type'] == Resource.IMAGE_SEQUENCE: value = base64.b64decode(value) else: value = str(value) # don't emit message since the message is already in the system update_sequence_value(resource, seq_name, timestamp, value, emit_message=False) db.session.commit() # update controller watchdog status elif message_type == 'watchdog': controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: controller_status = ControllerStatus.query.filter(ControllerStatus.id == controller.id).one() controller_status.last_watchdog_timestamp = datetime.datetime.utcnow() db.session.commit() # send emails elif message_type == 'send_email': controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: print('sending email') handle_send_email(controller.id, message_struct['parameters']) # send SMS messages elif message_type == 'send_sms' or message_type == 'send_text_message': controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: handle_send_text_message(controller.id, message_struct['parameters'])
def put(self): start_time = time.time() values = json.loads(request.values['values']) if 'timestamp' in request.values: timestamp = parse_json_datetime(request.values['timestamp']) # check for drift delta = datetime.datetime.utcnow() - timestamp drift = delta.total_seconds() #print 'drift', drift if abs(drift) > 30: # get current controller correction # fix(later): support user updates as well? auth = request.authorization start_key_time = time.time() key = find_key_fast(auth.password) # key is provided as HTTP basic auth password end_key_time = time.time() #print '---- key: %.2f' % (end_key_time - start_key_time) if key and key.access_as_controller_id: controller_id = key.access_as_controller_id controller_status = ControllerStatus.query.filter(ControllerStatus.id == controller_id).one() attributes = json.loads(controller_status.attributes) correction = attributes.get('timestamp_correction', 0) # if stored correction is reasonable, use it; otherwise store new correction if abs(correction - drift) > 100: correction = drift attributes['timestamp_correction'] = drift controller_status.attributes = json.dumps(attributes) db.session.commit() #print 'storing new correction (%.2f)' % correction else: pass #print 'applying previous correction (%.2f)' % correction timestamp += datetime.timedelta(seconds=correction) else: timestamp = datetime.datetime.utcnow() # for now, assume all sequences in same folder first_name = values.iterkeys().next() folder_name = first_name.rsplit('/', 1)[0] folder_resource = find_resource(folder_name) if folder_resource: # and access_level(folder_resource.query_permissions()) >= ACCESS_LEVEL_WRITE: for (full_name, value) in values.iteritems(): seq_name = full_name.rsplit('/', 1)[1] try: resource = Resource.query.filter(Resource.parent_id == folder_resource.id, Resource.name == seq_name, Resource.deleted == False).one() update_sequence_value(resource, full_name, timestamp, str(value), emit_message=False) # fix(later): revisit emit_message except NoResultFound: pass db.session.commit() end_time = time.time()
def process_web_socket_message(message_struct, ws_conn): message_type = message_struct['type'] message_debug = False # handle new connection; no longer used; websocket should be authenticated using HTTP basic auth if message_type == 'connect': pass # handle watchdog message (updates controller status record) elif message_type == 'watchdog': if ws_conn.controller_id: controller_status = ControllerStatus.query.filter( ControllerStatus.id == ws_conn.controller_id).one() controller_status.last_watchdog_timestamp = datetime.datetime.utcnow( ) db.session.commit() # handle ping (does nothing; used to keep connection active) elif message_type == 'ping': pass # handle subscription (used to subscribe to messages from one or more folders) elif message_type == 'subscribe': # process the subscription parameters = message_struct['parameters'] subscriptions = parameters.get('subscriptions', []) for subscription in subscriptions: # fix(clean): remove support for folder IDs and old message args folder_path = subscription.get('folder', subscription.get('folderId', None)) message_type = subscription.get( 'message_type', subscription.get('messageType', None)) include_children = subscription.get( 'include_children', subscription.get('includeChildren', False)) # fix(clean): remove "[self]" option if folder_path == 'self' or folder_path == '[self]': folder_id = ws_conn.controller_id elif hasattr(folder_path, 'strip'): resource = find_resource(folder_path) if not resource: print('unable to find subscription folder: %s' % folder_path) return folder_id = resource.id else: folder_id = folder_path # if subscription is allowed, store it # fix(later): send a message back if not allowed if ws_conn.access_level(folder_id) >= ACCESS_LEVEL_READ: if message_debug: print('subscribe folder: %s (%d), message type: %s' % (folder_path, folder_id, message_type)) ws_conn.subscriptions.append( MessageSubscription(folder_id, message_type, include_children=include_children)) # fix(soon): remove this case after clients are updated elif message_type == 'setNode' or message_type == 'updateSequence' or message_type == 'update_sequence': if ws_conn.controller_id: parameters = message_struct['parameters'] if message_type == 'setNode': # fix(soon): remove this case seq_name = parameters['node'] else: seq_name = parameters['sequence'] if not seq_name.startswith('/'): # handle relative sequence names resource = Resource.query.filter( Resource.id == ws_conn.controller_id).one() # this is ok for now since .. doesn't have special meaning in resource path (no way to escape controller folder) seq_name = resource.path() + '/' + seq_name timestamp = parameters.get( 'timestamp', '') # fix(soon): need to convert to datetime if not timestamp: timestamp = datetime.datetime.utcnow() value = parameters['value'] if 'encoded' in parameters: value = base64.b64decode(value) # remove this; require clients to use REST POST for images resource = find_resource(seq_name) if not resource: return system_attributes = json.loads( resource.system_attributes ) if resource.system_attributes else None if system_attributes and system_attributes[ 'data_type'] == Resource.IMAGE_SEQUENCE: value = base64.b64decode(value) else: value = str(value) update_sequence_value(resource, seq_name, timestamp, value) db.session.commit() # update a resource elif message_type == 'write_resource': parameters = message_struct['parameters'] if 'path' and 'data' in parameters: path = parameters['path'] if not path.startswith( '/'): # fix(soon): remove this after clients updated path = '/' + path data = parameters['data'] resource = find_resource(path) if resource: if ws_conn.access_level(resource.id) >= ACCESS_LEVEL_WRITE: timestamp = datetime.datetime.utcnow() update_sequence_value(resource, path, timestamp, data) db.session.commit() else: socket_sender.send_error(ws_conn, 'permission error: %s' % path) else: socket_sender.send_error(ws_conn, 'resource not found: %s' % path) else: socket_sender.send_error( ws_conn, 'expected data and path parameters for write_resource message') # handle request for detailed message logging elif message_type == 'debug_messaging': parameters = message_struct['parameters'] enable = bool(parameters['enable']) level = logging.DEBUG if enable else logging.INFO logging.getLogger().setLevel(level) # handle other action messages elif message_type in ('sendEmail', 'sendTextMessage', 'send_email', 'send_text_message'): if ws_conn.controller_id: # only support these messages from controllers, not browsers if message_type == 'sendEmail' or message_type == 'send_email': handle_send_email(ws_conn.controller_id, message_struct['parameters']) elif message_type == 'sendTextMessage' or message_type == 'send_text_message': handle_send_text_message(ws_conn.controller_id, message_struct['parameters']) # for other types, assume that we want to create a message record else: # figure out target folder if 'folder' in message_struct: folder_name = message_struct['folder'] if message_debug: print('message to folder: %s' % folder_name) if hasattr(folder_name, 'startswith') and folder_name.startswith('/'): if message_debug: print('message to folder name: %s' % folder_name) folder = find_resource(folder_name) # assumes leading slash if folder: folder_id = folder.id if message_debug: print('message to folder id: %d' % folder_id) else: print('message to unknown folder (%s)' % folder_name) return else: folder_id = folder_name # fix(soon): remove this case elif ws_conn.controller_id: folder_id = ws_conn.controller_id else: print('message (%s) without folder or controller; discarding' % message_type) return # if allowed, create a message for the folder if ws_conn.access_level(folder_id) >= ACCESS_LEVEL_WRITE: parameters = message_struct['parameters'] # fix(soon): can we move this spawn above access level check (might require request context) Thread(target=message_queue.add, daemon=True, args=[folder_id, None, message_type, parameters], kwargs={ 'sender_controller_id': ws_conn.controller_id, 'sender_user_id': ws_conn.user_id }).start()
def on_message(client, userdata, msg): # pylint: disable=unused-argument payload = msg.payload.decode() # handle full (JSON) messages if payload.startswith('{'): message_struct = json.loads(payload) for message_type, parameters in message_struct.items(): # update sequence values; doesn't support image sequence; should use REST API for image sequences if message_type == 'update': folder = find_resource( '/' + msg.topic ) # for now we assume these messages are published on controller channels if folder and folder.type in (Resource.BASIC_FOLDER, Resource.ORGANIZATION_FOLDER, Resource.CONTROLLER_FOLDER): timestamp = parameters.get('$t', '') if timestamp: timestamp = parse_json_datetime( timestamp ) # fix(soon): handle conversion errors else: timestamp = datetime.datetime.utcnow() for name, value in parameters.items(): if name != '$t': seq_name = '/' + msg.topic + '/' + name resource = find_resource(seq_name) if resource: # don't emit new message since UI will receive this message update_sequence_value(resource, seq_name, timestamp, value, emit_message=False) db.session.commit() # update controller watchdog status elif message_type == 'watchdog': controller = find_resource( '/' + msg.topic ) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: controller_status = ControllerStatus.query.filter( ControllerStatus.id == controller.id).one() controller_status.last_watchdog_timestamp = datetime.datetime.utcnow( ) db.session.commit() # send emails elif message_type == 'send_email': controller = find_resource( '/' + msg.topic ) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: print('sending email') handle_send_email(controller.id, parameters) # send SMS messages elif message_type == 'send_sms' or message_type == 'send_text_message': controller = find_resource( '/' + msg.topic ) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: handle_send_text_message(controller.id, parameters) # handle short (non-JSON) messages else: # print('MQTT: %s %s' % (msg.topic, payload)) if payload.startswith( 's,' ): # type 's' is "store and display new sequence value" parts = payload.split(',', 3) if len(parts) == 4: seq_name = '/' + msg.topic + '/' + parts[1] timestamp = parse_json_datetime( parts[2]) # fix(soon): handle conversion errors value = parts[3] resource = find_resource(seq_name) if resource and resource.type == Resource.SEQUENCE: # don't emit new message since UI will receive this message update_sequence_value(resource, seq_name, timestamp, value, emit_message=False) db.session.commit()
def process_web_socket_message(message_struct, ws_conn): type = message_struct['type'] message_debug = False # handle new connection (updates controller status record) if type == 'connect': # fix(soon): remove this parameters = message_struct['parameters'] print 'connect message' # clients/controllers should send authCode in connect message if 'authCode' in parameters: auth_code = parameters['authCode'] key = find_key_by_code(auth_code) if key and key.access_as_controller_id: controller_resource = Resource.query.filter( Resource.id == key.access_as_controller_id).one() # handle child controller if 'name' in parameters: key_resource = controller_resource controller_resource = None # look for a resource with the given name that is a child of the controller referenced by the key candidate_resources = Resource.query.filter( Resource.name == parameters['name'], Resource.deleted == False) for resource in candidate_resources: if resource.is_descendent_of(key_resource.id): controller_resource = resource break if not controller_resource: ws_conn.ws.close() print('unable to find child controller: %s' % parameters['name'] ) # fix(soon): what should we do in this case? return ws_conn.controller_id = controller_resource.id ws_conn.auth_method = 'authCode' try: controller_status = ControllerStatus.query.filter( ControllerStatus.id == ws_conn.controller_id).one() controller_status.last_connect_timestamp = datetime.datetime.utcnow( ) controller_status.client_version = parameters.get( 'version', None) db.session.commit() except NoResultFound: pass else: ws_conn.ws.close() print('invalid auth code' ) # fix(soon): what should we do in this case? # handle watchdog message (updates controller status record) elif type == 'watchdog': if ws_conn.controller_id: controller_status = ControllerStatus.query.filter( ControllerStatus.id == ws_conn.controller_id).one() controller_status.last_watchdog_timestamp = datetime.datetime.utcnow( ) db.session.commit() # handle ping (does nothing; used to keep connection active) elif type == 'ping': pass # handle subscription (used to subscribe to messages from one or more folders) elif type == 'subscribe': # process the subscription parameters = message_struct['parameters'] subscriptions = parameters.get('subscriptions', []) for subscription in subscriptions: folder_path = subscription.get( 'folder', subscription.get('folderId', None) ) # fix(clean): remove support for folder IDs and old message args message_type = subscription.get( 'message_type', subscription.get('messageType', None)) include_children = subscription.get( 'include_children', subscription.get('includeChildren', False)) # fix(clean): remove "[self]" option if folder_path == 'self' or folder_path == '[self]': folder_id = ws_conn.controller_id elif hasattr(folder_path, 'strip'): resource = find_resource(folder_path) if not resource: print('unable to find subscription folder: %s' % folder_path) return folder_id = resource.id else: folder_id = folder_path # if subscription is allowed, store it # fix(later): send a message back if not allowed if ws_conn.access_level(folder_id) >= ACCESS_LEVEL_READ: if message_debug: print('subscribe folder: %s (%d), message type: %s' % (folder_path, folder_id, message_type)) ws_conn.subscriptions.append( MessageSubscription(folder_id, message_type, include_children=include_children)) # fix(soon): remove this case after clients are updated elif type == 'setNode' or type == 'updateSequence' or type == 'update_sequence': if ws_conn.controller_id: parameters = message_struct['parameters'] if type == 'setNode': # fix(soon): remove this case seq_name = parameters['node'] else: seq_name = parameters['sequence'] if not seq_name.startswith('/'): # handle relative sequence names resource = Resource.query.filter( Resource.id == ws_conn.controller_id).one() seq_name = resource.path( ) + '/' + seq_name # this is ok for now since .. doesn't have special meaning in resource path (no way to escape controller folder) timestamp = parameters.get( 'timestamp', '') # fix(soon): need to convert to datetime if not timestamp: timestamp = datetime.datetime.utcnow() value = parameters['value'] if 'encoded' in parameters: value = base64.b64decode(value) # remove this; require clients to use REST POST for images resource = find_resource(seq_name) if not resource: return system_attributes = json.loads( resource.system_attributes ) if resource.system_attributes else None if system_attributes and system_attributes[ 'data_type'] == Resource.IMAGE_SEQUENCE: value = base64.b64decode(value) else: value = str(value) update_sequence_value(resource, seq_name, timestamp, value) db.session.commit() # update a resource elif type == 'write_resource': if 'path' and 'data' in parameters: path = parameters['path'] if not path.startswith( '/'): # fix(soon): remove this after clients updated path = '/' + path data = parameters['data'] resource = find_resource(path) if resource: if ws_conn.access_level(resource.id) >= ACCESS_LEVEL_WRITE: timestamp = datetime.datetime.utcnow() update_sequence_value(resource, path, timestamp, data) db.session.commit() else: socket_sender.send_error(ws_conn, 'permission error: %s' % path) else: socket_sender.send_error(ws_conn, 'resource not found: %s' % path) else: socket_sender.send_error( ws_conn, 'expected data and path parameters for write_resource message') # handle other action messages elif type in ('sendEmail', 'sendTextMessage', 'send_email', 'send_text_message'): if ws_conn.controller_id: # only support these messages from controllers, not browsers if type == 'sendEmail' or type == 'send_email': handle_send_email(ws_conn.controller_id, message_struct['parameters']) elif type == 'sendTextMessage' or type == 'send_text_message': handle_send_text_message(ws_conn.controller_id, message_struct['parameters']) # for other types, assume that we want to create a message record else: # figure out target folder if 'folder' in message_struct: folder_name = message_struct['folder'] if message_debug: print('message to folder: %s' % folder_name) if hasattr(folder_name, 'startswith') and folder_name.startswith('/'): if message_debug: print('message to folder name: %s' % folder_name) folder = find_resource(folder_name) # assumes leading slash if folder: folder_id = folder.id if message_debug: print('message to folder id: %d' % folder_id) else: print('message to unknown folder (%s)' % folder_name) return else: folder_id = folder_name # fix(soon): remove this case elif ws_conn.controller_id: folder_id = ws_conn.controller_id else: print('message (%s) without folder or controller; discarding' % type) return # if allowed, create a message for the folder if ws_conn.access_level(folder_id) >= ACCESS_LEVEL_WRITE: parameters = message_struct['parameters'] # fix(soon): can we move this spawn above access level check (might require request context) gevent.spawn(message_queue.add, folder_id, type, parameters, sender_controller_id=ws_conn.controller_id, sender_user_id=ws_conn.user_id)
def put(self, resource_path): r = find_resource('/' + resource_path) if not r: abort(404) # fix(later): revisit to avoid leaking file existance if access_level(r.query_permissions()) < ACCESS_LEVEL_WRITE: abort(403) args = request.values # update resource name/location if 'name' in args: new_name = args['name'] if new_name != r.name: try: Resource.query.filter(Resource.parent_id == r.parent_id, Resource.name == new_name, Resource.deleted == False).one() abort(400) # a resource already exists with this name except NoResultFound: pass r.name = new_name if 'parent' in args: parent_resource = find_resource(args['parent']) # expects leading slash if not parent_resource: abort(400) if access_level(parent_resource.query_permissions()) < ACCESS_LEVEL_WRITE: abort(403) try: Resource.query.filter(Resource.parent_id == parent_resource.id, Resource.name == r.name, Resource.deleted == False).one() abort(400) # a resource already exists with this name except NoResultFound: pass r.parent_id = parent_resource.id # update view if 'view' in args and current_user.is_authenticated: try: resource_view = ResourceView.query.filter(ResourceView.resource_id == r.id, ResourceView.user_id == current_user.id).one() resource_view.view = args['view'] except NoResultFound: resource_view = ResourceView() resource_view.resource_id = r.id resource_view.user_id = current_user.id resource_view.view = args['view'] db.session.add(resource_view) # update other resource metadata if 'user_attributes' in args: r.user_attributes = args['user_attributes'] if r.type == Resource.SEQUENCE: if 'data_type' in args or 'decimal_places' in args or 'max_history' in args or 'min_storage_interval' in args or 'units' in args: system_attributes = json.loads(r.system_attributes) if 'data_type' in args: system_attributes['data_type'] = args['data_type'] if args.get('decimal_places', '') != '': system_attributes['decimal_places'] = int(args['decimal_places']) # fix(later): safe convert if args.get('max_history', '') != '': system_attributes['max_history'] = int(args['max_history']) # fix(later): safe convert if args.get('units', '') != '': system_attributes['units'] = args['units'] if args.get('min_storage_interval', '') != '': system_attributes['min_storage_interval'] = int(args['min_storage_interval']) # fix(later): safe convert r.system_attributes = json.dumps(system_attributes) elif r.type == Resource.REMOTE_FOLDER: system_attributes = json.loads(r.system_attributes) if r.system_attributes else {} if 'remote_path' in args: system_attributes['remote_path'] = args['remote_path'] if 'controller_id' in args: system_attributes['controller_id'] = args['controller_id'] r.system_attributes = json.dumps(system_attributes) elif r.type == Resource.ORGANIZATION_FOLDER: system_attributes = json.loads(r.system_attributes) if r.system_attributes else {} if 'full_name' in args and args['full_name']: system_attributes['full_name'] = args['full_name'] r.system_attributes = json.dumps(system_attributes) elif r.type == Resource.CONTROLLER_FOLDER: if 'status' in args: try: controller_status = ControllerStatus.query.filter(ControllerStatus.id == r.id).one() status = json.loads(controller_status.attributes) status.update(json.loads(args['status'])) # add/update status (don't provide way to remove status fields; maybe should overwrite instead) controller_status.attributes = json.dumps(status) except NoResultFound: pass else: # fix(soon): remove this case if 'system_attributes' in args: r.system_attributes = args['system_attributes'] # note that this will overwrite any existing system attributes; client must preserve any that aren't modified # update resource contents/value if 'contents' in args or 'data' in args: # fix(later): remove contents option if 'contents' in args: data = args['contents'] else: data = str(args['data']) # convert unicode to regular string / fix(soon): revisit this timestamp = datetime.datetime.utcnow() if r.type == Resource.SEQUENCE: # fix(later): collapse these two cases? resource_path = resource.path() # fix(faster): don't need to use this if were given path as arg update_sequence_value(resource, resource_path, timestamp, data) else: add_resource_revision(r, timestamp, data) r.modification_timestamp = timestamp db.session.commit() return {'status': 'ok', 'id': r.id}