def worker(): # log that the worker process is starting worker_log('system', 'starting worker process') # start various worker threads gevent.spawn(controller_watchdog) gevent.spawn(sequence_truncator) # loop forever while True: # sleep for one second each loop gevent.sleep(1) # check for messages if False: messages = message_queue.receive() for message in messages: if message.type == 'start_worker_task': print('#### %s' % message.parameters) params = json.loads(message.parameters) if params['name'] == 'add_resource_revisions': print('#### starting add_resource_revisions') gevent.spawn(add_resource_revisions)
def on_connect(client, userdata, flags, rc): # pylint: disable=unused-argument if rc: worker_log('message_monitor', 'unable to connect to MQTT broker/server at %s:%d' % (mqtt_host, mqtt_port)) else: worker_log('message_monitor', 'connected to MQTT broker/server at %s:%d' % (mqtt_host, mqtt_port)) client.subscribe('#') # subscribe to all messages
def worker(blocking=True, debug=False): # log that the worker process is starting worker_log('system', 'starting worker process') # start various worker threads Thread(target=controller_watchdog, daemon=True).start() Thread(target=sequence_truncator, daemon=True).start() Thread(target=message_deleter, daemon=True).start() Thread(target=message_monitor, daemon=True).start() # loop forever while blocking: # sleep for one second each loop time.sleep(1) # check for messages if debug: messages = message_queue.receive() for message in messages: if message.type == 'start_worker_task': print('#### %s' % message.parameters) params = json.loads(message.parameters) if params['name'] == 'add_resource_revisions': print('#### starting add_resource_revisions')
def check_resource_migration(): worker_log( 'migrate_db', 'resources without org id: %d' % db.session.query(func.count(Resource.id)).filter( Resource.organization_id.is_(None)).scalar()) worker_log( 'migrate_db', 'file resources without last rev: %d' % db.session.query(func.count(Resource.id)).filter( Resource.type == Resource.FILE, Resource.last_revision_id.is_(None)).scalar())
def message_deleter(): worker_log('message_deleter', 'starting') while True: delete_count = 0 start_time = time.time() thresh = datetime.datetime.utcnow() - datetime.timedelta(hours=1) messages = Message.query.filter(Message.timestamp < thresh) delete_count = messages.count() messages.delete() db.session.commit() db.session.expunge_all() db.session.close() delete_time = time.time() - start_time # display diagnostic worker_log( 'message_deleter', 'deleted %d messages in %.3f seconds' % (delete_count, delete_time)) # sleep for 6 hours time.sleep(6 * 60 * 60)
def sequence_truncator(): verbose = True worker_log('sequence_truncator', 'starting') while True: truncate_count = 0 # loop over all sequences resources = Resource.query.filter(Resource.type == Resource.SEQUENCE) for resource in resources: # get number of revisions for this sequence rev_count = db.session.query(func.count( ResourceRevision.id)).filter( ResourceRevision.resource_id == resource.id).scalar() # get max history system_attributes = json.loads( resource.system_attributes ) if resource.system_attributes else {} max_history = system_attributes.get('max_history', 1) # if too many revisions (with 1000 item buffer), delete old ones # fix(later): revisit buffer for image sequences and others with large objects if rev_count > max_history + 1000: # determine timestamp of revision max_history records ago; # this could be made faster if we assumed that revisions are created sequentially revisions = ResourceRevision.query.with_entities( ResourceRevision.timestamp).filter( ResourceRevision.resource_id == resource.id).order_by( 'timestamp') boundary_timestamp = revisions[-max_history].timestamp # diagnostics if verbose: message = 'id: %s, path: %s, max hist: %d, revs: %d, first: %s, thresh: %s, last: %s' % ( resource.id, resource.path(), max_history, rev_count, revisions[0].timestamp.strftime('%Y-%m-%d'), boundary_timestamp.strftime('%Y-%m-%d'), revisions[-1].timestamp.strftime('%Y-%m-%d')) worker_log('sequence_truncator', message) # delete the old records # it is critical that we filter by resource ID and timestamp ResourceRevision.query.filter( ResourceRevision.resource_id == resource.id, ResourceRevision.timestamp < boundary_timestamp).delete() db.session.commit() truncate_count += 1 # display diagnostic if truncate_count: worker_log( 'sequence_truncator', 'done with truncation pass; truncated %d sequences' % truncate_count) # sleep for an hour gevent.sleep(60 * 60)
def message_monitor(): server_config = load_server_config() if 'MQTT_HOST' not in server_config: worker_log('message_monitor', 'MQTT host not configured') return worker_log('message_monitor', 'starting') mqtt_host = server_config['MQTT_HOST'] mqtt_port = server_config.get('MQTT_PORT', 443) # run this on connect/reconnect def on_connect(client, userdata, flags, rc): # pylint: disable=unused-argument if rc: worker_log( 'message_monitor', 'unable to connect to MQTT broker/server at %s:%d' % (mqtt_host, mqtt_port)) else: worker_log( 'message_monitor', 'connected to MQTT broker/server at %s:%d' % (mqtt_host, mqtt_port)) client.subscribe('#') # subscribe to all messages # run this on message def on_message(client, userdata, msg): # pylint: disable=unused-argument payload = msg.payload.decode() # handle full (JSON) messages if payload.startswith('{'): message_struct = json.loads(payload) for message_type, parameters in message_struct.items(): # update sequence values; doesn't support image sequence; should use REST API for image sequences if message_type == 'update': folder = find_resource( '/' + msg.topic ) # for now we assume these messages are published on controller channels if folder and folder.type in (Resource.BASIC_FOLDER, Resource.ORGANIZATION_FOLDER, Resource.CONTROLLER_FOLDER): timestamp = parameters.get('$t', '') if timestamp: timestamp = parse_json_datetime( timestamp ) # fix(soon): handle conversion errors else: timestamp = datetime.datetime.utcnow() for name, value in parameters.items(): if name != '$t': seq_name = '/' + msg.topic + '/' + name resource = find_resource(seq_name) if resource: # don't emit new message since UI will receive this message update_sequence_value(resource, seq_name, timestamp, value, emit_message=False) db.session.commit() # update controller watchdog status elif message_type == 'watchdog': controller = find_resource( '/' + msg.topic ) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: controller_status = ControllerStatus.query.filter( ControllerStatus.id == controller.id).one() controller_status.last_watchdog_timestamp = datetime.datetime.utcnow( ) db.session.commit() # send emails elif message_type == 'send_email': controller = find_resource( '/' + msg.topic ) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: print('sending email') handle_send_email(controller.id, parameters) # send SMS messages elif message_type == 'send_sms' or message_type == 'send_text_message': controller = find_resource( '/' + msg.topic ) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: handle_send_text_message(controller.id, parameters) # handle short (non-JSON) messages else: # print('MQTT: %s %s' % (msg.topic, payload)) if payload.startswith( 's,' ): # type 's' is "store and display new sequence value" parts = payload.split(',', 3) if len(parts) == 4: seq_name = '/' + msg.topic + '/' + parts[1] timestamp = parse_json_datetime( parts[2]) # fix(soon): handle conversion errors value = parts[3] resource = find_resource(seq_name) if resource and resource.type == Resource.SEQUENCE: # don't emit new message since UI will receive this message update_sequence_value(resource, seq_name, timestamp, value, emit_message=False) db.session.commit() # connect and run mqtt_client = mqtt.Client(transport='websockets') mqtt_client.on_connect = on_connect mqtt_client.on_message = on_message mqtt_client.username_pw_set( 'token', message_auth_token(0) ) # user_id 0 indicates that this is an internal connection from the server mqtt_client.tls_set() # enable SSL mqtt_client.connect(mqtt_host, mqtt_port) mqtt_client.loop_start() while True: gevent.sleep(60)
def message_monitor(): server_config = load_server_config() if 'MQTT_HOST' not in server_config: worker_log('message_monitor', 'MQTT host not configured') return worker_log('message_monitor', 'starting') mqtt_host = server_config['MQTT_HOST'] mqtt_port = server_config.get('MQTT_PORT', 443) # run this on connect/reconnect def on_connect(client, userdata, flags, rc): # pylint: disable=unused-argument if rc: worker_log('message_monitor', 'unable to connect to MQTT broker/server at %s:%d' % (mqtt_host, mqtt_port)) else: worker_log('message_monitor', 'connected to MQTT broker/server at %s:%d' % (mqtt_host, mqtt_port)) client.subscribe('#') # subscribe to all messages # run this on message def on_message(client, userdata, msg): # pylint: disable=unused-argument # print('MQTT: %s %s' % (msg.topic, msg.payload.decode())) message_struct = json.loads(msg.payload.decode()) message_type = message_struct['type'] if message_type == 'update_sequence': controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: parameters = message_struct['parameters'] seq_name = parameters['sequence'] if not seq_name.startswith('/'): # handle relative sequence names resource = Resource.query.filter(Resource.id == controller.id).one() # this is ok for now since .. doesn't have special meaning in resource path (no way to escape controller folder) seq_name = resource.path() + '/' + seq_name timestamp = parameters.get('timestamp', '') # fix(soon): need to convert to datetime if not timestamp: timestamp = datetime.datetime.utcnow() value = parameters['value'] if 'encoded' in parameters: value = base64.b64decode(value) # remove this; require clients to use REST POST for images resource = find_resource(seq_name) if not resource: return system_attributes = json.loads(resource.system_attributes) if resource.system_attributes else None if system_attributes and system_attributes['data_type'] == Resource.IMAGE_SEQUENCE: value = base64.b64decode(value) else: value = str(value) # don't emit message since the message is already in the system update_sequence_value(resource, seq_name, timestamp, value, emit_message=False) db.session.commit() # update controller watchdog status elif message_type == 'watchdog': controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: controller_status = ControllerStatus.query.filter(ControllerStatus.id == controller.id).one() controller_status.last_watchdog_timestamp = datetime.datetime.utcnow() db.session.commit() # send emails elif message_type == 'send_email': controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: print('sending email') handle_send_email(controller.id, message_struct['parameters']) # send SMS messages elif message_type == 'send_sms' or message_type == 'send_text_message': controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels if controller and controller.type == Resource.CONTROLLER_FOLDER: handle_send_text_message(controller.id, message_struct['parameters']) # connect and run mqtt_client = mqtt.Client(transport='websockets') mqtt_client.on_connect = on_connect mqtt_client.on_message = on_message mqtt_client.username_pw_set('token', message_auth_token(0)) # user_id 0 indicates that this is an internal connection from the server mqtt_client.tls_set() # enable SSL mqtt_client.connect(mqtt_host, mqtt_port) mqtt_client.loop_start() while True: gevent.sleep(60)
def controller_watchdog(): server_config = load_server_config() worker_log('controller_watchdog', 'starting') last_log_time = None while True: watchdog_check_count = 0 watchdog_expire_count = 0 try: # get list of controllers controllers = Resource.query.filter( Resource.type == Resource.CONTROLLER_FOLDER, not_(Resource.deleted)) for controller in controllers: system_attributes = json.loads( controller.system_attributes ) if controller.system_attributes else {} # if watchdog notifications are enabled if system_attributes.get( 'watchdog_minutes', 0) > 0 and 'watchdog_recipients' in system_attributes: watchdog_check_count += 1 try: # check controller status; if stale watchdog timestamp, send message (if not done already); # if no watchdog timestamp, don't send message (assume user is just setting on the controller for the first time) controller_status = ControllerStatus.query.filter( ControllerStatus.id == controller.id).one() # fix(soon): safe int convert time_thresh = datetime.datetime.utcnow( ) - datetime.timedelta( minutes=system_attributes['watchdog_minutes']) if controller_status.last_watchdog_timestamp and controller_status.last_watchdog_timestamp < time_thresh: watchdog_expire_count += 1 if controller_status.watchdog_notification_sent is False: # send notifications recipients = system_attributes[ 'watchdog_recipients'] worker_log( 'controller_watchdog', 'sending notification for %s to %s' % (controller.path(), recipients)) recipients = recipients.split(',') message = '%s is offline' % controller.path() if server_config[ 'PRODUCTION']: # only send message in production; this is very important for recipient in recipients: if '@' in recipient: send_email(recipient, message, message, server_config) else: send_text_message( recipient, message, server_config) controller_status.watchdog_notification_sent = True db.session.commit() else: if controller_status.watchdog_notification_sent: controller_status.watchdog_notification_sent = False db.session.commit() db.session.expire(controller_status) except NoResultFound: worker_log( 'controller_watchdog', 'controller status not found (%d)' % controller.id) # handle all exceptions because we don't want an error in this code (e.g. sending email or bad status/controller data) stopping all # notifications # pylint: disable=broad-except except Exception as e: print('controller_watchdog error: %s' % str(e)) worker_log('controller_watchdog', str(e)) # once an hour, log current status if (last_log_time is None) or time.time() - last_log_time > 60 * 60: worker_log( 'controller_watchdog', 'checked %d controllers; %d are currently expired' % (watchdog_check_count, watchdog_expire_count)) last_log_time = time.time() # wait one minute time.sleep(60)