def worker():
# log that the worker process is starting
worker_log('system', 'starting worker process')
# start various worker threads
gevent.spawn(controller_watchdog)
gevent.spawn(sequence_truncator)
# loop forever
while True:
# sleep for one second each loop
gevent.sleep(1)
# check for messages
if False:
messages = message_queue.receive()
for message in messages:
if message.type == 'start_worker_task':
print('#### %s' % message.parameters)
params = json.loads(message.parameters)
if params['name'] == 'add_resource_revisions':
print('#### starting add_resource_revisions')
gevent.spawn(add_resource_revisions)
def on_connect(client, userdata, flags, rc):
# pylint: disable=unused-argument
if rc:
worker_log('message_monitor', 'unable to connect to MQTT broker/server at %s:%d' % (mqtt_host, mqtt_port))
else:
worker_log('message_monitor', 'connected to MQTT broker/server at %s:%d' % (mqtt_host, mqtt_port))
client.subscribe('#') # subscribe to all messages
def worker(blocking=True, debug=False):
# log that the worker process is starting
worker_log('system', 'starting worker process')
# start various worker threads
Thread(target=controller_watchdog, daemon=True).start()
Thread(target=sequence_truncator, daemon=True).start()
Thread(target=message_deleter, daemon=True).start()
Thread(target=message_monitor, daemon=True).start()
# loop forever
while blocking:
# sleep for one second each loop
time.sleep(1)
# check for messages
if debug:
messages = message_queue.receive()
for message in messages:
if message.type == 'start_worker_task':
print('#### %s' % message.parameters)
params = json.loads(message.parameters)
if params['name'] == 'add_resource_revisions':
print('#### starting add_resource_revisions')
def check_resource_migration():
worker_log(
'migrate_db', 'resources without org id: %d' %
db.session.query(func.count(Resource.id)).filter(
Resource.organization_id.is_(None)).scalar())
worker_log(
'migrate_db', 'file resources without last rev: %d' %
db.session.query(func.count(Resource.id)).filter(
Resource.type == Resource.FILE,
Resource.last_revision_id.is_(None)).scalar())
def message_deleter():
worker_log('message_deleter', 'starting')
while True:
delete_count = 0
start_time = time.time()
thresh = datetime.datetime.utcnow() - datetime.timedelta(hours=1)
messages = Message.query.filter(Message.timestamp < thresh)
delete_count = messages.count()
messages.delete()
db.session.commit()
db.session.expunge_all()
db.session.close()
delete_time = time.time() - start_time
# display diagnostic
worker_log(
'message_deleter', 'deleted %d messages in %.3f seconds' %
(delete_count, delete_time))
# sleep for 6 hours
time.sleep(6 * 60 * 60)
def sequence_truncator():
verbose = True
worker_log('sequence_truncator', 'starting')
while True:
truncate_count = 0
# loop over all sequences
resources = Resource.query.filter(Resource.type == Resource.SEQUENCE)
for resource in resources:
# get number of revisions for this sequence
rev_count = db.session.query(func.count(
ResourceRevision.id)).filter(
ResourceRevision.resource_id == resource.id).scalar()
# get max history
system_attributes = json.loads(
resource.system_attributes
) if resource.system_attributes else {}
max_history = system_attributes.get('max_history', 1)
# if too many revisions (with 1000 item buffer), delete old ones
# fix(later): revisit buffer for image sequences and others with large objects
if rev_count > max_history + 1000:
# determine timestamp of revision max_history records ago;
# this could be made faster if we assumed that revisions are created sequentially
revisions = ResourceRevision.query.with_entities(
ResourceRevision.timestamp).filter(
ResourceRevision.resource_id == resource.id).order_by(
'timestamp')
boundary_timestamp = revisions[-max_history].timestamp
# diagnostics
if verbose:
message = 'id: %s, path: %s, max hist: %d, revs: %d, first: %s, thresh: %s, last: %s' % (
resource.id, resource.path(), max_history, rev_count,
revisions[0].timestamp.strftime('%Y-%m-%d'),
boundary_timestamp.strftime('%Y-%m-%d'),
revisions[-1].timestamp.strftime('%Y-%m-%d'))
worker_log('sequence_truncator', message)
# delete the old records
# it is critical that we filter by resource ID and timestamp
ResourceRevision.query.filter(
ResourceRevision.resource_id == resource.id,
ResourceRevision.timestamp < boundary_timestamp).delete()
db.session.commit()
truncate_count += 1
# display diagnostic
if truncate_count:
worker_log(
'sequence_truncator',
'done with truncation pass; truncated %d sequences' %
truncate_count)
# sleep for an hour
gevent.sleep(60 * 60)
def message_monitor():
server_config = load_server_config()
if 'MQTT_HOST' not in server_config:
worker_log('message_monitor', 'MQTT host not configured')
return
worker_log('message_monitor', 'starting')
mqtt_host = server_config['MQTT_HOST']
mqtt_port = server_config.get('MQTT_PORT', 443)
# run this on connect/reconnect
def on_connect(client, userdata, flags, rc):
# pylint: disable=unused-argument
if rc:
worker_log(
'message_monitor',
'unable to connect to MQTT broker/server at %s:%d' %
(mqtt_host, mqtt_port))
else:
worker_log(
'message_monitor', 'connected to MQTT broker/server at %s:%d' %
(mqtt_host, mqtt_port))
client.subscribe('#') # subscribe to all messages
# run this on message
def on_message(client, userdata, msg):
# pylint: disable=unused-argument
payload = msg.payload.decode()
# handle full (JSON) messages
if payload.startswith('{'):
message_struct = json.loads(payload)
for message_type, parameters in message_struct.items():
# update sequence values; doesn't support image sequence; should use REST API for image sequences
if message_type == 'update':
folder = find_resource(
'/' + msg.topic
) # for now we assume these messages are published on controller channels
if folder and folder.type in (Resource.BASIC_FOLDER,
Resource.ORGANIZATION_FOLDER,
Resource.CONTROLLER_FOLDER):
timestamp = parameters.get('$t', '')
if timestamp:
timestamp = parse_json_datetime(
timestamp
) # fix(soon): handle conversion errors
else:
timestamp = datetime.datetime.utcnow()
for name, value in parameters.items():
if name != '$t':
seq_name = '/' + msg.topic + '/' + name
resource = find_resource(seq_name)
if resource:
# don't emit new message since UI will receive this message
update_sequence_value(resource,
seq_name,
timestamp,
value,
emit_message=False)
db.session.commit()
# update controller watchdog status
elif message_type == 'watchdog':
controller = find_resource(
'/' + msg.topic
) # for now we assume these messages are published on controller channels
if controller and controller.type == Resource.CONTROLLER_FOLDER:
controller_status = ControllerStatus.query.filter(
ControllerStatus.id == controller.id).one()
controller_status.last_watchdog_timestamp = datetime.datetime.utcnow(
)
db.session.commit()
# send emails
elif message_type == 'send_email':
controller = find_resource(
'/' + msg.topic
) # for now we assume these messages are published on controller channels
if controller and controller.type == Resource.CONTROLLER_FOLDER:
print('sending email')
handle_send_email(controller.id, parameters)
# send SMS messages
elif message_type == 'send_sms' or message_type == 'send_text_message':
controller = find_resource(
'/' + msg.topic
) # for now we assume these messages are published on controller channels
if controller and controller.type == Resource.CONTROLLER_FOLDER:
handle_send_text_message(controller.id, parameters)
# handle short (non-JSON) messages
else:
# print('MQTT: %s %s' % (msg.topic, payload))
if payload.startswith(
's,'
): # type 's' is "store and display new sequence value"
parts = payload.split(',', 3)
if len(parts) == 4:
seq_name = '/' + msg.topic + '/' + parts[1]
timestamp = parse_json_datetime(
parts[2]) # fix(soon): handle conversion errors
value = parts[3]
resource = find_resource(seq_name)
if resource and resource.type == Resource.SEQUENCE:
# don't emit new message since UI will receive this message
update_sequence_value(resource,
seq_name,
timestamp,
value,
emit_message=False)
db.session.commit()
# connect and run
mqtt_client = mqtt.Client(transport='websockets')
mqtt_client.on_connect = on_connect
mqtt_client.on_message = on_message
mqtt_client.username_pw_set(
'token', message_auth_token(0)
) # user_id 0 indicates that this is an internal connection from the server
mqtt_client.tls_set() # enable SSL
mqtt_client.connect(mqtt_host, mqtt_port)
mqtt_client.loop_start()
while True:
gevent.sleep(60)
def message_monitor():
server_config = load_server_config()
if 'MQTT_HOST' not in server_config:
worker_log('message_monitor', 'MQTT host not configured')
return
worker_log('message_monitor', 'starting')
mqtt_host = server_config['MQTT_HOST']
mqtt_port = server_config.get('MQTT_PORT', 443)
# run this on connect/reconnect
def on_connect(client, userdata, flags, rc):
# pylint: disable=unused-argument
if rc:
worker_log('message_monitor', 'unable to connect to MQTT broker/server at %s:%d' % (mqtt_host, mqtt_port))
else:
worker_log('message_monitor', 'connected to MQTT broker/server at %s:%d' % (mqtt_host, mqtt_port))
client.subscribe('#') # subscribe to all messages
# run this on message
def on_message(client, userdata, msg):
# pylint: disable=unused-argument
# print('MQTT: %s %s' % (msg.topic, msg.payload.decode()))
message_struct = json.loads(msg.payload.decode())
message_type = message_struct['type']
if message_type == 'update_sequence':
controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels
if controller and controller.type == Resource.CONTROLLER_FOLDER:
parameters = message_struct['parameters']
seq_name = parameters['sequence']
if not seq_name.startswith('/'): # handle relative sequence names
resource = Resource.query.filter(Resource.id == controller.id).one()
# this is ok for now since .. doesn't have special meaning in resource path (no way to escape controller folder)
seq_name = resource.path() + '/' + seq_name
timestamp = parameters.get('timestamp', '') # fix(soon): need to convert to datetime
if not timestamp:
timestamp = datetime.datetime.utcnow()
value = parameters['value']
if 'encoded' in parameters:
value = base64.b64decode(value)
# remove this; require clients to use REST POST for images
resource = find_resource(seq_name)
if not resource:
return
system_attributes = json.loads(resource.system_attributes) if resource.system_attributes else None
if system_attributes and system_attributes['data_type'] == Resource.IMAGE_SEQUENCE:
value = base64.b64decode(value)
else:
value = str(value)
# don't emit message since the message is already in the system
update_sequence_value(resource, seq_name, timestamp, value, emit_message=False)
db.session.commit()
# update controller watchdog status
elif message_type == 'watchdog':
controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels
if controller and controller.type == Resource.CONTROLLER_FOLDER:
controller_status = ControllerStatus.query.filter(ControllerStatus.id == controller.id).one()
controller_status.last_watchdog_timestamp = datetime.datetime.utcnow()
db.session.commit()
# send emails
elif message_type == 'send_email':
controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels
if controller and controller.type == Resource.CONTROLLER_FOLDER:
print('sending email')
handle_send_email(controller.id, message_struct['parameters'])
# send SMS messages
elif message_type == 'send_sms' or message_type == 'send_text_message':
controller = find_resource('/' + msg.topic) # for now we assume these messages are published on controller channels
if controller and controller.type == Resource.CONTROLLER_FOLDER:
handle_send_text_message(controller.id, message_struct['parameters'])
# connect and run
mqtt_client = mqtt.Client(transport='websockets')
mqtt_client.on_connect = on_connect
mqtt_client.on_message = on_message
mqtt_client.username_pw_set('token', message_auth_token(0)) # user_id 0 indicates that this is an internal connection from the server
mqtt_client.tls_set() # enable SSL
mqtt_client.connect(mqtt_host, mqtt_port)
mqtt_client.loop_start()
while True:
gevent.sleep(60)
def controller_watchdog():
server_config = load_server_config()
worker_log('controller_watchdog', 'starting')
last_log_time = None
while True:
watchdog_check_count = 0
watchdog_expire_count = 0
try:
# get list of controllers
controllers = Resource.query.filter(
Resource.type == Resource.CONTROLLER_FOLDER,
not_(Resource.deleted))
for controller in controllers:
system_attributes = json.loads(
controller.system_attributes
) if controller.system_attributes else {}
# if watchdog notifications are enabled
if system_attributes.get(
'watchdog_minutes',
0) > 0 and 'watchdog_recipients' in system_attributes:
watchdog_check_count += 1
try:
# check controller status; if stale watchdog timestamp, send message (if not done already);
# if no watchdog timestamp, don't send message (assume user is just setting on the controller for the first time)
controller_status = ControllerStatus.query.filter(
ControllerStatus.id == controller.id).one()
# fix(soon): safe int convert
time_thresh = datetime.datetime.utcnow(
) - datetime.timedelta(
minutes=system_attributes['watchdog_minutes'])
if controller_status.last_watchdog_timestamp and controller_status.last_watchdog_timestamp < time_thresh:
watchdog_expire_count += 1
if controller_status.watchdog_notification_sent is False:
# send notifications
recipients = system_attributes[
'watchdog_recipients']
worker_log(
'controller_watchdog',
'sending notification for %s to %s' %
(controller.path(), recipients))
recipients = recipients.split(',')
message = '%s is offline' % controller.path()
if server_config[
'PRODUCTION']: # only send message in production; this is very important
for recipient in recipients:
if '@' in recipient:
send_email(recipient, message,
message, server_config)
else:
send_text_message(
recipient, message,
server_config)
controller_status.watchdog_notification_sent = True
db.session.commit()
else:
if controller_status.watchdog_notification_sent:
controller_status.watchdog_notification_sent = False
db.session.commit()
db.session.expire(controller_status)
except NoResultFound:
worker_log(
'controller_watchdog',
'controller status not found (%d)' % controller.id)
# handle all exceptions because we don't want an error in this code (e.g. sending email or bad status/controller data) stopping all
# notifications
# pylint: disable=broad-except
except Exception as e:
print('controller_watchdog error: %s' % str(e))
worker_log('controller_watchdog', str(e))
# once an hour, log current status
if (last_log_time is None) or time.time() - last_log_time > 60 * 60:
worker_log(
'controller_watchdog',
'checked %d controllers; %d are currently expired' %
(watchdog_check_count, watchdog_expire_count))
last_log_time = time.time()
# wait one minute
time.sleep(60)
