def __init__(self):
self.__queue = Queue()
self.__lock = Lock()
# Mapping of LID to deque (so can serialise messages for same LID). Appended to right, removed from left.
self.__lid_mapping = {}
self.__local = thread_local()
self.__new_msg = Event()
def __init__(self, name, num_workers=1, iotclient=None, daemonic=False):
self.__name = name
self.__num_workers = num_workers
self.__iotclient = iotclient
self.__daemonic = daemonic
#
self.__queue = LidSerialisedQueue()
self.__stop = Event()
self.__stop.set()
self.__threads = []
self.__cache = {}
def __init__(self, fname, iotclient, num_workers):
self.__fname = fname
self.__name = self.__fname_to_name(fname)
self.__workers = ThreadPool(self.__name,
num_workers=num_workers,
iotclient=iotclient)
self.__thread = Thread(target=self.__run,
name=('stash-%s' % self.__name))
self.__stop = Event()
self.__stash = None
self.__stash_lock = RLock()
self.__stash_hash = None
self.__pname = splitext(self.__fname)[0] + '_props.json'
self.__properties = None
self.__properties_changed = False
self.__load()
def main(): # pylint: disable=too-many-return-statements,too-many-branches
if len(argv) < 2:
if not exists(argv[1]):
return usage()
try:
cfg = Config(argv[1])
except:
logger.exception("Failed to load/parse Config file '%s'. Giving up.",
argv[1])
return 1
wwwpath = cfg.get(EXTMON2, 'wwwpath')
if wwwpath is None:
logger.error(
"Config file must have [extmon2] section with wwwpath = /path/to/storage"
)
return 1
wwwpath = abspath(wwwpath)
if not exists(wwwpath):
mkdir(wwwpath)
elif exists(wwwpath) and not isdir(wwwpath):
logger.error("Config file must have [extmon2] wwpath not a directory")
return 1
template = cfg.get(EXTMON2, 'template')
if template is None or not exists(template):
logger.error(
"Config file must have [extmon2] section with template = /path/to/file.html"
)
return 1
agent = cfg.get(EXTMON2, 'agent')
if agent is None or not exists(agent):
logger.error(
"Config file must have [extmon2] section with agent = /path/to/agent.ini"
)
return 1
feeds_list = cfg.get(EXTMON2, 'feeds')
if feeds_list is None:
logger.error(
"Config file must have [extmon2] feeds = \n\nFeedeName\n\tFeed_Two"
)
return 1
for feed in feeds_list:
if cfg.get(feed, 'guid') is None or cfg.get(feed, MAX_AGE) is None:
logger.error(
"Config section for feed [%s] must have guid and max_age",
feed)
return 1
stop_evt = Event()
thread = Thread(target=extmon, name='extmon', args=(
cfg,
stop_evt,
))
thread.start()
if 'IOTIC_BACKGROUND' in environ:
from signal import signal, SIGINT, SIGTERM
logger.info("Started in non-interactive mode.")
def exit_handler(signum, frame): # pylint: disable=unused-argument
logger.info('Shutdown requested')
stop_evt.set()
signal(SIGINT, exit_handler)
signal(SIGTERM, exit_handler)
while not stop_evt.is_set():
stop_evt.wait(timeout=5)
stop_evt.set()
else:
try:
while not stop_evt.is_set():
logger.info('Enter ctrl+c to exit')
stop_evt.wait(timeout=600)
except SystemExit:
pass
except KeyboardInterrupt:
pass
stop_evt.set()
logger.info("Waiting for thread to finish...")
thread.join()
return 0
class LidSerialisedQueue(object):
"""Thread-safe queue which ensures enqueued Messages for the same lid are not handled by multiple threads at the
same time."""
def __init__(self):
self.__queue = Queue()
self.__lock = Lock()
# Mapping of LID to deque (so can serialise messages for same LID). Appended to right, removed from left.
self.__lid_mapping = {}
self.__local = thread_local()
self.__new_msg = Event()
def thread_init(self):
"""Must be called in each thread which is to use this instance, before using get()!"""
# LID which is being processed by this thread, if any
self.__local.own_lid = None
@property
def empty(self):
return not self.__lid_mapping and self.__queue.empty()
def put(self, qmsg):
if not isinstance(qmsg, Message):
raise ValueError
self.__queue.put(qmsg)
self.__new_msg.set()
def __get_for_current_lid(self):
"""Returns next message for same LID as previous message, if available. None otherwise. MUST be called within
lock!
"""
local = self.__local
try:
return self.__lid_mapping[local.own_lid].popleft()
# No messages left for this LID (deque.popleft), remove LID association
except IndexError:
del self.__lid_mapping[local.own_lid]
local.own_lid = None
# No LIDs being processed (mapping)
except KeyError:
pass
return None
def get(self, timeout=None):
"""Raises queue.Empty exception if no messages are available after timeout"""
with self.__lock:
msg = self.__get_for_current_lid()
if not msg:
queue = self.__queue
lid_mapping = self.__lid_mapping
while True:
# Instead of blocking on get(), release lock so other threads have a chance to request existing lid
# messages (above, via __get_for_current_lid).
if not queue.qsize() and timeout:
self.__new_msg.clear()
try:
self.__lock.release()
self.__new_msg.wait(timeout)
finally:
self.__lock.acquire()
msg = queue.get_nowait()
# Enqueue message with LID already being dealt with in LID-specific queue, otherwise can process
# oneself.
if msg.lid in lid_mapping:
lid_mapping[msg.lid].append(msg)
else:
# Currently nobody else is processing messages with this LID, so can use oneself
self.__local.own_lid = msg.lid
lid_mapping[msg.lid] = deque()
break
return msg
class ThreadPool(object): # pylint: disable=too-many-instance-attributes
__share_time_fmt = '%Y-%m-%dT%H:%M:%S.%fZ'
def __init__(self, name, num_workers=1, iotclient=None, daemonic=False):
self.__name = name
self.__num_workers = num_workers
self.__iotclient = iotclient
self.__daemonic = daemonic
#
self.__queue = LidSerialisedQueue()
self.__stop = Event()
self.__stop.set()
self.__threads = []
self.__cache = {}
def start(self):
if self.__stop.is_set():
self.__stop.clear()
for i in range(0, self.__num_workers):
thread = Thread(target=self.__worker,
name=('tp-%s-%d' % (self.__name, i)))
thread.daemon = self.__daemonic
self.__threads.append(thread)
for thread in self.__threads:
thread.start()
def submit(self, lid, idx, diff, complete_cb=None):
self.__queue.put(Message(lid, idx, diff, complete_cb))
def stop(self):
if not self.__stop.is_set():
self.__stop.set()
for thread in self.__threads:
thread.join()
del self.__threads[:]
@property
def queue_empty(self):
return self.__queue.empty
def __worker(self):
logger.debug("Starting")
self.__queue.thread_init()
stop_is_set = self.__stop.is_set
queue_get = self.__queue.get
handle_thing_changes = self.__handle_thing_changes
while not stop_is_set():
try:
qmsg = queue_get(timeout=.25)
except Empty:
continue # queue.get timeout ignore
while True:
try:
handle_thing_changes(qmsg.lid, qmsg.diff)
except LinkException:
logger.warning("Network error, will retry lid '%s'",
qmsg.lid)
self.__stop.wait(timeout=1)
continue
except IOTAccessDenied:
logger.critical(
"IOTAccessDenied - Local limit exceeded - Aborting")
kill(getpid(), SIGUSR1)
return
except:
logger.error(
"Failed to process thing changes (Uncaught exception) - Aborting",
exc_info=DEBUG_ENABLED)
kill(getpid(), SIGUSR1)
return
break
logger.debug("completed thing %s", qmsg.lid)
if qmsg.complete_cb:
try:
qmsg.complete_cb(qmsg.lid, qmsg.idx)
except:
logger.error("complete_cb failed for %s",
qmsg.lid,
exc_info=DEBUG_ENABLED)
kill(getpid(), SIGUSR1)
return
@classmethod
def __lang_convert(cls, lang):
if lang == '':
return None
return lang
def __handle_thing_changes(self, lid, diff): # pylint: disable=too-many-branches
if lid not in self.__cache:
self.__cache[lid] = {
THING: self.__iotclient.create_thing(lid),
POINTS: {}
}
iotthing = self.__cache[lid][THING]
if PUBLIC in diff and diff[PUBLIC] is False:
iotthing.set_public(False)
thingmeta = None
for chg, val in diff.items():
if chg == TAGS and len(val):
iotthing.create_tag(val)
elif chg == LABELS and val:
if thingmeta is None:
thingmeta = iotthing.get_meta()
for lang, label in val.items():
thingmeta.set_label(label, lang=self.__lang_convert(lang))
elif chg == DESCRIPTIONS and val:
if thingmeta is None:
thingmeta = iotthing.get_meta()
for lang, description in val.items():
thingmeta.set_description(description,
lang=self.__lang_convert(lang))
elif chg == LOCATION and val[0] is not None:
if thingmeta is None:
thingmeta = iotthing.get_meta()
thingmeta.set_location(val[0], val[1])
if thingmeta is not None:
thingmeta.set()
for pid, pdiff in diff[POINTS].items():
self.__handle_point_changes(iotthing, lid, pid, pdiff)
if PUBLIC in diff and diff[PUBLIC] is True:
iotthing.set_public(True)
def __handle_point_changes(self, iotthing, lid, pid, pdiff): # pylint: disable=too-many-branches
if pid not in self.__cache[lid][POINTS]:
if pdiff[FOC] == R_FEED:
iotpoint = iotthing.create_feed(pid)
elif pdiff[FOC] == R_CONTROL:
iotpoint = iotthing.create_control(pid)
self.__cache[lid][POINTS][pid] = iotpoint
iotpoint = self.__cache[lid][POINTS][pid]
pointmeta = None
for chg, val in pdiff.items():
if chg == TAGS and len(val):
iotpoint.create_tag(val)
elif chg == RECENT:
iotpoint.set_recent_config(max_samples=pdiff[RECENT])
elif chg == LABELS and val:
if pointmeta is None:
pointmeta = iotpoint.get_meta()
for lang, label in val.items():
pointmeta.set_label(label, lang=self.__lang_convert(lang))
elif chg == DESCRIPTIONS and val:
if pointmeta is None:
pointmeta = iotpoint.get_meta()
for lang, description in val.items():
pointmeta.set_description(description,
lang=self.__lang_convert(lang))
if pointmeta is not None:
pointmeta.set()
sharedata = {}
for label, vdiff in pdiff[VALUES].items():
if SHAREDATA in vdiff:
sharedata[label] = vdiff[SHAREDATA]
self.__handle_value_changes(lid, pid, label, vdiff)
sharetime = None
if SHARETIME in pdiff:
sharetime = pdiff[SHARETIME]
if isinstance(sharetime, string_types):
try:
sharetime = datetime.strptime(sharetime,
self.__share_time_fmt)
except:
logger.warning(
"Failed to make datetime from time string '%s' !Will use None!",
sharetime)
sharetime = None
if len(sharedata):
iotpoint.share(data=sharedata, time=sharetime)
if SHAREDATA in pdiff:
iotpoint.share(data=pdiff[SHAREDATA], time=sharetime)
def __handle_value_changes(self, lid, pid, label, vdiff):
"""
Note: remove & add values if changed, share data if data
"""
iotpoint = self.__cache[lid][POINTS][pid]
if VTYPE in vdiff and vdiff[VTYPE] is not None:
iotpoint.create_value(label,
vdiff[VTYPE],
lang=vdiff[LANG],
description=vdiff[DESCRIPTION],
unit=vdiff[UNIT])
class Stash(object): # pylint: disable=too-many-instance-attributes
@classmethod
def __fname_to_name(cls, fname):
return splitext(path_split(fname)[-1])[0]
def __init__(self, fname, iotclient, num_workers):
self.__fname = fname
self.__name = self.__fname_to_name(fname)
self.__workers = ThreadPool(self.__name,
num_workers=num_workers,
iotclient=iotclient)
self.__thread = Thread(target=self.__run,
name=('stash-%s' % self.__name))
self.__stop = Event()
self.__stash = None
self.__stash_lock = RLock()
self.__stash_hash = None
self.__pname = splitext(self.__fname)[0] + '_props.json'
self.__properties = None
self.__properties_changed = False
self.__load()
def start(self):
self.__workers.start()
self.__submit_diffs()
self.__thread.start()
def stop(self):
if not self.__stop.is_set():
self.__stop.set()
self.__thread.join()
self.__workers.stop()
self.__save()
def __enter__(self):
self.start()
return self
def __exit__(self, typ, value, traceback):
self.stop()
def is_alive(self):
return self.__thread.is_alive()
def __load(self): # pylint: disable=too-many-branches
fsplit = splitext(self.__fname)
if fsplit[1] == '.json':
if exists(self.__fname):
# Migrate from json to ubjson
with self.__stash_lock:
with open(self.__fname, 'r') as f:
self.__stash = json.loads(f.read())
rename(self.__fname, self.__fname + '.old')
if fsplit[1] != '.ubjz':
self.__fname = fsplit[0] + '.ubjz'
if exists(self.__fname):
with self.__stash_lock:
with gzip_open(self.__fname, 'rb') as f:
self.__stash = ubjson.loadb(f.read())
elif self.__stash is None:
self.__stash = {
THINGS: {}, # Current/last state of Things
DIFF: {}, # Diffs not yet updated in Iotic Space
DIFFCOUNT: 0
} # Diff counter
if not exists(self.__pname):
self.__properties = {}
else:
with self.__stash_lock:
with open(self.__pname, 'r') as f:
self.__properties = json.loads(f.read())
with self.__stash_lock:
stash_copy = deepcopy(self.__stash)
self.__stash = {}
# Migrate built-in keys
for key, value in stash_copy.items():
if key in [THINGS, DIFF, DIFFCOUNT]:
logger.debug("--> Migrating built-in %s", key)
self.__stash[key] = stash_copy[key]
# Migrate bad keys
for key, value in stash_copy.items():
if key not in [THINGS, DIFF, DIFFCOUNT]:
if key not in stash_copy[THINGS]:
logger.info("--> Migrating key to THINGS %s", key)
self.__stash[THINGS][key] = value
self.__stash.pop(key, None)
# Remove redundant LAT/LONG (LOCATION used instead)
for el, et in self.__stash[THINGS].items(): # pylint: disable=unused-variable
et.pop(LAT, None)
et.pop(LONG, None)
self.__save()
def __calc_stashdump(self):
with self.__stash_lock:
stashdump = ubjson.dumpb(self.__stash)
m = md5()
m.update(stashdump)
stashhash = m.hexdigest()
if self.__stash_hash != stashhash:
self.__stash_hash = stashhash
return stashdump
return None
def __save(self):
stashdump = self.__calc_stashdump()
if stashdump is not None:
with gzip_open(self.__fname, 'wb') as f:
f.write(stashdump)
if len(self.__properties) and self.__properties_changed:
with self.__stash_lock:
with open(self.__pname, 'w') as f:
json.dump(self.__properties, f)
def get_property(self, key):
with self.__stash_lock:
if not isinstance(key, string_types):
raise ValueError("key must be string")
if key in self.__properties:
return self.__properties[key]
return None
def set_property(self, key, value=None):
with self.__stash_lock:
if not isinstance(key, string_types):
raise ValueError("key must be string")
if value is None and key in self.__properties:
del self.__properties[key]
if value is not None:
if isinstance(value, string_types) or isinstance(
value, number_types):
if key not in self.__properties or self.__properties[
key] != value:
self.__properties_changed = True
self.__properties[key] = value
else:
raise ValueError("value must be string or int")
def __run(self):
logger.info("Started.")
while not self.__stop.is_set():
self.__save()
self.__stop.wait(timeout=SAVETIME)
def create_thing(self, lid):
if lid in self.__stash[THINGS]:
thing = Thing(lid,
stash=self,
public=self.__stash[THINGS][lid][PUBLIC],
labels=self.__stash[THINGS][lid][LABELS],
descriptions=self.__stash[THINGS][lid][DESCRIPTIONS],
tags=self.__stash[THINGS][lid][TAGS],
points=self.__stash[THINGS][lid][POINTS],
lat=self.__stash[THINGS][lid][LOCATION][0],
long=self.__stash[THINGS][lid][LOCATION][1])
return thing
return Thing(lid, new=True, stash=self)
def __calc_diff(self, thing): # pylint: disable=too-many-branches
if not len(thing.changes):
changes = 0
for pid, point in thing.points.items():
changes += len(point.changes)
if changes == 0:
return None, None
ret = 0
diff = {}
if thing.new:
# Note: thing is new so no need to calculate diff.
# This shows the diff dict full layout
diff = {
LID: thing.lid,
TAGS: thing.tags,
LOCATION: thing.location,
LABELS: thing.labels,
DESCRIPTIONS: thing.descriptions,
POINTS: {}
}
# Prevent public setting to always be performed for new things
if PUBLIC in thing.changes:
diff[PUBLIC] = thing.public
for pid, point in thing.points.items():
diff[POINTS][pid] = self.__calc_diff_point(point)
else:
diff[LID] = thing.lid
diff[POINTS] = {}
for change in thing.changes:
if change == PUBLIC:
diff[PUBLIC] = thing.public
elif change == TAGS:
diff[TAGS] = thing.tags
elif change.startswith(LABEL):
if LABELS not in diff:
diff[LABELS] = {}
lang = change.replace(LABEL, '')
diff[LABELS][lang] = thing.labels[lang]
elif change.startswith(DESCRIPTION):
if DESCRIPTIONS not in diff:
diff[DESCRIPTIONS] = {}
lang = change.replace(DESCRIPTION, '')
diff[DESCRIPTIONS][lang] = thing.descriptions[lang]
elif change == LOCATION:
diff[LOCATION] = thing.location
for pid, point in thing.points.items():
diff[POINTS][pid] = self.__calc_diff_point(point)
with self.__stash_lock:
self.__stash[DIFF][str(self.__stash[DIFFCOUNT])] = diff
ret = self.__stash[DIFFCOUNT]
self.__stash[DIFFCOUNT] += 1
return ret, diff
def __calc_diff_point(self, point):
ret = {PID: point.lid, FOC: point.foc, VALUES: {}}
if point.new:
ret.update({LABELS: {}, DESCRIPTIONS: {}, RECENT: 0, TAGS: []})
for change in point.changes:
if change == TAGS:
ret[TAGS] = point.tags
elif change.startswith(LABEL):
if LABELS not in ret:
ret[LABELS] = {}
lang = change.replace(LABEL, '')
ret[LABELS][lang] = point.labels[lang]
elif change.startswith(DESCRIPTION):
if DESCRIPTIONS not in ret:
ret[DESCRIPTIONS] = {}
lang = change.replace(DESCRIPTION, '')
ret[DESCRIPTIONS][lang] = point.descriptions[lang]
elif change == RECENT:
ret[RECENT] = point.recent_config
elif change == SHAREDATA:
ret[SHAREDATA] = point.sharedata
elif change == SHARETIME:
ret[SHARETIME] = point.sharetime
elif change.startswith(VALUE):
label = change.replace(VALUE, '')
ret[VALUES][label] = self.__calc_value(point.values[label])
# applicable only if no value attributes have changed (share only)
elif change.startswith(VALUESHARE):
label = change.replace(VALUESHARE, '')
if label not in ret[VALUES]:
ret[VALUES][label] = {}
ret[VALUES][label].update(
{SHAREDATA: point.values[label].pop(SHAREDATA)})
return ret
@classmethod
def __calc_value(cls, value):
ret = {}
if VTYPE in value:
ret = {
VTYPE: value[VTYPE],
LANG: value[LANG],
DESCRIPTION: value[DESCRIPTION],
UNIT: value[UNIT]
}
if SHAREDATA in value:
ret[SHAREDATA] = value[SHAREDATA]
return ret
def __submit_diffs(self):
"""On start resubmit any diffs in the stash
"""
with self.__stash_lock:
for idx, diff in self.__stash[DIFF].items():
logger.info("Resubmitting diff for thing %s", diff[LID])
self.__workers.submit(diff[LID], idx, diff, self.__complete_cb)
def _finalise_thing(self, thing):
with thing.lock:
idx, diff = self.__calc_diff(thing)
if idx is not None:
self.__workers.submit(diff[LID], idx, diff, self.__complete_cb)
thing.clear_changes()
def __complete_cb(self, lid, idx):
with self.__stash_lock:
idx = str(idx)
diff = self.__stash[DIFF][idx]
try:
thing = self.__stash[THINGS][lid]
except KeyError:
thing = self.__stash[THINGS][lid] = {
PUBLIC: False,
LABELS: {},
DESCRIPTIONS: {},
TAGS: [],
POINTS: {},
LOCATION: (None, None)
}
empty = {}
# Updated later separately
points = diff.pop(POINTS, empty)
# Have to be merged since update only affects subset of all labels/descriptions
for item in (LABELS, DESCRIPTIONS):
thing[item].update(diff.pop(item, empty))
# Rest should be OK to replace (public, tags, location)
thing.update(diff)
# Points
for pid, pdiff in points.items():
try:
point = thing[POINTS][pid]
except KeyError:
thing[POINTS][pid] = point = {
PID: pid,
VALUES: {},
LABELS: {},
DESCRIPTIONS: {},
TAGS: []
}
# Updated later separately
values = pdiff.pop(VALUES, empty)
# Remove sharedata, sharetime before applying to stash
for item in (SHAREDATA, SHARETIME):
pdiff.pop(item, None)
# Have to be merged since update only affects subset of all labels/descriptions
for item in (LABELS, DESCRIPTIONS):
point[item].update(pdiff.pop(item, empty))
# Rest should be OK to replace (tags, foc, recent, share time, data)
point.update(pdiff)
# Values
for label, value in values.items():
# Don't remember value share data
value.pop(SHAREDATA, None)
try:
# Might only have data set so must merge
point[VALUES][label].update(value)
except KeyError:
point[VALUES][label] = value
del self.__stash[DIFF][idx]
@property
def queue_empty(self):
return self.__workers.queue_empty