def startOfNextDay(t):
"""Given a time, returns time of next midnight."""
s = ISO8601.epoch_seconds_to_ISO8601(t) # e.g. "2007-10-23T23:32:10Z
dt = ISO8601.epoch_seconds_to_datetime(t)
dt += datetime.timedelta(days=1)
s = dt.strftime("%Y-%m-%dT00:00:00") # We've assumed timezone!
return ISO8601.to_epoch_seconds(s)
def in_simulated_time(self, secs=None):
if g_get_sim_time:
t = g_get_sim_time()
else:
t = 0
return ISO8601.epoch_seconds_to_datetime(t).timetuple(
) # Logging might be emitted within sections where simLock is acquired, so we accept a small chance of duff time values in log messages, in order to allow diagnostics without deadlock
def nextUsageTime(t, daySpec, hourSpec):
"""Given a daySpec e.g. ["Mon","Tue"] and an hourspec e.g. '08:00-10:00'
work out a next time t which falls within that spec (picking randomly within the hour range)
if given a time already within spec, move to the NEXT such time)"""
(startHour, endHour) = toHours(hourSpec)
(h, d) = (hourInDay(t), dayOfWeek(t))
if (d in daySpec) and (h < endHour): # If already in spec, move beyond
t += 60 * 60 * endHour - h
while True: # Move to a valid day of the week
(h, d) = (hourInDay(t), dayOfWeek(t))
if (d in daySpec) and (h < endHour):
break
t = startOfNextDay(t)
chosenHour = startHour + (endHour - startHour) * random.random()
ts = ISO8601.epoch_seconds_to_ISO8601(t)
ts = ts[:11] + hourToHHMMSS(chosenHour) + ts[19:]
t = ISO8601.to_epoch_seconds(ts)
return t
def sun_angle(epochSecs, longitude, latitude):
dateS = ISO8601.epoch_seconds_to_ISO8601(epochSecs)
year = int(dateS[0:4])
month = int(dateS[5:7])
day = int(dateS[8:10])
hour = int(dateS[11:13])
minute = int(dateS[14:16])
sec = int(dateS[17:19])
(azimuthD, elevationD) = sunpos_2.sun_position(year, month, day, hour,
minute, sec, latitude,
longitude)
return azimuthD, elevationD
def jitter(t, X, amountS):
"""Return a random number (intended as a time offset, i.e. jitter) within the range +/-amountS
The jitter is different (but constant) for any given day in t (epoch secs)
and for any value X (which might be e.g. deviceID)"""
dt = ISO8601.epoch_seconds_to_datetime(t)
dayOfYear = int(dt.strftime("%j"))
year = int(dt.strftime("%Y"))
uniqueValue = year * 367 + dayOfYear + abs(
hash(X)
) # Note that hash is implementation-dependent so may give different results on different platforms
rand = utils.hashIt(uniqueValue, 100)
sign = int(str(uniqueValue)[0]) < 5
v = (rand / 100.0) * amountS
if sign:
v = -v
return v
def PLUGIN_query(self, params):
body = {}
for p in params:
if p in ["expect"]:
pass
elif p in ["start","end"]:
body.update({p : ISO8601.to_epoch_seconds(params[p])*1000}), # TODO: Needs * 1000 for ms?
else:
body.update({p : params[p]})
logging.info("DevicePilot query:\n"+json.dumps(body, sort_keys=True, indent=4, separators=(',', ': ')))
resp = self.session.post(self.url+"/query", verify=True, headers=set_headers(self.key), data=json.dumps(body))
assert resp.ok, str(resp.reason) + str(resp.text)
result = json.loads(resp.text)
logging.info("Query returned:\n"+json.dumps(result, sort_keys=True, indent=4, separators=(',', ': ')))
if "expect" in params:
expected = params["expect"]
if result == expected:
logging.info("Query returned expected result - PASS")
else:
logging.info("Expected result:\n"+json.dumps(expected, sort_keys=True, indent=4, separators=(',', ': ')))
assert False, "Query did not return expected result - FAIL"
def second_of_day(epochSecs):
s = ISO8601.epoch_seconds_to_ISO8601(epochSecs)
s = "1970-01-01" + s[10:]
return ISO8601.to_epoch_seconds(s)
def __init__(self, client, engine, instance_name, context, eventList):
"""<params> is a list of events. Note that our .event_count property is read from outside."""
def update_callback(device_id, time, properties):
for c in self.update_callbacks:
properties.update(c(properties)) # We MERGE the results of the callback with the original message
write_event_log(properties)
client.update_device(device_id, time, properties)
def query_action(params):
events = evt2csv.read_evt_str("".join(self.logtext))
query.do_query(params, events)
def change_property_action(params):
def set_it(d):
if params.get("is_attribute", False):
d.__dict__[params["property_name"]] = params["property_value"]
logging.info("Set attribute "+str(params["property_name"])+" on device "+d.get_property("$id")+" to "+str(params["property_value"]))
else:
d.set_property(params["property_name"], params["property_value"], timestamp=ts)
logging.info("Set property "+str(params["property_name"])+" on device "+d.get_property("$id")+" to "+str(params["property_value"]))
d = device_factory.get_devices_by_property( params["identity_property"],
params["identity_value"])
logging.info("change property acting on "+str(len(d))+" matching devices")
if "$ts" in params:
ts = conftime.richTime(params["$ts"])
else:
ts = None
logging.info("change_property "+str(params))
for the_d in d:
set_it(the_d)
def client_action(args):
(name, params) = args
if "PLUGIN_"+name in dir(client):
logging.info("Plug-in client action "+str(name))
getattr(client, "PLUGIN_"+name)(params) # Programmatically call the method
else:
logging.error("Ignoring action '"+str(name)+"' as client "+str(client.__class__.__name__)+" does not support it")
def write_event_log(properties):
"""Write .evt entry"""
return
s = pendulum.from_timestamp(properties["$ts"]).to_datetime_string()+" "
for k in sorted(properties.keys()):
s += str(k) + ","
try:
s += json.dumps(properties[k]) # Use dumps not str so we preserve type in output
# s += properties[k].encode('ascii', 'ignore') # Python 2.x barfs if you try to write unicode into an ascii file
except:
s += "<unicode encoding error>"
s += ","
s += "\n"
self.logfile.write(s)
# self.logtext.append(s)
self.event_count += 1
restart_log = context.get("restart_log",True)
self.client = client
self.event_count = 0
self.update_callbacks = []
mkdir_p(LOG_DIRECTORY)
self.file_mode = "at"
if restart_log:
self.file_mode = "wt"
#self.logfile = open(LOG_DIRECTORY+instance_name+".evt", self.file_mode) # This was unbuffered, but Python3 now doesn't allow text files to be unbuffered
#self.logfile.write("*** New simulation starting at real time "+datetime.now().ctime()+" (local)\n")
self.logtext = [] # TODO: Probably a Bad Idea to store this in memory. Instead when we want this we should probably close the logfile, read it and then re-open it. We store as an array because appending to a large string gets very slow
at_time = engine.get_now()
for event in eventList:
timespec = event.get("at", "PT0S")
if timespec == "end":
end_time = engine.get_end_time() # This may not be a time
assert type(end_time) in [int, float], "An event is defined at 'end', but simulation end time is not a definitive time"
at_time = engine.get_end_time() - 0.001 # Ensure they happen BEFORE the end, as sim end time is non-inclusive
elif timespec[0] in "-+P": # Time relative to current sim time
at_time = at_time + isodate.parse_duration(timespec).total_seconds()
else:
at_time = ISO8601.to_epoch_seconds(timespec)
action = event.get("action", None)
repeats = event.get("repeats", 1) # MAY also specify a repeat and interval
interval = event.get("interval","PT0S")
while repeats > 0:
# Built-in actions. TODO: Make these plug-in too?
if action is None:
pass
elif "create_device" in action:
engine.register_event_at(at_time,
device_factory.create_device,
(instance_name, client, engine, update_callback, context, action["create_device"]),
None)
elif "use_model" in action:
engine.register_event_at(at_time,
model.use_model,
(instance_name, client, engine, update_callback, context, action["use_model"]), None)
elif "query" in action:
engine.register_event_at(at_time,
query_action,
action["query"],
None)
elif "change_property" in action:
engine.register_event_at(at_time,
change_property_action,
action["change_property"],
None)
elif "install_analyser" in action:
logging.info("Installing analyser")
self.analyser = analyse.Analyser()
self.update_callbacks.append(self.analyser.process)
else: # Plug-in actions
name = action.keys()[0]
if not name.startswith("client."):
logging.error("Ignoring unrecognised action "+name)
else:
engine.register_event_at(at_time,
client_action,
(name[7:], action[name]),
None)
##
## elif "delete_demo_devices" in action:
## if "deleteDemoDevices" in dir(client):
## client.deleteDemoDevices()
## else:
## logging.warning("Ignoring unknown event action type "+str(event["action"]))
at_time += isodate.parse_duration(interval).total_seconds()
repeats -= 1
def test(datestr):
print(datestr, ":",
get_intensity(ISO8601.to_epoch_seconds(datestr), False))
def __init__(self, client, engine, instance_name, context, eventList):
"""<params> is a list of events. Note that our .event_count property is read from outside."""
def update_callback(device_id, time, properties):
for c in self.update_callbacks:
properties.update(c(properties)) # We MERGE the results of the callback with the original message
if self.do_write_log:
write_event_log(properties)
if explode_factor is None:
client.update_device(device_id, time, properties)
else:
new_props = properties.copy()
for i in range(explode_factor):
eid = str(device_id) + "_" + str(i) # Each exploded device is identical, except for trailing "-N" ID (and label, if exists)
new_props["$id"] = eid
if "label" in new_props:
new_props["label"] = properties["label"] + "_" + str(i)
client.update_device(eid, time, new_props)
def query_action(params):
events = evt2csv.read_evt_str("".join(self.logtext))
query.do_query(params, events)
def change_property_action(params):
def set_it(d):
if params.get("is_attribute", False):
d.__dict__[params["property_name"]] = params["property_value"]
logging.info("Set attribute "+str(params["property_name"])+" on device "+d.get_property("$id")+" to "+str(params["property_value"]))
else:
d.set_property(params["property_name"], params["property_value"], timestamp=ts)
logging.info("Set property "+str(params["property_name"])+" on device "+d.get_property("$id")+" to "+str(params["property_value"]))
d = device_factory.get_devices_by_property( params["identity_property"], params["identity_value"])
if "identity_property2" in params:
d2 = device_factory.get_devices_by_property( params["identity_property2"], params["identity_value2"])
d = list(set(d) & set(d2))
logging.info("change property acting on "+str(len(d))+" matching devices")
if "$ts" in params:
ts = conftime.richTime(params["$ts"])
else:
ts = None
logging.info("change_property "+str(params))
for the_d in d:
set_it(the_d)
def dump_periodic_metadata(params):
interval = isodate.parse_duration(params["interval"]).total_seconds()
metadata_list = set(params["metadata"])
metadata = []
for d in device_factory.get_devices():
p = {}
for k,v in d.get_properties().items():
if k in metadata_list:
p[k] = v
if len(p) > 0:
# logging.info("For device " + d.get_property("$id") + " setting properties "+str(p))
# d.set_properties(p)
p["$id"] = d.get_property("$id")
metadata.append(p)
fname_stem = METADATA_DIRECTORY + instance_name
open(fname_stem + ".tmp","wt").write(json.dumps(metadata))
os.rename(fname_stem + ".tmp", fname_stem + ".json") # So change is atomic (don't want ^C to leave partially-written file)
engine.register_event_at(engine.get_now() + interval, dump_periodic_metadata, params, None)
def client_action(args):
(name, params) = args
if "PLUGIN_"+name in dir(client):
logging.info("Plug-in client action "+str(name))
getattr(client, "PLUGIN_"+name)(params) # Programmatically call the method
else:
logging.error("Ignoring action '"+str(name)+"' as client "+str(client.__class__.__name__)+" does not support it")
@functools.lru_cache(maxsize=128)
def dt_string(t):
return pendulum.from_timestamp(t).to_datetime_string() + " " # For some reason, this is very slow
def write_event_log(properties):
"""Write .evt entry"""
s = dt_string(properties["$ts"])
for k in sorted(properties.keys()):
s += str(k) + ","
try:
s += json.dumps(properties[k]) # Use dumps not str so we preserve type in output
# s += properties[k].encode('ascii', 'ignore') # Python 2.x barfs if you try to write unicode into an ascii file
except:
logging.error("Encoding error in events.py::write_event_log")
s += "<unicode encoding error>"
s += ","
s += "\n"
self.logfile.write(s)
self.event_count += 1
restart_log = context.get("restart_log", True)
self.do_write_log = context.get("write_log", True)
shard_size = context.get("shard_size", None)
shard_start = context.get("shard_start", 0) # If not specified, we are shard 0, so get to create the other shards
explode_factor = context.get("explode_factor", None)
if explode_factor is not None:
logging.info("Running with explode_factor="+str(explode_factor))
self.event_count = 0
self.update_callbacks = []
device_count = 0 # Used to shard device creation
if self.do_write_log:
mkdir_p(LOG_DIRECTORY)
self.file_mode = "at"
if restart_log:
self.file_mode = "wt"
self.logfile = open(LOG_DIRECTORY+instance_name+".evt", self.file_mode) # This was unbuffered, but Python3 now doesn't allow text files to be unbuffered
self.logfile.write("*** New simulation starting at real time "+datetime.now().ctime()+" (local)\n")
else:
self.logfile = None
logging.info("Not writing an event log")
self.logtext = [] # TODO: Probably a Bad Idea to store this in memory. Instead when we want this we should probably close the logfile, read it and then re-open it. We store as an array because appending to a large string gets very slow
at_time = engine.get_now()
for event in eventList:
timespec = event.get("at", "PT0S")
if timespec.startswith("now"):
dt = datetime.fromtimestamp(time.time()) # Take time in whole seconds (so if we ever want to repeat the run, we can start it at the exact same time)
dt.microsecond = 0
realtime = dt.timestamp()
delta = isodate.parse_duration(timespec[3:]).total_seconds()
at_time = realtime + delta
elif timespec == "end":
end_time = engine.get_end_time() # This may not be a time
assert type(end_time) in [int, float], "An event is defined at 'end', but simulation end time is not a definitive time"
at_time = engine.get_end_time() - 0.001 # Ensure they happen BEFORE the end, as sim end time is non-inclusive
elif timespec[0] in "-+P": # Time relative to current 'at' time
at_time = at_time + isodate.parse_duration(timespec).total_seconds()
else:
at_time = ISO8601.to_epoch_seconds(timespec)
action = event.get("action", None)
repeats = event.get("repeats", 1) # MAY also specify a repeat and interval
interval = event.get("interval","PT0S")
time_advance = event.get("time_advance", True)
insert_time = at_time
while repeats > 0:
# Built-in actions. TODO: Make these plug-in too?
if action is None:
pass
elif "create_device" in action:
do_create = True # Is this device in our shard?
if shard_size is not None:
if device_count < shard_start:
do_create = False
if device_count >= shard_start + shard_size:
do_create = False
if shard_start == 0: # We're the first shard
if device_count > 0:
if (device_count % shard_size) == 0:
logging.info("CREATE NEW SHARD for devices starting at "+str(device_count))
args = sys.argv.copy()
args = ["python3"] + args + ["{ shard_start : " + str(device_count) + " } "]
logging.info(str(args))
if do_create:
engine.register_event_at(insert_time, device_factory.create_device,
(instance_name, client, engine, update_callback, context, action["create_device"]),
None)
device_count += 1
elif "use_model" in action:
engine.register_event_at(insert_time,
model.use_model,
(instance_name, client, engine, update_callback, context, action["use_model"]), None)
elif "query" in action:
engine.register_event_at(insert_time,
query_action,
action["query"],
None)
elif "change_property" in action:
engine.register_event_at(insert_time,
change_property_action,
action["change_property"],
None)
elif "install_analyser" in action:
logging.info("Installing analyser")
self.analyser = analyse.Analyser()
self.update_callbacks.append(self.analyser.process)
elif "periodic_metadata" in action:
engine.register_event_at(insert_time, dump_periodic_metadata, action["periodic_metadata"], None)
else: # Plug-in actions
name = action.keys()[0]
if not name.startswith("client."):
logging.error("Ignoring unrecognised action "+name)
else:
engine.register_event_at(insert_time,
client_action,
(name[7:], action[name]),
None)
##
## elif "delete_demo_devices" in action:
## if "deleteDemoDevices" in dir(client):
## client.deleteDemoDevices()
## else:
## logging.warning("Ignoring unknown event action type "+str(event["action"]))
insert_time += isodate.parse_duration(interval).total_seconds()
repeats -= 1
if time_advance:
at_time = insert_time
def dayOfWeek(t):
"""Returns e.g. 'Mon'"""
dt = ISO8601.epoch_seconds_to_datetime(t)
return dt.strftime("%a")
def hourInDay(t):
"""Returns hour in day (as a float, to nearest second, e.g. 07:30 is 7.5)"""
dt = ISO8601.epoch_seconds_to_datetime(t)
return int(dt.strftime("%H")) + int(dt.strftime("%M")) / 60.0 + int(
dt.strftime("%S")) / 3600.0
def get_now_str(self):
return str(ISO8601.epoch_seconds_to_ISO8601(self.get_now()))