def retrieve(self, pk_value):
try:
document = self.__documents[pk_value]
except KeyError:
return None
node = PathDict()
for path in document.paths():
if path != self.pk_path:
node.append(path, document.node(path))
return node
def construct_from_jstr(cls, jstr):
if not jstr:
return None
# document...
document = PathDict.construct_from_jstr(jstr)
if not document:
return None
# upload...
upload_node = document.node(cls.UPLOAD_FIELD)
upload = LocalizedDatetime.construct_from_iso8601(upload_node)
if upload is None:
raise ValueError(upload_node)
# rec...
rec_node = document.node(cls.REC_FIELD)
rec = LocalizedDatetime.construct_from_iso8601(rec_node)
if rec is None:
raise ValueError(rec_node)
# offset...
td = upload - rec
offset = Timedelta(days=td.days, seconds=td.seconds)
return UploadInterval(upload, rec, offset)
def construct_from_jstr(cls, jstr):
try:
jdict = json.loads(jstr, object_pairs_hook=OrderedDict)
except ValueError:
return None
return CSVDict(PathDict(jdict))
def site_conf(cls, datum: PathDict):
jdict = datum.node('status.val.airnow')
conf = AirNowSiteConf.construct_from_jdict(jdict)
if conf is None:
raise KeyError("no site configuration found for %s" % str(jdict))
return conf
def construct_from_jdict(cls, jdict):
if not jdict:
return None
topic = jdict.get('topic')
date = datetime.datetime.fromtimestamp(jdict.get('date') / 1e3) # Warning: JavaScript timestamp!
message_jstr = jdict.get('message')
message_jdict = PathDict.construct_from_jstr(message_jstr)
return MessageEvent(topic, date, message_jdict)
def datum(self, datum):
we_v = float(datum.node(self.__path + '.weV'))
ae_v = float(datum.node(self.__path + '.aeV'))
diff = we_v - ae_v
conversion = (diff * 1000) / float(
self.__sensitivity
) # value [ppb] = raw [mV] / sensitivity [mV / ppb]
target = PathDict()
target.copy(datum, 'rec', self.__path + '.weV', self.__path + '.aeV')
target.append(self.__path + '.diff', round(diff, 6))
target.append(self.__path + '.conv', round(conversion, 6))
return target.node()
def append(self, datetime: LocalizedDatetime, sample: PathDict):
# initialise...
if not self.__initialised:
for node in self.__nodes:
try:
paths = sample.paths(node)
except KeyError:
continue
except IndexError as ex:
paths = None
print("sample_aggregate: %s: IndexError: %s" % (node, ex), file=sys.stderr)
sys.stderr.flush()
exit(1)
for path in paths:
if path == 'rec':
continue
self.__precisions[path] = Precision()
self.__regressions[path] = LinearRegression() if Datum.is_numeric(sample.node(path)) else \
CategoricalRegression()
self.__initialised = True
# values...
for path in self.__precisions.keys():
try:
value = sample.node(path)
except KeyError:
continue
if value is None:
continue
try:
self.__precisions[path].widen(value)
self.__regressions[path].append(datetime, value)
except InvalidOperation:
continue
def append(self, document: PathDict):
pk_value = document.node(self.pk_path)
if self.pk_is_iso8601:
datetime = LocalizedDatetime.construct_from_iso8601(pk_value)
if datetime is None:
raise ValueError(pk_value)
pk_value = datetime
self.__documents[pk_value] = document
def right(self):
# paths...
pk_path = self.__right.pk_path
right_path = self.__right.set_path
left_path = self.__left.set_path
# join...
for pk in self.__right.pk_values():
right = self.__right.retrieve(pk)
left = self.__left.retrieve(pk)
yield PathDict.union((pk_path, pk), (left_path, left),
(right_path, right))
def datum(self, sample):
if not sample.has_path(self.__path):
return None
value = sample.node(self.__path)
self.__func.append(value)
avg = self.__func.compute()
if avg is None:
return None
target = PathDict()
if sample.has_path('rec'):
target.copy(sample, 'rec')
target.append(self.__path + '.src', value)
target.append(self.__path + '.avg', round(avg, self.__precision))
return target.node()
def datum(self, sample):
if not sample.has_path(self.__path):
return None
value = sample.node(self.__path)
self.__func.append(value)
if not self.__func.has_tally():
return None
avg = self.__func.compute()
if avg is None:
return None
target = PathDict()
target.copy(datum, 'rec')
target.append(self.__path + '.src', value)
target.append(self.__path + '.avg', round(avg, 6))
return target.node()
def full(self):
# paths...
pk_path = self.__left.pk_path
right_path = self.__right.set_path
left_path = self.__left.set_path
# keys...
right_pk_values = set(self.__right.pk_values())
left_pk_values = set(self.__left.pk_values())
pk_values = sorted(right_pk_values | left_pk_values)
# join...
for pk in pk_values:
right = self.__right.retrieve(pk)
left = self.__left.retrieve(pk)
yield PathDict.union((pk_path, pk), (left_path, left),
(right_path, right))
def aqcsv_rec(self, datum: PathDict):
localised = LocalizedDatetime.construct_from_jdict(datum.node('rec'))
timezone = self.timezone(datum)
return AQCSVDatetime(localised.datetime, timezone.zone)
def includes(self, path):
for sub_path in self.sub_paths:
if PathDict.sub_path_includes_path(sub_path, path):
return not self.exclude
return self.exclude
print("node: %s" % cmd, file=sys.stderr)
sys.stderr.flush()
try:
# ------------------------------------------------------------------------------------------------------------
# run...
if cmd.array:
print('[', end='')
node = None
first = True
for line in sys.stdin:
jstr = line.strip()
datum = PathDict.construct_from_jstr(jstr)
if datum is None:
continue
document_count += 1
if cmd.exclude and not cmd.sub_paths:
continue # everything is excluded
# build...
if not cmd.sub_paths:
target = datum
else:
target = PathDict()
def gps(cls, datum: PathDict):
jdict = datum.node('status.val.gps')
return GPSDatum.construct_from_jdict(jdict)
def timezone(cls, datum: PathDict):
jdict = datum.node('status.val.tz')
return Timezone.construct_from_jdict(jdict)
def status_tag(cls, datum: PathDict):
return datum.node('status.tag')
# ----------------------------------------------------------------------------------------------------------------
# resources...
lpf = LowPassFilter.construct(cmd.delta, cmd.cut_off)
if cmd.verbose:
print("sample_low_pass: %s" % lpf, file=sys.stderr)
sys.stderr.flush()
try:
# ------------------------------------------------------------------------------------------------------------
# run...
for line in sys.stdin:
datum = PathDict.construct_from_jstr(line)
if datum is None:
break
document_count += 1
value = datum.node(cmd.path)
if value is None:
break
target = PathDict()
if datum.has_path('rec'):
target.copy(datum, 'rec')
cmd = CmdNode()
if not cmd.is_valid():
cmd.print_help(sys.stderr)
exit(2)
if cmd.verbose:
print(cmd, file=sys.stderr)
sys.stderr.flush()
try:
# ------------------------------------------------------------------------------------------------------------
# run...
for line in sys.stdin:
datum = PathDict.construct_from_jstr(line)
if datum is None:
continue
if cmd.ignore and not datum.has_path(cmd.path):
continue
node = datum.node(cmd.path)
print(JSONify.dumps(node))
sys.stdout.flush()
# ----------------------------------------------------------------------------------------------------------------
# end...
def datum(self, sample):
if not sample.has_path(self.__path):
return None
try:
rec_node = sample.node('rec')
except KeyError:
return None
rec = LocalizedDatetime.construct_from_jdict(rec_node)
value = sample.node(self.__path)
self.__func.append(rec, value)
if not self.__func.has_regression():
return None
slope, intercept = self.__func.line()
if slope is None:
return None
target = PathDict()
if sample.has_path('rec'):
target.copy(sample, 'rec')
target.append(self.__path + '.src', value)
target.append(self.__path + '.slope', round(slope, self.__precision))
target.append(self.__path + '.intercept',
round(intercept, self.__precision))
return target.node()
def as_json(self):
jdict = PathDict()
jdict.append('domain', self.domain())
jdict.append('domain-midpoint', self.domain_midpoint())
jdict.append(self.ind_name + '.min', self.ind_min())
jdict.append(self.ind_name + '.avg', self.ind_avg())
jdict.append(self.ind_name + '.max', self.ind_max())
for name in self.dep_names:
jdict.append(name + '.avg', self.dep_avg(name))
jdict.append(name + '.stdev', self.dep_stdev(name))
jdict.append('samples', len(self))
return jdict
ndir.power_on()
if cmd.default:
calib = calib_class.default()
# save...
ndir.store_calib(calib)
elif cmd.restart:
ndir.reload_calib()
elif cmd.set():
# retrieve...
calib = ndir.retrieve_calib()
dictionary = PathDict.construct_from_jstr(JSONify.dumps(calib))
# validate...
if not dictionary.has_path(cmd.path):
print("ndir_calib: field name not known: %s" % cmd.path,
file=sys.stderr)
exit(2)
# set...
dictionary.append(cmd.path, cmd.value)
calib = calib_class.construct_from_jdict(dictionary.as_json())
# validate...
dictionary = PathDict.construct_from_jstr(JSONify.dumps(calib))
if dictionary.node(cmd.path) is None:
'"scs-status": {"tally": 1.0, "interval": 60.0}, ' \
'"scs-climate": {"tally": 1.0, "interval": 60.0}, "scs-gases": {"tally": 1.0, "interval": 10.0}}, ' \
'"gps": {"pos": [50.83390976, -0.13919364], "qual": 1.0, "elv": 44.1}, ' \
'"airnow": {"site": 826987654321.0}}, ' \
'"tag": "scs-bgx-401"}}'
print(jstr)
print("=")
duration = 1
mapping = DatumMapping("particulates", "pm1")
print(mapping)
print("-")
datum = PathDict.construct_from_jstr(jstr)
print(datum)
print("-")
print(" tag: %s" % mapping.environment_tag(datum))
print(" value: %s" % mapping.value(datum))
print(" source: %s" % mapping.source(datum))
print(" mapping: %s" % mapping.aqcsv_source_mapping(datum))
print("-")
print(JSONify.dumps(mapping))
print("-")
record = mapping.aqcsv_record(datum, duration)
print(record)
print("-")
def datum(self, datum):
latest = float(datum.node(self.__path))
if self.__aggregate is None:
self.__aggregate = latest
return None
self.__aggregate = (0.9 * self.__aggregate) + (0.1 * latest)
error = latest - self.__aggregate
target = PathDict()
target.copy(datum, 'rec')
target.append(self.__path + '.src', latest)
target.append(self.__path + '.agr', round(self.__aggregate, 6))
target.append(self.__path + '.err', round(error, 6))
return target.node()
def report(self, localised_datetime):
report = PathDict()
# rec...
report.append(self.__iso_path, localised_datetime.as_iso8601())
# values...
for path, precision in self.__precisions.items():
regression = self.__regressions[path]
if self.__regressions[path].has_midpoint():
_, midpoint = regression.midpoint(precision.digits)
if self.__min_max:
report.append(path + '.min', regression.min(precision.digits))
report.append(path + '.mid', midpoint)
report.append(path + '.max', regression.max(precision.digits))
else:
report.append(path, midpoint)
return report
cmd = CmdSampleAH()
if not cmd.is_valid():
cmd.print_help(sys.stderr)
exit(2)
if cmd.verbose:
print("sample_ah: %s" % cmd, file=sys.stderr)
try:
# ------------------------------------------------------------------------------------------------------------
# run...
for line in sys.stdin:
jstr = line.strip()
datum = PathDict.construct_from_jstr(jstr)
if datum is None:
continue
document_count += 1
paths = datum.paths()
# rH / t...
if cmd.rh_path not in paths or cmd.t_path not in paths:
continue
rh_node = datum.node(cmd.rh_path)
t_node = datum.node(cmd.t_path)
def value(self, datum: PathDict):
species_path = '.'.join([self.topic, 'val', self.species])
return datum.node(species_path)
def environment_tag(self, datum: PathDict):
tag_path = '.'.join([self.topic, 'tag'])
return datum.node(tag_path)
def source(self, datum: PathDict):
source_path = '.'.join([self.topic, 'src'])
return datum.node(source_path)