def test_timestream_create(self):
ts = TimeStream()
ts.version = 1
ts.create(self.tmp_path)
self.assertEqual(ts.path, self.tmp_path)
self.assertDictEqual(ts.image_data, {})
self.assertDictEqual(ts.data, {})
def main():
dirs = filter(ists, glob(r'*/*'))
fullres = filter(isfullres, dirs)
# lowres = filter(is640, dirs)
expts = []
expt_dict = {}
# ts_dict = {}
user = "******"
ts = TimeStream()
for fr_path in fullres:
fr = path.basename(fr_path.rstrip("/"))
ts_name = tsname(fr)
ts.load(fr_path)
expt, loc, _ = splitname(ts_name)
try:
expt_dict[expt]["expt_id"] = expt
except KeyError:
expt_dict[expt] = deepcopy(expt_fmt)
expt_dict[expt]["expt_id"] = expt
expt_dict[expt]["user"] = user
expt_dict[expt]["spp"] = "Unknown Species"
expt_dict[expt]["location"] = loc
expt_dict[expt]["start_date"] = ts.start_datetime.strftime("%Y-%m-%d")
expt_dict[expt]["start_time"] = ts.start_datetime.strftime("%H:%M")
expt_dict[expt]["end_date"] = ts.end_datetime.strftime("%Y-%m-%d")
expt_dict[expt]["end_time"] = ts.end_datetime.strftime("%H:%M")
expt_dict[expt]["timestreams"].append(fr)
# print("TS at {} updates dict for {} to:\n{!r}".format(fr, expt,
# expt_dict[expt]))
for expt, dct in expt_dict.items():
expts.append(dct)
expts_fmt[0]["experiments"] = expts
print(json.dumps(expts_fmt, indent=4))
def test_ts_image_path_assign_parent(self):
"""Test TimeStreamImage path assignment with valid parameters & parent"""
ts = TimeStream()
ts.load(helpers.FILES["timestream"])
img = TimeStreamImage()
img.parent_timestream = ts
img.path = helpers.TS_FILES_JPG[0]
self.assertEqual(img.path, helpers.TS_FILES_JPG[0])
self.assertEqual(img.datetime, helpers.TS_DATES_PARSED[0])
def test_iter_by_timepoints_withgaps(self):
"""Test TimeStream().iter_by_timepoints with a complete timestream"""
ts = TimeStream()
ts.load(helpers.FILES["timestream_gaps"])
res = ts.iter_by_timepoints()
self.assertTrue(isgenerator(res))
for iii, image in enumerate(res):
# Check lazy-loading
self.assertIsNone(image._pixels)
# We don't check path, as it's got a different timestream name
self.assertEqual(image.datetime,
helpers.TS_GAPS_DATES_PARSED[iii])
def _check_ts_instance_ts_v1(self, ts_path):
"""Check members of a TimeStream class instance"""
inst = TimeStream()
inst.load(ts_path)
self.assertEqual(inst.path, ts_path)
self.assertEqual(inst.version, 1)
self.assertEqual(inst.name, path.basename(ts_path))
for key in TS_MANIFEST_KEYS:
if key == "name":
continue
self.assertEqual(getattr(inst, key),
helpers.TS_DICT_PARSED[key])
return inst
def test_timestream_create_bad(self):
ts = TimeStream()
with self.assertRaises(ValueError):
ts.create(self.tmp_path, version=3)
with self.assertRaises(ValueError):
ts.create("/not_a/valid/path")
with self.assertRaises(TypeError):
ts.create("relative/path")
with self.assertRaises(TypeError):
ts.create(123)
def test_iter_by_files(self):
"""Test TimeStream().iter_by_files with a good timestream"""
ts = TimeStream()
ts.load(helpers.FILES["timestream"])
res = ts.iter_by_files()
self.assertTrue(isgenerator(res))
for iii, image in enumerate(res):
self.assertIsNot(image, None)
self.assertEqual(image.path, helpers.TS_FILES_JPG[iii])
self.assertEqual(image.datetime,
helpers.TS_DATES_PARSED[iii])
self.assertIsNot(image.pixels, None)
self.assertEqual(image.pixels.dtype, helpers.TS_JPG_DTYPE)
self.assertEqual(image.pixels.shape, helpers.TS_JPG_SHAPE)
def test_iter_by_timepoints_full(self):
"""Test TimeStream().iter_by_timepoints with a complete timestream"""
ts = TimeStream()
ts.load(helpers.FILES["timestream"])
res = ts.iter_by_timepoints()
self.assertTrue(isgenerator(res))
for iii, image in enumerate(res):
# Check lazy-loading
self.assertIsNot(image, None)
self.assertIsNone(image._pixels)
self.assertEqual(image.path, helpers.TS_FILES_JPG[iii])
self.assertEqual(image.datetime,
helpers.TS_DATES_PARSED[iii])
self.assertIsNot(image.pixels, None)
self.assertEqual(image.pixels.dtype, helpers.TS_JPG_DTYPE)
self.assertEqual(image.pixels.shape, helpers.TS_JPG_SHAPE)
def test_iter_by_timepoints_withgaps_no_rm_gaps(self):
"""Test TimeStream().iter_by_timepoints with a complete timestream"""
ts = TimeStream()
ts.load(helpers.FILES["timestream_gaps"])
res = ts.iter_by_timepoints(remove_gaps=False)
self.assertTrue(isgenerator(res))
for iii, image in enumerate(res):
if iii in {3, 5}:
# Missing images
self.assertEqual(0, len(image.pixels))
continue
self.assertIsNot(image, None)
# Check lazy-loading
self.assertIsNone(image._pixels)
# We don't check path, as it's got a different timestream name
self.assertEqual(image.datetime,
helpers.TS_DATES_PARSED[iii])
def test_read_metatdata_weird(self):
"""Do weird things to TimeStream instance and check methods raise"""
inst = TimeStream()
with self.assertRaises(RuntimeError):
inst.read_metadata()
inst.load(helpers.FILES["timestream"])
del inst.path
with self.assertRaises(AttributeError):
inst.read_metadata()
def main(opts):
setup_module_logging(logging.ERROR)
ts_name = opts["-i"]
out_fh = open(opts['-o'], 'w')
ts_info = ts_get_manifest(ts_name)
times = setup_header(ts_info)
out_csv = csv.writer(out_fh)
header = ["Date", ]
header.extend(times)
out_csv.writerow(header)
ts = TimeStream()
ts.load(ts_name)
res_dict = {}
print("Collecting image data for {}...".format(ts_name))
count = 0
for img in ts.iter_by_timepoints():
if count % 5 == 0:
print("Processed {} images!".format(count), end='\r')
sys.stdout.flush()
count += 1
img_dt = img.datetime
try:
res_dict[img_dt.date()][img_dt.time().isoformat()] = 1
except KeyError:
res_dict[img_dt.date()] = {img_dt.time().isoformat(): 1}
print("Processed {} images!".format(count))
print("Done collecting image sums, now making the table")
for this_date, times in sorted(res_dict.items()):
row = []
row.append(this_date.isoformat())
start_today = datetime.combine(this_date.today(), time.min)
end_today = datetime.combine(this_date.today(), time.max)
all_times = iter_date_range(start_today, end_today,
ts_info['interval'] * 60)
for timepoint in all_times:
try:
row.append(times[timepoint.time().isoformat()])
except KeyError:
row.append("0")
out_csv.writerow(row)
print("All done!")
def derandomize(tsoutpath, derandStruct, timestamps):
plc = pipeconf.PCFGSection("--")
plc.setVal("pipeline._0.name", "derandomize")
plc.setVal("pipeline._0.derandStruct", derandStruct)
plc.setVal("pipeline._1.name", "imagewrite")
plc.setVal("pipeline._1.outstream", "outts")
outts = TimeStream()
outts.name = "derandomized"
outts.create(tsoutpath)
ctx = pipeconf.PCFGSection("--")
ctx.setVal("outts.outts", outts)
pl = pipeline.ImagePipeline(plc.pipeline, ctx)
# 4. Execute pipeline
for i in range(len(timestamps)):
ts = timestamps[i]
try:
result = pl.process(ctx, [ts], True)
except Exception as e:
print("Skipped {}".format(ts))
continue
def test_timestream_write(self):
ts = TimeStream()
ts.version = 1
ts.create(self.tmp_path, ext="jpg")
self.assertEqual(ts.path, self.tmp_path)
self.assertDictEqual(ts.image_data, {})
self.assertDictEqual(ts.data, {})
for _ in range(10):
img = TimeStreamImage()
arr = np.arange(300, dtype="uint8")
arr = arr.reshape((10, 10, 3))
date = dt.datetime.now()
str_date = ts_format_date(date)
img.pixels = arr
img.datetime = date
img.data["fake"] = True
ts.write_image(img)
self.assertIn(str_date, ts.image_data)
self.assertDictEqual(img.data, ts.image_data[str_date])
self.assertTrue(path.exists, img.path)
def test_timestream_str(self):
"""Test TimeStream str(instance) with good timestream"""
inst = TimeStream()
inst.load(helpers.FILES["timestream"])
self.assertEqual(str(inst), helpers.TS_STR)
def test_timestream_load_bad(self):
"""Test TimeStream initialisation with bad/non timestream"""
with self.assertRaises(ValueError):
inst = TimeStream()
inst.load(helpers.FILES["not_a_timestream"])
with self.assertRaises(ValueError):
inst = TimeStream()
inst.load(helpers.FILES["timestream_bad"])
with self.assertRaises(TypeError):
inst = TimeStream()
inst.load(None)
with self.assertRaises(ValueError):
inst = TimeStream()
inst.version = 42
inst.load(helpers.FILES["timestream"])
def _derandomize(self):
# 0. Get the output directory
tsoutpath = QtGui.QFileDialog.getExistingDirectory(self, \
"Select Output Derandomization Directory", self._currDir, \
QtGui.QFileDialog.ShowDirsOnly \
| QtGui.QFileDialog.DontResolveSymlinks)
tsoutpath = str(tsoutpath)
if tsoutpath == "": # Handle the cancel
return
self._currDir = os.path.dirname(str(tsoutpath))
# 1. Gather all timestamps
timestamps = []
for r in range(self._ui.tslist.rowCount()-1):
i = self._ui.tslist.item(r,1)
tst = i.data(QtCore.Qt.UserRole).toPyObject()
timestamps = timestamps + tst.timestamps
timestamps = sorted(set(timestamps))
# 2. Get derandStruct
derandStruct = self._createDerandStruct()
# 3. Create pipeline components
plc = pipeconf.PCFGSection("--")
plc.setVal("pipeline._0.name", "derandomize")
plc.setVal("pipeline._0.derandStruct", derandStruct)
plc.setVal("pipeline._1.name", "imagewrite")
plc.setVal("pipeline._1.outstream", "outts")
outts = TimeStream()
outts.name = "derandomized"
outts.create(tsoutpath)
ctx = pipeconf.PCFGSection("--")
ctx.setVal("outts.outts", outts)
pl = pipeline.ImagePipeline(plc.pipeline, ctx)
# 4. Execute pipeline
self._ui.bCancelClicked = False
self._ui.pbts.setVisible(True)
self._ui.bCancel.setVisible(True)
self._ui.pbts.setMinimum(0)
self._ui.pbts.setMaximum(len(timestamps))
self._ui.pbts.reset()
for i in range(len(timestamps)):
self._ui.pbts.setValue(i)
QtGui.qApp.processEvents()
ts = timestamps[i]
try:
result = pl.process(ctx, [ts], True)
except Exception as e:
errmsg = QtGui.QErrorMessage(self)
errmsg.setWindowTitle("Error Derandomizing to ". \
format(tsoutpath))
errmsg.showMessage(str(e))
break
if self._ui.bCancelClicked:
break
self._ui.pbts.setValue(self._ui.pbts.maximum())
self._ui.pbts.setVisible(False)
self._ui.bCancel.setVisible(False)
