def test_timestream_create(self): ts = TimeStream() ts.version = 1 ts.create(self.tmp_path) self.assertEqual(ts.path, self.tmp_path) self.assertDictEqual(ts.image_data, {}) self.assertDictEqual(ts.data, {})
def main(): dirs = filter(ists, glob(r'*/*')) fullres = filter(isfullres, dirs) # lowres = filter(is640, dirs) expts = [] expt_dict = {} # ts_dict = {} user = "******" ts = TimeStream() for fr_path in fullres: fr = path.basename(fr_path.rstrip("/")) ts_name = tsname(fr) ts.load(fr_path) expt, loc, _ = splitname(ts_name) try: expt_dict[expt]["expt_id"] = expt except KeyError: expt_dict[expt] = deepcopy(expt_fmt) expt_dict[expt]["expt_id"] = expt expt_dict[expt]["user"] = user expt_dict[expt]["spp"] = "Unknown Species" expt_dict[expt]["location"] = loc expt_dict[expt]["start_date"] = ts.start_datetime.strftime("%Y-%m-%d") expt_dict[expt]["start_time"] = ts.start_datetime.strftime("%H:%M") expt_dict[expt]["end_date"] = ts.end_datetime.strftime("%Y-%m-%d") expt_dict[expt]["end_time"] = ts.end_datetime.strftime("%H:%M") expt_dict[expt]["timestreams"].append(fr) # print("TS at {} updates dict for {} to:\n{!r}".format(fr, expt, # expt_dict[expt])) for expt, dct in expt_dict.items(): expts.append(dct) expts_fmt[0]["experiments"] = expts print(json.dumps(expts_fmt, indent=4))
def test_ts_image_path_assign_parent(self): """Test TimeStreamImage path assignment with valid parameters & parent""" ts = TimeStream() ts.load(helpers.FILES["timestream"]) img = TimeStreamImage() img.parent_timestream = ts img.path = helpers.TS_FILES_JPG[0] self.assertEqual(img.path, helpers.TS_FILES_JPG[0]) self.assertEqual(img.datetime, helpers.TS_DATES_PARSED[0])
def test_iter_by_timepoints_withgaps(self): """Test TimeStream().iter_by_timepoints with a complete timestream""" ts = TimeStream() ts.load(helpers.FILES["timestream_gaps"]) res = ts.iter_by_timepoints() self.assertTrue(isgenerator(res)) for iii, image in enumerate(res): # Check lazy-loading self.assertIsNone(image._pixels) # We don't check path, as it's got a different timestream name self.assertEqual(image.datetime, helpers.TS_GAPS_DATES_PARSED[iii])
def _check_ts_instance_ts_v1(self, ts_path): """Check members of a TimeStream class instance""" inst = TimeStream() inst.load(ts_path) self.assertEqual(inst.path, ts_path) self.assertEqual(inst.version, 1) self.assertEqual(inst.name, path.basename(ts_path)) for key in TS_MANIFEST_KEYS: if key == "name": continue self.assertEqual(getattr(inst, key), helpers.TS_DICT_PARSED[key]) return inst
def test_timestream_create_bad(self): ts = TimeStream() with self.assertRaises(ValueError): ts.create(self.tmp_path, version=3) with self.assertRaises(ValueError): ts.create("/not_a/valid/path") with self.assertRaises(TypeError): ts.create("relative/path") with self.assertRaises(TypeError): ts.create(123)
def test_iter_by_files(self): """Test TimeStream().iter_by_files with a good timestream""" ts = TimeStream() ts.load(helpers.FILES["timestream"]) res = ts.iter_by_files() self.assertTrue(isgenerator(res)) for iii, image in enumerate(res): self.assertIsNot(image, None) self.assertEqual(image.path, helpers.TS_FILES_JPG[iii]) self.assertEqual(image.datetime, helpers.TS_DATES_PARSED[iii]) self.assertIsNot(image.pixels, None) self.assertEqual(image.pixels.dtype, helpers.TS_JPG_DTYPE) self.assertEqual(image.pixels.shape, helpers.TS_JPG_SHAPE)
def test_iter_by_timepoints_full(self): """Test TimeStream().iter_by_timepoints with a complete timestream""" ts = TimeStream() ts.load(helpers.FILES["timestream"]) res = ts.iter_by_timepoints() self.assertTrue(isgenerator(res)) for iii, image in enumerate(res): # Check lazy-loading self.assertIsNot(image, None) self.assertIsNone(image._pixels) self.assertEqual(image.path, helpers.TS_FILES_JPG[iii]) self.assertEqual(image.datetime, helpers.TS_DATES_PARSED[iii]) self.assertIsNot(image.pixels, None) self.assertEqual(image.pixels.dtype, helpers.TS_JPG_DTYPE) self.assertEqual(image.pixels.shape, helpers.TS_JPG_SHAPE)
def test_iter_by_timepoints_withgaps_no_rm_gaps(self): """Test TimeStream().iter_by_timepoints with a complete timestream""" ts = TimeStream() ts.load(helpers.FILES["timestream_gaps"]) res = ts.iter_by_timepoints(remove_gaps=False) self.assertTrue(isgenerator(res)) for iii, image in enumerate(res): if iii in {3, 5}: # Missing images self.assertEqual(0, len(image.pixels)) continue self.assertIsNot(image, None) # Check lazy-loading self.assertIsNone(image._pixels) # We don't check path, as it's got a different timestream name self.assertEqual(image.datetime, helpers.TS_DATES_PARSED[iii])
def test_read_metatdata_weird(self): """Do weird things to TimeStream instance and check methods raise""" inst = TimeStream() with self.assertRaises(RuntimeError): inst.read_metadata() inst.load(helpers.FILES["timestream"]) del inst.path with self.assertRaises(AttributeError): inst.read_metadata()
def main(opts): setup_module_logging(logging.ERROR) ts_name = opts["-i"] out_fh = open(opts['-o'], 'w') ts_info = ts_get_manifest(ts_name) times = setup_header(ts_info) out_csv = csv.writer(out_fh) header = ["Date", ] header.extend(times) out_csv.writerow(header) ts = TimeStream() ts.load(ts_name) res_dict = {} print("Collecting image data for {}...".format(ts_name)) count = 0 for img in ts.iter_by_timepoints(): if count % 5 == 0: print("Processed {} images!".format(count), end='\r') sys.stdout.flush() count += 1 img_dt = img.datetime try: res_dict[img_dt.date()][img_dt.time().isoformat()] = 1 except KeyError: res_dict[img_dt.date()] = {img_dt.time().isoformat(): 1} print("Processed {} images!".format(count)) print("Done collecting image sums, now making the table") for this_date, times in sorted(res_dict.items()): row = [] row.append(this_date.isoformat()) start_today = datetime.combine(this_date.today(), time.min) end_today = datetime.combine(this_date.today(), time.max) all_times = iter_date_range(start_today, end_today, ts_info['interval'] * 60) for timepoint in all_times: try: row.append(times[timepoint.time().isoformat()]) except KeyError: row.append("0") out_csv.writerow(row) print("All done!")
def derandomize(tsoutpath, derandStruct, timestamps): plc = pipeconf.PCFGSection("--") plc.setVal("pipeline._0.name", "derandomize") plc.setVal("pipeline._0.derandStruct", derandStruct) plc.setVal("pipeline._1.name", "imagewrite") plc.setVal("pipeline._1.outstream", "outts") outts = TimeStream() outts.name = "derandomized" outts.create(tsoutpath) ctx = pipeconf.PCFGSection("--") ctx.setVal("outts.outts", outts) pl = pipeline.ImagePipeline(plc.pipeline, ctx) # 4. Execute pipeline for i in range(len(timestamps)): ts = timestamps[i] try: result = pl.process(ctx, [ts], True) except Exception as e: print("Skipped {}".format(ts)) continue
def test_timestream_write(self): ts = TimeStream() ts.version = 1 ts.create(self.tmp_path, ext="jpg") self.assertEqual(ts.path, self.tmp_path) self.assertDictEqual(ts.image_data, {}) self.assertDictEqual(ts.data, {}) for _ in range(10): img = TimeStreamImage() arr = np.arange(300, dtype="uint8") arr = arr.reshape((10, 10, 3)) date = dt.datetime.now() str_date = ts_format_date(date) img.pixels = arr img.datetime = date img.data["fake"] = True ts.write_image(img) self.assertIn(str_date, ts.image_data) self.assertDictEqual(img.data, ts.image_data[str_date]) self.assertTrue(path.exists, img.path)
def test_timestream_str(self): """Test TimeStream str(instance) with good timestream""" inst = TimeStream() inst.load(helpers.FILES["timestream"]) self.assertEqual(str(inst), helpers.TS_STR)
def test_timestream_load_bad(self): """Test TimeStream initialisation with bad/non timestream""" with self.assertRaises(ValueError): inst = TimeStream() inst.load(helpers.FILES["not_a_timestream"]) with self.assertRaises(ValueError): inst = TimeStream() inst.load(helpers.FILES["timestream_bad"]) with self.assertRaises(TypeError): inst = TimeStream() inst.load(None) with self.assertRaises(ValueError): inst = TimeStream() inst.version = 42 inst.load(helpers.FILES["timestream"])
def _derandomize(self): # 0. Get the output directory tsoutpath = QtGui.QFileDialog.getExistingDirectory(self, \ "Select Output Derandomization Directory", self._currDir, \ QtGui.QFileDialog.ShowDirsOnly \ | QtGui.QFileDialog.DontResolveSymlinks) tsoutpath = str(tsoutpath) if tsoutpath == "": # Handle the cancel return self._currDir = os.path.dirname(str(tsoutpath)) # 1. Gather all timestamps timestamps = [] for r in range(self._ui.tslist.rowCount()-1): i = self._ui.tslist.item(r,1) tst = i.data(QtCore.Qt.UserRole).toPyObject() timestamps = timestamps + tst.timestamps timestamps = sorted(set(timestamps)) # 2. Get derandStruct derandStruct = self._createDerandStruct() # 3. Create pipeline components plc = pipeconf.PCFGSection("--") plc.setVal("pipeline._0.name", "derandomize") plc.setVal("pipeline._0.derandStruct", derandStruct) plc.setVal("pipeline._1.name", "imagewrite") plc.setVal("pipeline._1.outstream", "outts") outts = TimeStream() outts.name = "derandomized" outts.create(tsoutpath) ctx = pipeconf.PCFGSection("--") ctx.setVal("outts.outts", outts) pl = pipeline.ImagePipeline(plc.pipeline, ctx) # 4. Execute pipeline self._ui.bCancelClicked = False self._ui.pbts.setVisible(True) self._ui.bCancel.setVisible(True) self._ui.pbts.setMinimum(0) self._ui.pbts.setMaximum(len(timestamps)) self._ui.pbts.reset() for i in range(len(timestamps)): self._ui.pbts.setValue(i) QtGui.qApp.processEvents() ts = timestamps[i] try: result = pl.process(ctx, [ts], True) except Exception as e: errmsg = QtGui.QErrorMessage(self) errmsg.setWindowTitle("Error Derandomizing to ". \ format(tsoutpath)) errmsg.showMessage(str(e)) break if self._ui.bCancelClicked: break self._ui.pbts.setValue(self._ui.pbts.maximum()) self._ui.pbts.setVisible(False) self._ui.bCancel.setVisible(False)