def main():
init_logging()
config = load_config()
build_config = config.build
db = TrackDatabase(os.path.join(config.tracks_folder, "dataset.hdf5"))
dataset = Dataset(db, "dataset", config)
tracks_loaded, total_tracks = dataset.load_tracks()
print(
"Loaded {}/{} tracks, found {:.1f}k segments".format(
tracks_loaded, total_tracks, len(dataset.segments) / 1000
)
)
for key, value in dataset.filtered_stats.items():
if value != 0:
print(" {} filtered {}".format(key, value))
print()
show_tracks_breakdown(dataset)
print()
show_segments_breakdown(dataset)
print()
show_cameras_breakdown(dataset)
print()
print("Splitting data set into train / validation")
datasets = split_dataset_by_cameras(db, dataset, build_config)
# if build_config.use_previous_split:
# split = get_previous_validation_bins(build_config.previous_split)
# datasets = split_dataset(db, dataset, build_config, split)
# else:
# datasets = split_dataset(db, dataset, build_config)
pickle.dump(datasets, open(dataset_db_path(config), "wb"))
def main():
args = parse_args()
init_logging()
config = Config.load_from_file(args.config_file)
thermal_config = ThermalConfig.load_from_file(args.thermal_config_file)
print("detecting on " + args.cptv)
with open(args.cptv, "rb") as f:
reader = CPTVReader(f)
headers = HeaderInfo(
res_x=reader.x_resolution,
res_y=reader.y_resolution,
fps=9,
brand="",
model="",
frame_size=reader.x_resolution * reader.y_resolution * 2,
pixel_bits=16,
)
motion_detector = MotionDetector(
thermal_config, config.tracking.motion_config.dynamic_thresh, None,
headers)
for i, frame in enumerate(reader):
motion_detector.process_frame(frame)
def main():
logging.root.removeHandler(absl.logging._absl_handler)
absl.logging._warn_preinit_stderr = False
init_logging()
args = parse_args()
config = Config.load_from_file(args.config_file)
thermal_config = ThermalConfig.load_from_file(args.thermal_config_file)
if args.cptv:
return parse_cptv(args.cptv, config, thermal_config)
try:
os.unlink(SOCKET_NAME)
except OSError:
if os.path.exists(SOCKET_NAME):
raise
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(SOCKET_NAME)
sock.listen(1)
while True:
logging.info("waiting for a connection")
connection, client_address = sock.accept()
logging.info("connection from %s", client_address)
try:
handle_connection(connection, config, thermal_config)
finally:
# Clean up the connection
connection.close()
def main():
init_logging()
args = parse_args()
config = Config.load_from_file(args.config_file)
if args.cptv:
return parse_cptv(args.cptv, config, args.thermal_config_file,
args.preview_type)
try:
os.unlink(SOCKET_NAME)
except OSError:
if os.path.exists(SOCKET_NAME):
raise
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(SOCKET_NAME)
sock.listen(1)
while True:
logging.info("waiting for a connection")
connection, client_address = sock.accept()
logging.info("connection from %s", client_address)
try:
handle_connection(connection, config, args.thermal_config_file)
except:
logging.error("Error with connection", exc_info=True)
# return
finally:
# Clean up the connection
try:
connection.close()
except:
pass
def parse_params():
parser = argparse.ArgumentParser()
parser.add_argument(
"-target",
default=None,
help=
'Target to process, "all" processes all folders, "test" runs test cases, "clean" to remove banned clips from db, or a "cptv" file to run a single source.',
)
parser.add_argument(
"-p",
"--create-previews",
action="count",
help="Create MP4 previews for tracks (can be slow)",
)
parser.add_argument(
"-t",
"--test-file",
default="tests.txt",
help="File containing test cases to run",
)
parser.add_argument("-v",
"--verbose",
action="count",
help="Display additional information.")
parser.add_argument(
"-r",
"--reprocess",
action="count",
help="Re process clips that already exist in the database",
)
parser.add_argument(
"-i",
"--show-build-information",
action="count",
help="Show openCV build information and exit.",
)
parser.add_argument("-c",
"--config-file",
help="Path to config file to use")
args = parser.parse_args()
if args.show_build_information:
print(cv2.getBuildInformation())
return None, None
init_logging()
config = Config.load_from_file(args.config_file)
if args.create_previews:
config.loader.preview = "tracking"
if args.verbose:
config.tracking.verbose = True
return config, args
def main():
init_logging()
args = parse_args()
config = load_config(args.config_file)
# return
# import yaml
#
# with open("defualtstest.yml", "w") as f:
# yaml.dump(config.as_dict(), f)
test_clips = config.build.test_clips()
if test_clips is None:
test_clips = []
logging.info("# of test clips are %s", len(test_clips))
db_file = os.path.join(config.tracks_folder, "dataset.hdf5")
dataset = Dataset(db_file,
"dataset",
config,
consecutive_segments=args.consecutive_segments)
tracks_loaded, total_tracks = dataset.load_tracks()
dataset.labels.sort()
print("Loaded {}/{} tracks, found {:.1f}k segments".format(
tracks_loaded, total_tracks,
len(dataset.segments) / 1000))
for key, value in dataset.filtered_stats.items():
if value != 0:
print(" {} filtered {}".format(key, value))
print()
show_tracks_breakdown(dataset)
print()
show_segments_breakdown(dataset)
print()
show_sample_frames_breakdown(dataset)
print()
show_cameras_breakdown(dataset)
print()
print("Splitting data set into train / validation")
datasets = split_randomly(db_file, dataset, config, args, test_clips)
validate_datasets(datasets, test_clips, args.date)
print_counts(dataset, *datasets)
base_dir = config.tracks_folder
for dataset in datasets:
dataset.saveto_numpy(os.path.join(base_dir))
for dataset in datasets:
dataset.clear_samples()
dataset.db = None
logging.info("saving to %s",
f"{os.path.join(base_dir, dataset.name)}.dat")
pickle.dump(dataset,
open(f"{os.path.join(base_dir, dataset.name)}.dat", "wb"))
def main():
logging.root.removeHandler(absl.logging._absl_handler)
absl.logging._warn_preinit_stderr = False
parser = argparse.ArgumentParser()
parser.add_argument(
"source",
help='a CPTV file to process, or a folder name, or "all" for all files within subdirectories of source folder.',
)
parser.add_argument(
"-p",
"--preview-type",
help="Create MP4 previews of this type (can be slow), this overrides the config",
)
parser.add_argument(
"-v", "--verbose", action="count", help="Display additional information."
)
parser.add_argument(
"-o",
"--meta-to-stdout",
action="count",
help="Print metadata to stdout instead of saving to file.",
)
parser.add_argument("-c", "--config-file", help="Path to config file to use")
parser.add_argument(
"-T", "--timestamps", action="store_true", help="Emit log timestamps"
)
parser.add_argument(
"--cache",
type=str2bool,
nargs="?",
const=True,
default=None,
help="Dont keep video frames in memory for classification later, but cache them to disk (Best for large videos, but slower)",
)
args = parser.parse_args()
config = Config.load_from_file(args.config_file)
config.validate()
init_logging(args.timestamps)
# parse command line arguments
if args.preview_type:
config.classify.preview = args.preview_type
if args.verbose:
config.classify_tracking.verbose = True
if args.meta_to_stdout:
config.classify.meta_to_stdout = True
extractor = TrackExtractor(config, cache_to_disk=args.cache)
extractor.extract(args.source)
def main():
conf, job_name = load_config()
init_logging()
# tf.logging.set_verbosity(3)
os.makedirs(conf.train.train_dir, exist_ok=True)
if job_name == "search":
axis_search(conf)
else:
train_model(job_name, conf, conf.train.hyper_params)
def main():
conf, args = load_config()
init_logging()
# tf.logging.set_verbosity(3)
os.makedirs(conf.train.train_dir, exist_ok=True)
train_model(
args.name,
conf,
conf.train.hyper_params,
grid_search=args.grid,
weights=args.weights,
)
def main():
logging.root.removeHandler(absl.logging._absl_handler)
absl.logging._warn_preinit_stderr = False
init_logging()
args = parse_args()
config = Config.load_from_file()
thermal_config = ThermalConfig.load_from_file()
proccesor = None
if thermal_config.motion.run_classifier:
classifier = get_classifier(config)
proccesor = PiClassifier(config, thermal_config, classifier)
else:
proccesor = MotionDetector(
config.res_x,
config.res_y,
thermal_config,
config.tracking.dynamic_thresh,
CPTVRecorder(thermal_config),
)
if args.cptv:
with open(args.cptv, "rb") as f:
reader = CPTVReader(f)
for frame in reader:
proccesor.process_frame(frame)
proccesor.disconnected()
return
service = SnapshotService(proccesor)
try:
os.unlink(SOCKET_NAME)
except OSError:
if os.path.exists(SOCKET_NAME):
raise
sock = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET)
sock.bind(SOCKET_NAME)
sock.listen(1)
while True:
logging.info("waiting for a connection")
connection, client_address = sock.accept()
logging.info("connection from %s", client_address)
try:
handle_connection(connection, proccesor)
finally:
# Clean up the connection
connection.close()
def main():
init_logging()
args = parse_args()
conf = load_config(args)
global null_tags, classes
null_tags = conf.evaluate.null_tags
classes = conf.labels
visits = get_visits(conf.classify.classify_folder,
conf.evaluate.new_visit_threshold)
if conf.evaluate.show_extended_evaluation:
print_evaluation(visits)
else:
print_summary(visits)
def main():
init_logging()
args = parse_args()
config = Config.load_from_file(args.config_file)
service = ClassifyService(config)
while True:
try:
service.run()
except KeyboardInterrupt:
logging.info("Keyboard interupt closing down")
break
except PermissionError:
logging.error("Error with permissions", exc_info=True)
break
except:
logging.error("Error with service restarting", exc_info=True)
def main():
args = parse_args()
init_logging()
config = Config.load_from_file()
thermal_config = ThermalConfig.load_from_file()
location_config = LocationConfig.load_from_file()
res_x = config.res_x
res_y = config.res_y
print("detecting on " + args.cptv)
motion_detector = MotionDetector(
res_x,
res_y,
thermal_config.motion,
location_config,
thermal_config.recorder,
config.tracking.dynamic_thresh,
None,
)
with open(args.cptv, "rb") as f:
reader = CPTVReader(f)
for i, frame in enumerate(reader):
motion_detector.process_frame(frame)
def main():
init_logging()
args = parse_args()
config = load_config(args.config_file)
db = TrackDatabase(os.path.join(config.tracks_folder, "dataset.hdf5"))
dataset = Dataset(
db, "dataset", config, consecutive_segments=args.consecutive_segments
)
tracks_loaded, total_tracks = dataset.load_tracks(before_date=args.date)
print(
"Loaded {}/{} tracks, found {:.1f}k segments".format(
tracks_loaded, total_tracks, len(dataset.segments) / 1000
)
)
for key, value in dataset.filtered_stats.items():
if value != 0:
print(" {} filtered {}".format(key, value))
print()
show_tracks_breakdown(dataset)
print()
show_segments_breakdown(dataset)
print()
show_important_frames_breakdown(dataset)
print()
show_cameras_breakdown(dataset)
print()
print("Splitting data set into train / validation")
datasets = split_dataset_by_cameras(db, dataset, config, args)
if args.date is None:
args.date = datetime.datetime.now(pytz.utc) - datetime.timedelta(days=7)
test = test_dataset(db, config, args.date)
datasets = (*datasets, test)
print_counts(dataset, *datasets)
print_cameras(*datasets)
pickle.dump(datasets, open(dataset_db_path(config), "wb"))
def preloader(
q,
epoch_queue,
labels,
name,
db,
segments_by_id,
params,
label_mapping,
numpy_meta,
):
global train_queue, jobs
train_queue = q
init_logging()
"""add a segment into buffer"""
logging.info(
" -started async fetcher for %s augment=%s numpyfile %s preload amount %s mem %s",
name,
params.augment,
numpy_meta.filename,
params.maximum_preload,
psutil.Process(os.getpid()).memory_info().rss / 1024**2,
)
epoch = 0
# this does the data pre processing
p_list = []
processes = 1
if name == "train":
processes = 4
preload_amount = max(1, params.maximum_preload)
max_jobs = max(1, 2 * preload_amount // 3)
while True:
with multiprocessing.Pool(
processes,
init_process,
(labels, params, label_mapping),
maxtasksperchild=30,
) as pool:
item = get_with_timeout(epoch_queue, 1,
f"epoch_queue preloader {name}")
if item == "STOP":
logging.info("%s preloader received stop", name)
pool.terminate()
return
try:
epoch, batches = item
count = 0
logging.debug(
"%s preloader got %s batches for epoch %s mem %s",
name,
len(batches),
epoch,
psutil.Process(os.getpid()).memory_info().rss / 1024**2,
)
total = 0
# Once process_batch starts to back up
loaded_up_to = 0
while len(batches) > 0:
logging.debug(
"%s preloader memory %s",
name,
psutil.Process(os.getpid()).memory_info().rss /
1024**2,
)
while jobs > max_jobs:
logging.debug("%s waiting for jobs to complete %s",
name, jobs)
time.sleep(5)
next_load = batches[:preload_amount]
logging.debug(
"%s preloader loading %s - %s ",
name,
loaded_up_to,
loaded_up_to + len(next_load),
)
loaded_up_to = loaded_up_to + len(next_load)
batch_data, track_frames = load_batch_frames(
numpy_meta,
next_load,
segments_by_id,
name,
)
chunk_size = len(batch_data) // processes
data = []
for batch_i, segments in enumerate(batch_data):
start = time.time()
segment_data = [None] * len(segments)
for i, seg in enumerate(segments):
frame_data = get_cached_frames(track_frames, seg)
segment_data[i] = frame_data
data.append((segments, segment_data))
batch_data[batch_i] = None
if len(data) > chunk_size or batch_i == (
len(batch_data) - 1):
pool.map_async(process_batch,
data,
callback=processed_data)
jobs += len(data)
data = []
del batch_data
del track_frames
del batches[:preload_amount]
gc.collect()
logging.debug(
"%s preloader loaded up to %s",
name,
loaded_up_to,
)
total += 1
del batches
gc.collect()
logging.info("%s preloader loaded epoch %s batches", name,
epoch)
pool.close()
pool.join()
except Exception as inst:
logging.error("%s preloader epoch %s error %s",
name,
epoch,
inst,
exc_info=True)
def main():
logging.root.removeHandler(absl.logging._absl_handler)
absl.logging._warn_preinit_stderr = False
parser = argparse.ArgumentParser()
parser.add_argument(
"source",
help=
'a CPTV file to process, or a folder name, or "all" for all files within subdirectories of source folder.',
)
parser.add_argument(
"-p",
"--preview-type",
help=
"Create MP4 previews of this type (can be slow), this overrides the config",
)
parser.add_argument("-v",
"--verbose",
action="count",
help="Display additional information.")
parser.add_argument(
"--start-date",
help=
"Only clips on or after this day will be processed (format YYYY-MM-DD)",
)
parser.add_argument(
"--end-date",
help=
"Only clips on or before this day will be processed (format YYYY-MM-DD)",
)
parser.add_argument("-c",
"--config-file",
help="Path to config file to use")
parser.add_argument(
"--processor-folder",
help=
"When running from thermal-processing use this to specify the folder for both the source cptv and output mp4. With this option the metadata will be sent to stdout.",
)
parser.add_argument("-T",
"--timestamps",
action="store_true",
help="Emit log timestamps")
parser.add_argument(
"-m",
"--model-file",
help="Path to model file to use, will override config model",
)
args = parser.parse_args()
config = Config.load_from_file(args.config_file)
config.validate()
init_logging(args.timestamps)
# parse command line arguments
if args.preview_type:
config.classify.preview = args.preview_type
if args.verbose:
config.classify_tracking.verbose = True
if args.processor_folder:
config.classify.meta_to_stdout = True
config.base_data_folder = args.processor_folder
config.classify.classify_folder = args.processor_folder
config.source_folder = args.processor_folder
model_file = config.classify.model
if args.model_file:
model_file = args.model_file
clip_classifier = ClipClassifier(config, config.classify_tracking,
model_file)
# parse start and end dates
if args.start_date:
clip_classifier.start_date = datetime.strptime(args.start_date,
"%Y-%m-%d")
if args.end_date:
clip_classifier.end_date = datetime.strptime(args.end_date, "%Y-%m-%d")
if config.classify.preview != Previewer.PREVIEW_NONE:
logging.info("Creating previews")
if not config.use_gpu:
logging.info("GPU mode disabled.")
if not os.path.exists(clip_classifier.model_file + ".meta"):
logging.error(
"No model found named '{}'.".format(clip_classifier.model_file +
".meta"))
exit(13)
# just fetch the classifier now so it doesn't impact the benchmarking on the first clip analysed.
_ = clip_classifier.classifier
if os.path.splitext(args.source)[-1].lower() == ".cptv":
source_file = tools.find_file_from_cmd_line(config.source_folder,
args.source)
if source_file is None:
return
clip_classifier.process_file(source_file)
else:
folder = config.source_folder
if args.source != "all":
os.path.join(config.source_folder, folder)
clip_classifier.process_all(folder)
def main():
config, args = parse_params()
init_logging()
if config and args:
load_clips(config, args)
def main():
logging.root.removeHandler(absl.logging._absl_handler)
absl.logging._warn_preinit_stderr = False
parser = argparse.ArgumentParser()
parser.add_argument(
"source",
help=
'a CPTV file to process, or a folder name, or "all" for all files within subdirectories of source folder.',
)
parser.add_argument(
"-p",
"--preview-type",
help=
"Create MP4 previews of this type (can be slow), this overrides the config",
)
parser.add_argument("-v",
"--verbose",
action="count",
help="Display additional information.")
parser.add_argument("-c",
"--config-file",
help="Path to config file to use")
parser.add_argument(
"-o",
"--meta-to-stdout",
action="count",
help="Print metadata to stdout instead of saving to file.",
)
parser.add_argument("-T",
"--timestamps",
action="store_true",
help="Emit log timestamps")
parser.add_argument(
"-m",
"--model-file",
help="Path to model file to use, will override config model",
)
parser.add_argument(
"--reuse-prediction-frames",
action="count",
help="Use supplied prediction frames from metadata.txt",
)
parser.add_argument(
"--cache",
type=str2bool,
nargs="?",
const=True,
default=None,
help=
"Dont keep video frames in memory for classification later, but cache them to disk (Best for large videos, but slower)",
)
args = parser.parse_args()
config = Config.load_from_file(args.config_file)
config.validate()
init_logging(args.timestamps)
# parse command line arguments
if args.preview_type:
config.classify.preview = args.preview_type
if args.verbose:
config.classify_tracking.verbose = True
if args.meta_to_stdout:
config.classify.meta_to_stdout = True
model = None
keras_model = None
if args.model_file:
model = ModelConfig.load({
"id": 1,
"model_file": args.model_file,
"name": args.model_file
})
model.validate()
clip_classifier = ClipClassifier(
config,
model,
)
clip_classifier.process(
args.source,
cache=args.cache,
reuse_frames=args.reuse_prediction_frames,
)
