def test_api_calls():
from vital.config import config
from vital.modules import modules
from sprokit.pipeline import process
from sprokit.pipeline import process_factory
modules.load_known_modules()
proc_type = 'orphan'
c = config.empty_config()
process_factory.create_process(proc_type, '')
process_factory.create_process(proc_type, '', c)
process_factory.types()
process_factory.description(proc_type)
process_factory.Process.property_no_threads
process_factory.Process.property_no_reentrancy
process_factory.Process.property_unsync_input
process_factory.Process.property_unsync_output
process_factory.Process.port_heartbeat
process_factory.Process.config_name
process_factory.Process.config_type
process_factory.Process.type_any
process_factory.Process.type_none
process_factory.Process.type_data_dependent
process_factory.Process.type_flow_dependent
process_factory.Process.flag_output_const
process_factory.Process.flag_input_static
process_factory.Process.flag_input_mutable
process_factory.Process.flag_input_nodep
process_factory.Process.flag_required
def test_api_calls():
from sprokit.pipeline import config
from vital.modules import modules
from sprokit.pipeline import process
from sprokit.pipeline import process_factory
modules.load_known_modules()
proc_type = 'orphan'
c = config.empty_config()
process_factory.create_process(proc_type, '')
process_factory.create_process(proc_type, '', c)
process_factory.types()
process_factory.description(proc_type)
process_factory.Process.property_no_threads
process_factory.Process.property_no_reentrancy
process_factory.Process.property_unsync_input
process_factory.Process.property_unsync_output
process_factory.Process.port_heartbeat
process_factory.Process.config_name
process_factory.Process.config_type
process_factory.Process.type_any
process_factory.Process.type_none
process_factory.Process.type_data_dependent
process_factory.Process.type_flow_dependent
process_factory.Process.flag_output_const
process_factory.Process.flag_input_static
process_factory.Process.flag_input_mutable
process_factory.Process.flag_input_nodep
process_factory.Process.flag_required
def test_register():
from vital.config import config
from vital.modules import modules
from sprokit.pipeline import pipeline
from sprokit.pipeline import scheduler_factory
modules.load_known_modules()
sched_type = 'python_example'
sched_desc = 'simple description'
scheduler_factory.add_scheduler(sched_type, sched_desc,
example_scheduler(True))
if not sched_desc == scheduler_factory.description(sched_type):
test_error("Description was not preserved when registering")
p = pipeline.Pipeline()
try:
s = scheduler_factory.create_scheduler(sched_type, p)
if s is None:
raise Exception()
except:
test_error("Could not create newly registered scheduler type")
def test_detect(self):
modules.load_known_modules()
detector = ImageObjectDetector.create("example_detector")
image = Image()
image_container = ImageContainer(image)
detections = detector.detect(image_container)
nose.tools.ok_(detections is not None, "Unexpected empty detections")
nose.tools.assert_equal(len(detections), 1)
def test_detect(self):
modules.load_known_modules()
detector = ImageObjectDetector.create("example_detector")
image = Image()
image_container = ImageContainer(image)
detections = detector.detect(image_container)
nose.tools.ok_(detections is not None,
"Unexpected empty detections" )
nose.tools.assert_equal(len(detections), 1)
def create_process(type, name, conf):
from vital.modules import modules
from sprokit.pipeline import process_factory
modules.load_known_modules()
p = process_factory.create_process(type, name, conf)
return p
def create_process(type, name, conf):
from vital.modules import modules
from sprokit.pipeline import process_factory
modules.load_known_modules()
p = process_factory.create_process(type, name, conf)
return p
def test_extra_modules():
from vital.modules import modules
from sprokit.pipeline import process_factory
modules.load_known_modules()
types = process_factory.types()
if 'extra_test_python_process' not in types:
test_error("Failed to load extra Python processes")
def test_masking():
from vital.modules import modules
from sprokit.pipeline import process_factory
modules.load_known_modules()
types = process_factory.types()
if 'test_python_process' in types:
test_error("Failed to mask out Python processes")
def test_config(self):
modules.load_known_modules()
detector = ImageObjectDetector.create("example_detector")
# Verify that 6 config values are present in example_detector
nose.tools.assert_equal(len(detector.get_configuration()), 6)
test_cfg = _dummy_detector_cfg()
# Verify that the detector has different configuration before setting to test
nose.tools.assert_not_equal(detector.check_configuration(test_cfg), False)
detector.set_configuration(test_cfg)
# Verify that the config value is being set properly
nose.tools.assert_equal(detector.check_configuration(test_cfg), True)
def test_extra_modules():
from sprokit.pipeline import config
from vital.modules import modules
from sprokit.pipeline import process_factory
modules.load_known_modules()
types = process_factory.types()
if 'extra_test_python_process' not in types:
test_error("Failed to load extra Python processes")
def test_masking():
from sprokit.pipeline import config
from vital.modules import modules
from sprokit.pipeline import process_factory
modules.load_known_modules()
types = process_factory.types()
if 'test_python_process' in types:
test_error("Failed to mask out Python processes")
def test_config(self):
modules.load_known_modules()
detector = ImageObjectDetector.create("example_detector")
# Verify that 6 config values are present in example_detector
nose.tools.assert_equal(len(detector.get_configuration()), 6)
test_cfg = _dummy_detector_cfg()
# Verify that the detector has different configuration before setting to test
nose.tools.assert_not_equal(detector.check_configuration(test_cfg),
False)
detector.set_configuration(test_cfg)
# Verify that the config value is being set properly
nose.tools.assert_equal(detector.check_configuration(test_cfg), True)
def test_simple_pipeline(path):
from sprokit.pipeline import config
from sprokit.pipeline import pipeline
from vital.modules import modules
from sprokit.pipeline_util import bake
from sprokit.pipeline_util import load
blocks = load.load_pipe_file(path)
modules.load_known_modules()
bake.bake_pipe_file(path)
with open(path, 'r') as fin:
bake.bake_pipe(fin)
bake.bake_pipe_blocks(blocks)
bake.extract_configuration(blocks)
def test_simple_pipeline(path):
from vital.config import config
from sprokit.pipeline import pipeline
from vital.modules import modules
from sprokit.pipeline_util import bake
from sprokit.pipeline_util import load
blocks = load.load_pipe_file(path)
modules.load_known_modules()
bake.bake_pipe_file(path)
with open(path, 'r') as fin:
bake.bake_pipe(fin)
bake.bake_pipe_blocks(blocks)
bake.extract_configuration(blocks)
def test_api_calls():
from sprokit.pipeline import config
from vital.modules import modules
from sprokit.pipeline import pipeline
from sprokit.pipeline import scheduler_factory
modules.load_known_modules()
sched_type = 'thread_per_process'
c = config.empty_config()
p = pipeline.Pipeline()
scheduler_factory.create_scheduler(sched_type, p)
scheduler_factory.create_scheduler(sched_type, p, c)
scheduler_factory.types()
scheduler_factory.description(sched_type)
scheduler_factory.default_type
def run_pipeline(sched_type, pipe, conf):
from sprokit.pipeline import config
from vital.modules import modules
from sprokit.pipeline import scheduler_factory
import sys
modules.load_known_modules()
if sched_type in cpp_scheds:
expect_exception("trying to run a python process on a C++ scheduler", RuntimeError,
scheduler_factory.create_scheduler, sched_type, pipe, conf)
else:
s = scheduler_factory.create_scheduler(sched_type, pipe, conf)
s.start()
s.wait()
def run_pipeline(sched_type, pipe, conf):
from vital.config import config
from vital.modules import modules
from sprokit.pipeline import scheduler_factory
import sys
modules.load_known_modules()
if sched_type in cpp_scheds:
expect_exception("trying to run a python process on a C++ scheduler", RuntimeError,
scheduler_factory.create_scheduler, sched_type, pipe, conf)
else:
s = scheduler_factory.create_scheduler(sched_type, pipe, conf)
s.start()
s.wait()
def test_api_calls():
from vital.config import config
from vital.modules import modules
from sprokit.pipeline import pipeline
from sprokit.pipeline import scheduler_factory
modules.load_known_modules()
sched_type = 'thread_per_process'
c = config.empty_config()
p = pipeline.Pipeline()
scheduler_factory.create_scheduler(sched_type, p)
scheduler_factory.create_scheduler(sched_type, p, c)
scheduler_factory.types()
scheduler_factory.description(sched_type)
scheduler_factory.default_type
def test_pythonpath():
from sprokit.pipeline import config
from vital.modules import modules
from sprokit.pipeline import process_factory
from sprokit.pipeline import scheduler_factory
modules.load_known_modules()
types = process_factory.types()
if 'pythonpath_test_process' not in types:
test_error("Failed to load extra Python processes accessible from PYTHONPATH")
types = scheduler_factory.types()
if 'pythonpath_test_scheduler' not in types:
test_error("Failed to load extra Python schedulers accessible from PYTHONPATH")
def test_pythonpath():
from vital.modules import modules
from sprokit.pipeline import process_factory
from sprokit.pipeline import scheduler_factory
modules.load_known_modules()
types = process_factory.types()
if 'pythonpath_test_process' not in types:
test_error(
"Failed to load extra Python processes accessible from PYTHONPATH")
types = scheduler_factory.types()
if 'pythonpath_test_scheduler' not in types:
test_error(
"Failed to load extra Python schedulers accessible from PYTHONPATH"
)
def test_wrapper_api():
from vital.config import config
from vital.modules import modules
from sprokit.pipeline import pipeline
from sprokit.pipeline import process_factory
from sprokit.pipeline import scheduler_factory
sched_type = 'python_example'
sched_desc = 'simple description'
modules.load_known_modules()
scheduler_factory.add_scheduler(sched_type, sched_desc,
example_scheduler(False))
p = pipeline.Pipeline()
proc_type = 'orphan'
proc_name = 'orphan'
proc = process_factory.create_process(proc_type, proc_name)
p.add_process(proc)
def check_scheduler(s):
if s is None:
test_error("Got a 'None' scheduler")
return
s.start()
s.pause()
s.resume()
s.stop()
s.start()
s.wait()
del s
p.reset()
p.setup_pipeline()
s = scheduler_factory.create_scheduler(sched_type, p)
check_scheduler(s)
def test_nested_config(self):
modules.load_known_modules()
detector = ImageObjectDetector.create("SimpleImageObjectDetector")
nested_cfg = config.empty_config()
ImageObjectDetector.get_nested_algo_configuration(
"detector", nested_cfg, detector)
# Verify that test cfg is set to configuration inside detector
# nested configuration uses the name of a detector as an additional configuration
# key thus it is checked against 7 rather than 6
#nose.tools.assert_equal(len(nested_cfg), 7)
#test_cfg = _dummy_detector_cfg()
#test_cfg.set_value("example_detector:type", "example_detector")
#ImageObjectDetector.set_nested_algo_configuration( "example_detector",
# test_cfg,
# detector )
nose.tools.assert_equal(
ImageObjectDetector.check_nested_algo_configuration(
"detector", nested_cfg), True)
def test_wrapper_api():
from sprokit.pipeline import config
from vital.modules import modules
from sprokit.pipeline import pipeline
from sprokit.pipeline import process_factory
from sprokit.pipeline import scheduler_factory
sched_type = 'python_example'
sched_desc = 'simple description'
modules.load_known_modules()
scheduler_factory.add_scheduler(sched_type, sched_desc, example_scheduler(False))
p = pipeline.Pipeline()
proc_type = 'orphan'
proc_name = 'orphan'
proc = process_factory.create_process(proc_type, proc_name)
p.add_process(proc)
def check_scheduler(s):
if s is None:
test_error("Got a 'None' scheduler")
return
s.start()
s.pause()
s.resume()
s.stop()
s.start()
s.wait()
del s
p.reset()
p.setup_pipeline()
s = scheduler_factory.create_scheduler(sched_type, p)
check_scheduler(s)
def test_nested_config(self):
modules.load_known_modules()
detector = ImageObjectDetector.create("SimpleImageObjectDetector")
nested_cfg = config.empty_config()
ImageObjectDetector.get_nested_algo_configuration( "detector",
nested_cfg,
detector )
# Verify that test cfg is set to configuration inside detector
# nested configuration uses the name of a detector as an additional configuration
# key thus it is checked against 7 rather than 6
#nose.tools.assert_equal(len(nested_cfg), 7)
#test_cfg = _dummy_detector_cfg()
#test_cfg.set_value("example_detector:type", "example_detector")
#ImageObjectDetector.set_nested_algo_configuration( "example_detector",
# test_cfg,
# detector )
nose.tools.assert_equal(ImageObjectDetector.check_nested_algo_configuration(
"detector",
nested_cfg), True)
def test_api_calls():
from sprokit.pipeline import config
from sprokit.pipeline import datum
from sprokit.pipeline import edge
from vital.modules import modules
from sprokit.pipeline import process
from sprokit.pipeline import process_factory
from sprokit.pipeline import stamp
e = edge.Edge()
e.makes_dependency()
e.has_data()
e.full_of_data()
e.datum_count()
d = datum.complete()
s = stamp.new_stamp(1)
ed = edge.EdgeDatum(d, s)
e.push_datum(ed)
e.get_datum()
e.push_datum(ed)
e.peek_datum()
e.pop_datum()
modules.load_known_modules()
p = process_factory.create_process('orphan', 'process_name')
e.set_upstream_process(p)
e.set_downstream_process(p)
e.mark_downstream_as_complete()
e.is_downstream_complete()
e.config_dependency
e.config_capacity
def test_api_calls():
from vital.config import config
from sprokit.pipeline import datum
from sprokit.pipeline import edge
from vital.modules import modules
from sprokit.pipeline import process
from sprokit.pipeline import process_factory
from sprokit.pipeline import stamp
e = edge.Edge()
e.makes_dependency()
e.has_data()
e.full_of_data()
e.datum_count()
d = datum.complete()
s = stamp.new_stamp(1)
ed = edge.EdgeDatum(d, s)
e.push_datum(ed)
e.get_datum()
e.push_datum(ed)
e.peek_datum()
e.pop_datum()
modules.load_known_modules()
p = process_factory.create_process('orphan', 'process_name')
e.set_upstream_process(p)
e.set_downstream_process(p)
e.mark_downstream_as_complete()
e.is_downstream_complete()
e.config_dependency
e.config_capacity
def test_simple_pipeline(path):
import os
from sprokit.pipeline import pipeline
from vital.modules import modules
from sprokit.pipeline_util import bake
from sprokit.pipeline_util import export_
modules.load_known_modules()
p = bake.bake_pipe_file(path)
r, w = os.pipe()
name = 'graph'
export_.export_dot(w, p, name)
p.setup_pipeline()
export_.export_dot(w, p, name)
os.close(r)
os.close(w)
def test_register():
from sprokit.pipeline import config
from vital.modules import modules
from sprokit.pipeline import pipeline
from sprokit.pipeline import scheduler_factory
modules.load_known_modules()
sched_type = 'python_example'
sched_desc = 'simple description'
scheduler_factory.add_scheduler(sched_type, sched_desc, example_scheduler(True))
if not sched_desc == scheduler_factory.description(sched_type):
test_error("Description was not preserved when registering")
p = pipeline.Pipeline()
try:
s = scheduler_factory.create_scheduler(sched_type, p)
if s is None:
raise Exception()
except:
test_error("Could not create newly registered scheduler type")
def test_cluster_multiplier(path):
from sprokit.pipeline import config
from sprokit.pipeline import pipeline
from vital.modules import modules
from sprokit.pipeline_util import bake
from sprokit.pipeline_util import load
blocks = load.load_cluster_file(path)
modules.load_known_modules()
bake.bake_cluster_file(path)
with open(path, 'r') as fin:
bake.bake_cluster(fin)
info = bake.bake_cluster_blocks(blocks)
conf = config.empty_config()
info.type()
info.description()
info.create()
info.create(conf)
bake.register_cluster(info)
def test_cluster_multiplier(path):
from vital.config import config
from sprokit.pipeline import pipeline
from vital.modules import modules
from sprokit.pipeline_util import bake
from sprokit.pipeline_util import load
blocks = load.load_cluster_file(path)
modules.load_known_modules()
bake.bake_cluster_file(path)
with open(path, 'r') as fin:
bake.bake_cluster(fin)
info = bake.bake_cluster_blocks(blocks)
conf = config.empty_config()
info.type()
info.description()
info.create()
info.create(conf)
bake.register_cluster(info)
def test_load():
from vital.modules import modules
modules.load_known_modules()
def test_load():
from vital.modules import modules
modules.load_known_modules()
def test_registered_names(self):
modules.load_known_modules()
registered_detectors = ImageObjectDetector.registered_names()
print("All registered image object detectors")
for detectors in registered_detectors:
print(" " + detectors)
def test_bad_detect(self):
modules.load_known_modules()
detector = ImageObjectDetector.create("example_detector")
detector.detect("Image")
def test_create(self):
modules.load_known_modules()
registered_detector = ImageObjectDetector.registered_names()[0]
nose.tools.ok_(registered_detector is not None,
"No instance returned from the factory method")
def test_api_calls():
from vital.config import config
from sprokit.pipeline import edge
from vital.modules import modules
from sprokit.pipeline import pipeline
from sprokit.pipeline import process
from sprokit.pipeline import process_cluster
from sprokit.pipeline import process_factory
p = pipeline.Pipeline()
proc_type1 = 'numbers'
proc_type2 = 'print_number'
proc_type3 = 'orphan_cluster'
proc_name1 = 'src'
proc_name2 = 'snk'
proc_name3 = 'orp'
port_name1 = 'number'
port_name2 = 'number'
modules.load_known_modules()
proc1 = process_factory.create_process(proc_type1, proc_name1)
conf_name = 'output'
c = config.empty_config()
c.set_value(conf_name, 'test-python-pipeline-api_calls-print_number.txt')
proc2 = process_factory.create_process(proc_type2, proc_name2, c)
proc3 = process_factory.create_process(proc_type3, proc_name3)
p.add_process(proc1)
p.add_process(proc2)
p.add_process(proc3)
p.connect(proc_name1, port_name1, proc_name2, port_name2)
p.process_names()
p.process_by_name(proc_name1)
p.cluster_names()
p.cluster_by_name(proc_name3)
p.connections_from_addr(proc_name1, port_name1)
p.connection_to_addr(proc_name2, port_name2)
p.disconnect(proc_name1, port_name1, proc_name2, port_name2)
p.remove_process(proc_name1)
p.remove_process(proc_name3)
# Restore the pipeline so that setup_pipeline works.
p.add_process(proc1)
p.connect(proc_name1, port_name1, proc_name2, port_name2)
p.setup_pipeline()
p.upstream_for_process(proc_name2)
p.upstream_for_port(proc_name2, port_name2)
p.downstream_for_process(proc_name1)
p.downstream_for_port(proc_name1, port_name1)
p.sender_for_port(proc_name2, port_name2)
p.receivers_for_port(proc_name1, port_name1)
p.edge_for_connection(proc_name1, port_name1, proc_name2, port_name2)
p.input_edges_for_process(proc_name2)
p.input_edge_for_port(proc_name2, port_name2)
p.output_edges_for_process(proc_name1)
p.output_edges_for_port(proc_name1, port_name1)
p.is_setup()
p.setup_successful()
c = config.empty_config()
p.reconfigure(c)
p.reset()
def set_configuration(self, cfg_in):
cfg = self.get_configuration()
cfg.merge_config(cfg_in)
# Read configs from file
self._identifier = str(cfg.get_value("identifier"))
self._mode = str(cfg.get_value("mode"))
self._arch = str(cfg.get_value("arch"))
self._seed_model = str(cfg.get_value("seed_model"))
self._train_directory = str(cfg.get_value("train_directory"))
self._output_directory = str(cfg.get_value("output_directory"))
self._output_prefix = str(cfg.get_value("output_prefix"))
self._pipeline_template = str(cfg.get_value("pipeline_template"))
self._gpu_count = int(cfg.get_value("gpu_count"))
self._gt_frames_only = strtobool(cfg.get_value("gt_frames_only"))
self._augmentation = str(cfg.get_value("augmentation"))
self._chip_width = str(cfg.get_value("chip_width"))
self._chip_overlap = str(cfg.get_value("chip_overlap"))
self._max_epochs = str(cfg.get_value("max_epochs"))
self._batch_size = str(cfg.get_value("batch_size"))
self._learning_rate = str(cfg.get_value("learning_rate"))
self._timeout = str(cfg.get_value("timeout"))
self._backbone = str(cfg.get_value("backbone"))
self._pipeline_template = str(cfg.get_value("pipeline_template"))
self._max_scale_wrt_chip = float(cfg.get_value("max_scale_wrt_chip"))
self._no_format = strtobool(cfg.get_value("no_format"))
self._aux_image_labels = str(cfg.get_value("aux_image_labels"))
self._aux_image_extensions = str(cfg.get_value("aux_image_extensions"))
# Check GPU-related variables
gpu_memory_available = 0
gpu_param_adj = 1
if torch.cuda.is_available():
if self._gpu_count < 0:
self._gpu_count = torch.cuda.device_count()
gpu_param_adj = self._gpu_count
for i in range(self._gpu_count):
single_gpu_mem = torch.cuda.get_device_properties(
i).total_memory
if gpu_memory_available == 0:
gpu_memory_available = single_gpu_mem
else:
gpu_memory_available = min(gpu_memory_available,
single_gpu_mem)
if self._mode == "detector":
if self._batch_size == "auto":
if len(self._aux_image_labels) > 0:
if gpu_memory_available >= 22e9:
self._batch_size = str(2 * gpu_param_adj)
else:
self._batch_size = str(1 * gpu_param_adj)
elif gpu_memory_available > 9.5e9:
self._batch_size = str(4 * gpu_param_adj)
elif gpu_memory_available >= 7.5e9:
self._batch_size = str(3 * gpu_param_adj)
elif gpu_memory_available >= 4.5e9:
self._batch_size = str(2 * gpu_param_adj)
else:
self._batch_size = str(1 * gpu_param_adj)
if self._learning_rate == "auto":
self._learning_rate = str(1e-3)
elif self._mode == "frame_classifier":
if self._batch_size == "auto":
if gpu_memory_available > 9.5e9:
self._batch_size = str(64 * gpu_param_adj)
elif gpu_memory_available >= 7.5e9:
self._batch_size = str(32 * gpu_param_adj)
elif gpu_memory_available >= 4.5e9:
self._batch_size = str(16 * gpu_param_adj)
else:
self._batch_size = str(8 * gpu_param_adj)
if self._learning_rate == "auto":
self._learning_rate = str(5e-3)
else:
print("Invalid mode string " + self._mode)
return False
# Make required directories and file streams
if self._train_directory is not None:
if not os.path.exists(self._train_directory):
os.mkdir(self._train_directory)
self._training_file = os.path.join(self._train_directory,
self._tmp_training_file)
self._validation_file = os.path.join(self._train_directory,
self._tmp_validation_file)
else:
self._training_file = self._tmp_training_file
self._validation_file = self._tmp_validation_file
if self._output_directory is not None:
if not os.path.exists(self._output_directory):
os.mkdir(self._output_directory)
from vital.modules.modules import load_known_modules
load_known_modules()
if not self._no_format:
self._training_writer = \
DetectedObjectSetOutput.create( "coco" )
self._validation_writer = \
DetectedObjectSetOutput.create( "coco" )
writer_conf = self._training_writer.get_configuration()
writer_conf.set_value("aux_image_labels", self._aux_image_labels)
writer_conf.set_value("aux_image_extensions",
self._aux_image_extensions)
self._training_writer.set_configuration(writer_conf)
writer_conf = self._validation_writer.get_configuration()
writer_conf.set_value("aux_image_labels", self._aux_image_labels)
writer_conf.set_value("aux_image_extensions",
self._aux_image_extensions)
self._validation_writer.set_configuration(writer_conf)
self._training_writer.open(self._training_file)
self._validation_writer.open(self._validation_file)
# Set default architecture if unset
if not self._arch:
if self._mode == "frame_classifier":
self._arch = "resnet50"
else:
self._arch = "cascade"
# Initialize persistent variables
self._training_data = []
self._validation_data = []
self._sample_count = 0
return True
def set_configuration(self, cfg_in):
cfg = self.get_configuration()
cfg.merge_config(cfg_in)
# Read configs from file
self._identifier = str(cfg.get_value("identifier"))
self._mode = str(cfg.get_value("mode"))
self._seed_model = str(cfg.get_value("seed_model"))
self._train_directory = str(cfg.get_value("train_directory"))
self._output_directory = str(cfg.get_value("output_directory"))
self._output_prefix = str(cfg.get_value("output_prefix"))
self._pipeline_template = str(cfg.get_value("pipeline_template"))
self._gpu_count = int(cfg.get_value("gpu_count"))
self._gt_frames_only = strtobool(cfg.get_value("gt_frames_only"))
self._augmentation = str(cfg.get_value("augmentation"))
self._chip_width = str(cfg.get_value("chip_width"))
self._chip_overlap = str(cfg.get_value("chip_overlap"))
self._max_epochs = str(cfg.get_value("max_epochs"))
self._batch_size = str(cfg.get_value("batch_size"))
self._timeout = str(cfg.get_value("timeout"))
self._backbone = str(cfg.get_value("backbone"))
self._pipeline_template = str(cfg.get_value("pipeline_template"))
self._max_scale_wrt_chip = float(cfg.get_value("max_scale_wrt_chip"))
self._no_format = strtobool(cfg.get_value("no_format"))
# Check GPU-related variables
gpu_memory_available = 0
if torch.cuda.is_available():
if self._gpu_count < 0:
self._gpu_count = torch.cuda.device_count()
for i in range(self._gpu_count):
single_gpu_mem = torch.cuda.get_device_properties(
i).total_memory
if gpu_memory_available == 0:
gpu_memory_available = single_gpu_mem
else:
gpu_memory_available = min(gpu_memory_available,
single_gpu_mem)
if self._mode == "detector":
if self._batch_size == "auto":
if gpu_memory_available > 9e9:
self._batch_size = "4"
elif gpu_memory_available >= 7e9:
self._batch_size = "3"
else:
self._batch_size = "2"
elif self._mode == "frame_classifier":
if self._batch_size == "auto":
if gpu_memory_available > 9e9:
self._batch_size = "64"
elif gpu_memory_available >= 7e9:
self._batch_size = "32"
else:
self._batch_size = "16"
else:
print("Invalid mode string " + self._mode)
return False
# Make required directories and file streams
if self._train_directory is not None:
if not os.path.exists(self._train_directory):
os.mkdir(self._train_directory)
self._training_file = os.path.join(self._train_directory,
self._tmp_training_file)
self._validation_file = os.path.join(self._train_directory,
self._tmp_validation_file)
else:
self._training_file = self._tmp_training_file
self._validation_file = self._tmp_validation_file
if self._output_directory is not None:
if not os.path.exists(self._output_directory):
os.mkdir(self._output_directory)
from vital.modules.modules import load_known_modules
load_known_modules()
if not self._no_format:
self._training_writer = \
DetectedObjectSetOutput.create( "coco" )
self._validation_writer = \
DetectedObjectSetOutput.create( "coco" )
self._training_writer.open(self._training_file)
self._validation_writer.open(self._validation_file)
# Initialize persistent variables
self._training_data = []
self._validation_data = []
self._sample_count = 0
return True
def test_api_calls():
from sprokit.pipeline import config
from sprokit.pipeline import edge
from vital.modules import modules
from sprokit.pipeline import pipeline
from sprokit.pipeline import process
from sprokit.pipeline import process_cluster
from sprokit.pipeline import process_factory
p = pipeline.Pipeline()
proc_type1 = 'numbers'
proc_type2 = 'print_number'
proc_type3 = 'orphan_cluster'
proc_name1 = 'src'
proc_name2 = 'snk'
proc_name3 = 'orp'
port_name1 = 'number'
port_name2 = 'number'
modules.load_known_modules()
proc1 = process_factory.create_process(proc_type1, proc_name1)
conf_name = 'output'
c = config.empty_config()
c.set_value(conf_name, 'test-python-pipeline-api_calls-print_number.txt')
proc2 = process_factory.create_process(proc_type2, proc_name2, c)
proc3 = process_factory.create_process(proc_type3, proc_name3)
p.add_process(proc1)
p.add_process(proc2)
p.add_process(proc3)
p.connect(proc_name1, port_name1,
proc_name2, port_name2)
p.process_names()
p.process_by_name(proc_name1)
p.cluster_names()
p.cluster_by_name(proc_name3)
p.connections_from_addr(proc_name1, port_name1)
p.connection_to_addr(proc_name2, port_name2)
p.disconnect(proc_name1, port_name1,
proc_name2, port_name2)
p.remove_process(proc_name1)
p.remove_process(proc_name3)
# Restore the pipeline so that setup_pipeline works.
p.add_process(proc1)
p.connect(proc_name1, port_name1,
proc_name2, port_name2)
p.setup_pipeline()
p.upstream_for_process(proc_name2)
p.upstream_for_port(proc_name2, port_name2)
p.downstream_for_process(proc_name1)
p.downstream_for_port(proc_name1, port_name1)
p.sender_for_port(proc_name2, port_name2)
p.receivers_for_port(proc_name1, port_name1)
p.edge_for_connection(proc_name1, port_name1,
proc_name2, port_name2)
p.input_edges_for_process(proc_name2)
p.input_edge_for_port(proc_name2, port_name2)
p.output_edges_for_process(proc_name1)
p.output_edges_for_port(proc_name1, port_name1)
p.is_setup()
p.setup_successful()
c = config.empty_config()
p.reconfigure(c)
p.reset()
def test_bad_detect(self):
modules.load_known_modules()
detector = ImageObjectDetector.create("example_detector")
detector.detect("Image")
def test_create(self):
modules.load_known_modules()
registered_detector = ImageObjectDetector.registered_names()[0]
nose.tools.ok_(registered_detector is not None,
"No instance returned from the factory method")
def test_registered_names(self):
modules.load_known_modules()
registered_detectors = ImageObjectDetector.registered_names()
print("All registered image object detectors")
for detectors in registered_detectors:
print(" " + detectors)
