def run_demo(movie_file):
"""
In this demo, an image is being predicted by a set of predictive encoders looking
at different aspects of the input. There is a set of future encoders digesting two 8x8 frames to predict
the next one, another set taking 8 4x4 frames to predict 4 subsequent frames and another set taking
32 2x2 frames to predict 16 subsequent frames. Additionally there is one unit taking the whole image as
8x8 block whose internal representations are shared as context with all the other units.
The system has feedback connections, more temporal areas feed back to more spatial areas. Also cross-like
neighbourhood of lateral projections is instantiated.
The simulation is synchronous, runs in single stage.
"""
filename = "demo03_state.p.gz"
if movie_file != "":
cam = cv2.VideoCapture(movie_file)
else:
cam = cv2.VideoCapture(-1)
if not cam.isOpened():
logging.error("Either cannot read the input file or no camera found!")
exit(1)
if os.path.isfile(filename):
state_dict = CoreUtils.load_model(filename)
else:
state_dict = generate_dict()
manager = Manager(state_dict, 1000000, cam=cam)
CoreUtils.run_model(state_dict, manager)
CoreUtils.save_model(state_dict, filename)
def run_demo(movie_file):
"""
In this demo, an image is being predicted by a set of predictive encoders looking
at different aspects of the input. There is a set of future encoders digesting two 8x8 frames to predict
the next one, another set taking 8 4x4 frames to predict 4 subsequent frames and another set taking
32 2x2 frames to predict 16 subsequent frames. Additionally there is one unit taking the whole image as
8x8 block whose internal representations are shared as context with all the other units.
The system has feedback connections, more temporal areas feed back to more spatial areas. Also cross-like
neighbourhood of lateral projections is instantiated.
The simulation is synchronous, runs in single stage.
"""
filename = "demo03_state.p.gz"
if movie_file != "":
cam = cv2.VideoCapture(movie_file)
else:
cam = cv2.VideoCapture(-1)
if not cam.isOpened():
logging.error("Either cannot read the input file or no camera found!")
exit(1)
if os.path.isfile(filename):
state_dict = CoreUtils.load_model(filename)
else:
state_dict = generate_dict()
manager = Manager(state_dict, 1000000, cam=cam)
CoreUtils.run_model(state_dict, manager)
CoreUtils.save_model(state_dict, filename)
def run_demo():
"""
In this simple demo the crticial temperature Ising model is run synchronously by a set of units on a large 1000x1000
domain. To make things fast the worker code if written in cython.
:return:
"""
filename = "demo01_state.p.gz"
if os.path.isfile(filename):
sate_dict = CoreUtils.load_model(filename)
else:
sate_dict = generate_dict()
manager = Manager(sate_dict, 1000000)
CoreUtils.run_model(sate_dict, manager)
CoreUtils.save_model(sate_dict, filename)
def run_demo():
"""
In this simple demo the crticial temperature Ising model is run synchronously by a set of units on a large 1000x1000
domain. To make things fast the worker code if written in cython.
:return:
"""
filename = "demo01_state.p.gz"
if os.path.isfile(filename):
sate_dict = CoreUtils.load_model(filename)
else:
sate_dict = generate_dict()
manager = Manager(sate_dict, 1000000)
CoreUtils.run_model(sate_dict, manager)
CoreUtils.save_model(sate_dict, filename)
def run_demo(movie_file):
"""
In this demo a simple future/predictive encoder is being instantiated to predict a camera image
based on two previous frames.
"""
filename = "demo02_state.p.gz"
if movie_file != "":
cam = cv2.VideoCapture(movie_file)
else:
cam = cv2.VideoCapture(-1)
if not cam.isOpened():
logging.error("Either cannot read the input file or no camera found!")
exit(1)
if os.path.isfile(filename):
dict = CoreUtils.load_model(filename)
else:
dict = generate_dict()
manager = Manager(dict, 1000, cam=cam)
CoreUtils.run_model(dict, manager)
CoreUtils.save_model(dict, filename)
def run_demo(movie_file):
"""
In this demo a simple future/predictive encoder is being instantiated to predict a camera image
based on two previous frames.
"""
filename = "demo02_state.p.gz"
if movie_file != "":
cam = cv2.VideoCapture(movie_file)
else:
cam = cv2.VideoCapture(-1)
if not cam.isOpened():
logging.error("Either cannot read the input file or no camera found!")
exit(1)
if os.path.isfile(filename):
dict = CoreUtils.load_model(filename)
else:
dict = generate_dict()
manager = Manager(dict, 1000, cam=cam)
CoreUtils.run_model(dict, manager)
CoreUtils.save_model(dict, filename)
def run_demo(movie_file):
"""
In this demo a future/predictive encoder is being instantiated to predict a camera image
based on two previous frames. The encoder predicts the signal and its own error on that signal, that is
additional set of units are trying to predict the magnitude of error between the prediction and the signal.
In addition the hidden layer activations from the previous step of execution are used as the context block.
Second order error is calculated as the error of the error prediction. Also, the learning rate of the primary signal
is modulated by the magnitude of the second order error.
"""
if movie_file != "":
cam = cv2.VideoCapture(movie_file)
else:
cam = cv2.VideoCapture(-1)
if not cam.isOpened():
logging.error("Either cannot read the input file or no camera found!")
exit(1)
filename = "demo04_state.p.gz"
if os.path.isfile(filename):
state_dict = CoreUtils.load_model(filename)
else:
state_dict = generate_dict()
manager = Manager(state_dict, 1000000, cam=cam)
CoreUtils.run_model(state_dict, manager)
CoreUtils.save_model(state_dict, filename)
def run_demo(movie_file):
"""
In this demo a future/predictive encoder is being instantiated to predict a camera image
based on two previous frames. The encoder predicts the signal and its own error on that signal, that is
additional set of units are trying to predict the magnitude of error between the prediction and the signal.
In addition the hidden layer activations from the previous step of execution are used as the context block.
Second order error is calculated as the error of the error prediction. Also, the learning rate of the primary signal
is modulated by the magnitude of the second order error.
"""
if movie_file != "":
cam = cv2.VideoCapture(movie_file)
else:
cam = cv2.VideoCapture(-1)
if not cam.isOpened():
logging.error("Either cannot read the input file or no camera found!")
exit(1)
filename = "demo04_state.p.gz"
if os.path.isfile(filename):
state_dict = CoreUtils.load_model(filename)
else:
state_dict = generate_dict()
manager = Manager(state_dict, 1000000, cam=cam)
CoreUtils.run_model(state_dict, manager)
CoreUtils.save_model(state_dict, filename)
def run_model(evaluate=False,
filename="",
cores="",
name="",
description="",
remote="",
display=False,
dataset="",
meta={},
options_given={},
storage=None,
port="9000",
checkpoint=True,
upgrade_only=False):
"""
In this demo a future/predictive encoder is being instantiated to predict a camera image
based on two previous frames. The system is built into a three layer hierarchy in which each next
layer is predicting the hidden activations of the lower one.
In addition the errors from each later are being backpropagated down to the previous layer.
In addition to that, errors generated at context blocka are also being backpropagated to the originating
unit. Consequently the error and signals flows in both directions through the entire system.
"""
if options_given == {} and filename == "" and remote == "":
logging.error("No options were given, don't know what to run! Try running with -h option.")
exit()
options = PVM_options.parse_options(options_given)
if remote != "":
filename = storage.get(remote)
logging.info("Loaded a remote simulation dict %s" % remote)
logging.info("Following options were given: %s" % json.dumps(options, sort_keys=True, indent=4))
if os.path.isfile(filename):
simulation_dict = CoreUtils.load_model(filename)
if "options" in simulation_dict:
options = PVM_options.parse_options(options_given, options_in_the_dict=simulation_dict['options'])
else:
options = PVM_options.parse_options(options_given)
logging.info("Loaded the dictionary")
if cores is not "":
simulation_dict['num_proc'] = int(cores)
else:
simulation_dict['num_proc'] = min(2*mp.cpu_count()/3, simulation_dict["stage0_size"]/2)
PVM_Create.upgrade(simulation_dict)
PVM_Create.upgrade_dictionary_to_ver1_0(simulation_dict)
logging.info("Running on %d cpu's" % simulation_dict['num_proc'])
else:
options = PVM_options.parse_options(options)
simulation_dict = PVM_Create.generate_dict_options(name=name,
description=description,
options=options
)
if cores is not "":
simulation_dict['num_proc'] = int(cores)
else:
if options["model_type"] != "tiny":
simulation_dict['num_proc'] = 2*mp.cpu_count()/3
else:
simulation_dict['num_proc'] = 1
logging.info("Generated the dictionary")
logging.info("Full set of options: %s" % json.dumps(options, sort_keys=True, indent=4))
if options["new_name"] != "":
simulation_dict['name'] = options["new_name"]
options["new_name"] = ""
if "disable_lateral" in options.keys() and options["disable_lateral"] == "1":
simulation_dict["disable_lateral"] = True
if "disable_feedback" in options.keys() and options["disable_feedback"] == "1":
simulation_dict["disable_feedback"] = True
if dataset == "":
dataset = options["dataset"]
else:
options["dataset"] = dataset
PVM_set = PVM_datasets.PVMDataset(dataset, storage=storage)
PVM_Create.apply_options(simulation_dict, options)
if upgrade_only:
CoreUtils.save_model(simulation_dict, filename)
to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash'])
from_path = filename
storage.put(from_path=from_path, to_folder=to_folder, overwrite=True)
return
if options['supervised'] == "1":
logging.info("Running in the supervised mode")
for (i, k) in enumerate(simulation_dict['learning_rates']):
k[0] = 0
logging.info("Setting learning rate %d to zero")
simulation_dict['readout_learning_rate'][0] = float(options["supervised_rate"])
logging.info("Setting additional_learning_rate to %f" % simulation_dict['readout_learning_rate'][0])
if not evaluate:
status_file = "/tmp/%s_%s_%s" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash'])
f = open(status_file, "w")
branch = subprocess.Popen('git rev-parse --abbrev-ref HEAD', shell=True, stdout=subprocess.PIPE).stdout.read()
f.write("BRANCH=%s\n" % cleanup(branch))
f.write("TIMESTAMP=%s\n" % simulation_dict['timestamp'])
f.write("NAME=%s\n" % simulation_dict['name'])
f.write("HASH=%s\n" % simulation_dict['hash'])
f.write("DATASET=%s\n" % dataset)
f.write("OPTIONS=%s\n" % json.dumps(options))
f.close()
remove_artifact_files = True
if meta == {}:
logging.info("Not running on an Amazon EC2 instance, apparently")
logging.info("Not running on an Amazon EC2 instance, apparently: So, not automatically removing downloaded artifact files.")
remove_artifact_files = False
elif options['supervised'] != '1':
host = meta['public-ipv4']
logging.info("Running on amazon instance %s. Adding active job" % host)
storage.put(from_path=status_file, to_folder='DARPA/active_jobs/', overwrite=True)
# Train
signal = PVM_SignalProvider.SimpleSignalProvider(files=PVM_set.training,
storage=storage,
frame_resolution=(simulation_dict['input_array'].shape[1], simulation_dict['input_array'].shape[0]),
heatmap_resolution=simulation_dict['readout_arrays'][0].shape[:2][::-1],
channel="default",
remove_files=remove_artifact_files,
reverse=(int(options['reverse']) > 0))
manager = Manager(simulation_dict,
int(options['steps']),
signal_provider=signal,
record=False,
video_recorder=PVM_display_helper.VideoRecorder(rec_filename="PVM_recording.avi"),
do_display=display,
checkpoint=checkpoint,
checkpoint_storage=storage,
dataset_name=dataset)
CoreUtils.run_model(simulation_dict, manager, port=int(port))
if filename != "" and options['supervised'] != "1":
CoreUtils.save_model(simulation_dict, filename)
to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash'])
from_path = filename
storage.put(from_path=from_path, to_folder=to_folder, overwrite=True)
if remove_artifact_files:
os.remove(from_path)
elif options['supervised'] == "1":
CoreUtils.save_model(simulation_dict, "PVM_state_supervised_%s_%d_%d_%f.p.gz" % (dataset, simulation_dict['N'][0], int(options['steps']), float(options['supervised_rate'])))
to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash'])
from_path = "./PVM_state_supervised_%s_%d_%d_%f.p.gz" % (dataset, simulation_dict['N'][0], int(options['steps']), float(options['supervised_rate']))
storage.put(from_path=from_path, to_folder=to_folder, overwrite=True)
if remove_artifact_files:
os.remove(from_path)
else:
CoreUtils.save_model(simulation_dict, "PVM_state_final.p.gz")
to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash'])
from_path = "./PVM_state_final.p.gz"
storage.put(from_path=from_path, to_folder=to_folder, overwrite=True)
if remove_artifact_files:
os.remove(from_path)
else:
print "Evaluating the system"
logging.info("Evaluating the system")
to_folder = "PVM_models/%s_%s_%s/eval_%09d/" % (simulation_dict['timestamp'], simulation_dict['name'], simulation_dict['hash'], simulation_dict['N'])
# Evaluate
signal = PVM_SignalProvider.SimpleSignalProvider(files=PVM_set.testing,
storage=storage,
frame_resolution=(simulation_dict['input_array'].shape[1], simulation_dict['input_array'].shape[0]),
heatmap_resolution=simulation_dict['readout_array_float00'].shape[:2][::-1],
channel="default",
reverse=(int(options['reverse']) > 0))
name = "PVM_train_eval_%s_%09d_test_combined.avi" % (simulation_dict['hash'], simulation_dict['N'])
manager = Manager(simulation_dict,
steps_to_run=-1,
signal_provider=signal,
record=True,
video_recorder=PVM_display_helper.VideoRecorder(rec_filename=name),
do_display=display,
evaluate=True,
collect_error=True)
manager.freeze_learning()
CoreUtils.run_model(simulation_dict, manager, port=int(port))
from_path = name
storage.put(from_path=from_path, to_folder=to_folder, overwrite=True)
os.remove(name)
logging.info("Finished on test files")
# Individual files
for (i, test) in enumerate(PVM_set.all):
print "Running on %s" % test
logging.info("Running on %s" % test)
name = "PVM_eval_%s_%09d_%01d_%s.avi" % (simulation_dict['hash'], simulation_dict['N'], i, test[1])
signal = PVM_SignalProvider.SimpleSignalProvider(files=[test],
storage=storage,
frame_resolution=(simulation_dict['input_array'].shape[1], simulation_dict['input_array'].shape[0]),
heatmap_resolution=simulation_dict['readout_array_float00'].shape[:2][::-1],
channel="default")
manager = Manager(simulation_dict,
steps_to_run=-1,
signal_provider=signal,
record=True,
video_recorder=PVM_display_helper.VideoRecorder(rec_filename=name),
do_display=display,
evaluate=True)
CoreUtils.run_model(simulation_dict, manager, port=int(port))
from_path = name
storage.put(from_path=from_path, to_folder=to_folder, overwrite=True)
os.remove(name)
logging.info("Finished on %s" % test)
def run_model(evaluate=False,
filename="",
cores="",
name="",
description="",
remote="",
display=False,
dataset="",
meta={},
options_given={},
storage=None,
port="9000",
checkpoint=True,
upgrade_only=False):
"""
In this demo a future/predictive encoder is being instantiated to predict a camera image
based on two previous frames. The system is built into a three layer hierarchy in which each next
layer is predicting the hidden activations of the lower one.
In addition the errors from each later are being backpropagated down to the previous layer.
In addition to that, errors generated at context blocka are also being backpropagated to the originating
unit. Consequently the error and signals flows in both directions through the entire system.
"""
if options_given == {} and filename == "" and remote == "":
logging.error(
"No options were given, don't know what to run! Try running with -h option."
)
exit()
options = PVM_options.parse_options(options_given)
if remote != "":
filename = storage.get(remote)
logging.info("Loaded a remote simulation dict %s" % remote)
logging.info("Following options were given: %s" %
json.dumps(options, sort_keys=True, indent=4))
if os.path.isfile(filename):
simulation_dict = CoreUtils.load_model(filename)
if "options" in simulation_dict:
options = PVM_options.parse_options(
options_given, options_in_the_dict=simulation_dict['options'])
else:
options = PVM_options.parse_options(options_given)
logging.info("Loaded the dictionary")
if cores is not "":
simulation_dict['num_proc'] = int(cores)
else:
simulation_dict['num_proc'] = min(
2 * mp.cpu_count() / 3, simulation_dict["stage0_size"] / 2)
PVM_Create.upgrade(simulation_dict)
PVM_Create.upgrade_dictionary_to_ver1_0(simulation_dict)
logging.info("Running on %d cpu's" % simulation_dict['num_proc'])
else:
options = PVM_options.parse_options(options)
simulation_dict = PVM_Create.generate_dict_options(
name=name, description=description, options=options)
if cores is not "":
simulation_dict['num_proc'] = int(cores)
else:
if options["model_type"] != "tiny":
simulation_dict['num_proc'] = 2 * mp.cpu_count() / 3
else:
simulation_dict['num_proc'] = 1
logging.info("Generated the dictionary")
logging.info("Full set of options: %s" %
json.dumps(options, sort_keys=True, indent=4))
if options["new_name"] != "":
simulation_dict['name'] = options["new_name"]
options["new_name"] = ""
if "disable_lateral" in options.keys(
) and options["disable_lateral"] == "1":
simulation_dict["disable_lateral"] = True
if "disable_feedback" in options.keys(
) and options["disable_feedback"] == "1":
simulation_dict["disable_feedback"] = True
if dataset == "":
dataset = options["dataset"]
else:
options["dataset"] = dataset
PVM_set = PVM_datasets.PVMDataset(dataset, storage=storage)
PVM_Create.apply_options(simulation_dict, options)
if upgrade_only:
CoreUtils.save_model(simulation_dict, filename)
to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'],
simulation_dict['name'],
simulation_dict['hash'])
from_path = filename
storage.put(from_path=from_path, to_folder=to_folder, overwrite=True)
return
if options['supervised'] == "1":
logging.info("Running in the supervised mode")
for (i, k) in enumerate(simulation_dict['learning_rates']):
k[0] = 0
logging.info("Setting learning rate %d to zero")
simulation_dict['readout_learning_rate'][0] = float(
options["supervised_rate"])
logging.info("Setting additional_learning_rate to %f" %
simulation_dict['readout_learning_rate'][0])
if not evaluate:
status_file = "/tmp/%s_%s_%s" % (simulation_dict['timestamp'],
simulation_dict['name'],
simulation_dict['hash'])
f = open(status_file, "w")
branch = subprocess.Popen('git rev-parse --abbrev-ref HEAD',
shell=True,
stdout=subprocess.PIPE).stdout.read()
f.write("BRANCH=%s\n" % cleanup(branch))
f.write("TIMESTAMP=%s\n" % simulation_dict['timestamp'])
f.write("NAME=%s\n" % simulation_dict['name'])
f.write("HASH=%s\n" % simulation_dict['hash'])
f.write("DATASET=%s\n" % dataset)
f.write("OPTIONS=%s\n" % json.dumps(options))
f.close()
remove_artifact_files = True
if meta == {}:
logging.info("Not running on an Amazon EC2 instance, apparently")
logging.info(
"Not running on an Amazon EC2 instance, apparently: So, not automatically removing downloaded artifact files."
)
remove_artifact_files = False
elif options['supervised'] != '1':
host = meta['public-ipv4']
logging.info("Running on amazon instance %s. Adding active job" %
host)
storage.put(from_path=status_file,
to_folder='DARPA/active_jobs/',
overwrite=True)
# Train
signal = PVM_SignalProvider.SimpleSignalProvider(
files=PVM_set.training,
storage=storage,
frame_resolution=(simulation_dict['input_array'].shape[1],
simulation_dict['input_array'].shape[0]),
heatmap_resolution=simulation_dict['readout_arrays'][0].shape[:2]
[::-1],
channel="default",
remove_files=remove_artifact_files,
reverse=(int(options['reverse']) > 0))
manager = Manager(simulation_dict,
int(options['steps']),
signal_provider=signal,
record=False,
video_recorder=PVM_display_helper.VideoRecorder(
rec_filename="PVM_recording.avi"),
do_display=display,
checkpoint=checkpoint,
checkpoint_storage=storage,
dataset_name=dataset)
CoreUtils.run_model(simulation_dict, manager, port=int(port))
if filename != "" and options['supervised'] != "1":
CoreUtils.save_model(simulation_dict, filename)
to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'],
simulation_dict['name'],
simulation_dict['hash'])
from_path = filename
storage.put(from_path=from_path,
to_folder=to_folder,
overwrite=True)
if remove_artifact_files:
os.remove(from_path)
elif options['supervised'] == "1":
CoreUtils.save_model(
simulation_dict, "PVM_state_supervised_%s_%d_%d_%f.p.gz" %
(dataset, simulation_dict['N'][0], int(
options['steps']), float(options['supervised_rate'])))
to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'],
simulation_dict['name'],
simulation_dict['hash'])
from_path = "./PVM_state_supervised_%s_%d_%d_%f.p.gz" % (
dataset, simulation_dict['N'][0], int(
options['steps']), float(options['supervised_rate']))
storage.put(from_path=from_path,
to_folder=to_folder,
overwrite=True)
if remove_artifact_files:
os.remove(from_path)
else:
CoreUtils.save_model(simulation_dict, "PVM_state_final.p.gz")
to_folder = "PVM_models/%s_%s_%s" % (simulation_dict['timestamp'],
simulation_dict['name'],
simulation_dict['hash'])
from_path = "./PVM_state_final.p.gz"
storage.put(from_path=from_path,
to_folder=to_folder,
overwrite=True)
if remove_artifact_files:
os.remove(from_path)
else:
print "Evaluating the system"
logging.info("Evaluating the system")
to_folder = "PVM_models/%s_%s_%s/eval_%09d/" % (
simulation_dict['timestamp'], simulation_dict['name'],
simulation_dict['hash'], simulation_dict['N'])
# Evaluate
signal = PVM_SignalProvider.SimpleSignalProvider(
files=PVM_set.testing,
storage=storage,
frame_resolution=(simulation_dict['input_array'].shape[1],
simulation_dict['input_array'].shape[0]),
heatmap_resolution=simulation_dict['readout_array_float00'].
shape[:2][::-1],
channel="default",
reverse=(int(options['reverse']) > 0))
name = "PVM_train_eval_%s_%09d_test_combined.avi" % (
simulation_dict['hash'], simulation_dict['N'])
manager = Manager(
simulation_dict,
steps_to_run=-1,
signal_provider=signal,
record=True,
video_recorder=PVM_display_helper.VideoRecorder(rec_filename=name),
do_display=display,
evaluate=True,
collect_error=True)
manager.freeze_learning()
CoreUtils.run_model(simulation_dict, manager, port=int(port))
from_path = name
storage.put(from_path=from_path, to_folder=to_folder, overwrite=True)
os.remove(name)
logging.info("Finished on test files")
# Individual files
for (i, test) in enumerate(PVM_set.all):
print "Running on %s" % test
logging.info("Running on %s" % test)
name = "PVM_eval_%s_%09d_%01d_%s.avi" % (
simulation_dict['hash'], simulation_dict['N'], i, test[1])
signal = PVM_SignalProvider.SimpleSignalProvider(
files=[test],
storage=storage,
frame_resolution=(simulation_dict['input_array'].shape[1],
simulation_dict['input_array'].shape[0]),
heatmap_resolution=simulation_dict['readout_array_float00'].
shape[:2][::-1],
channel="default")
manager = Manager(simulation_dict,
steps_to_run=-1,
signal_provider=signal,
record=True,
video_recorder=PVM_display_helper.VideoRecorder(
rec_filename=name),
do_display=display,
evaluate=True)
CoreUtils.run_model(simulation_dict, manager, port=int(port))
from_path = name
storage.put(from_path=from_path,
to_folder=to_folder,
overwrite=True)
os.remove(name)
logging.info("Finished on %s" % test)