def save_models(tmp_dir, data_dir, config_file):
## Train
model_dir = tmp_dir + "/models"
if not os.path.exists(model_dir):
os.mkdir(model_dir)
model_prefix = model_dir + "/my-model-" + str(uuid.uuid4())[:8]
num_training_layers = 310
train.train(config_file, data_dir, model_prefix, num_training_layers)
## Check all Keras and TF models files are present
suffixes = [
".h5", ".json", "-labels.json", ".pb", ".ckpt.meta", ".ckpt.index",
".ckpt.data-00000-of-00001"
]
for suffix in suffixes:
model_path = model_prefix + suffix
assert os.path.exists(model_path)
## Run freeze_graph
freeze.freeze(model_prefix)
frozen_graph_path = model_prefix + "-frozen.pb"
assert os.path.exists(frozen_graph_path)
return frozen_graph_path, model_prefix
def test_replace_data_should_change_metadata(blank):
bcstate = blank
md_a = bcstate.get_metadata()
hash_a = hash(freeze.freeze(md_a))
bcstate.replace_data([(1, 1,), (2, 2,)], [10., 10.])
md_b = bcstate.get_metadata()
hash_b = hash(freeze.freeze(md_b))
assert hash_a != hash_b
def main():
parser = optparse.OptionParser("%prog - I diff repo manifests")
parser.add_option("--filter", dest="filters", action="append")
parser.add_option("--diff", "-d", dest="diff", action="store_false", default=True)
parser.add_option("--format", dest="out_format", default="report")
parser.add_option("--output", dest="output", default=None)
parser.add_option("--root", dest="root", default=os.getcwd())
options, args = parser.parse_args()
if not options.output:
output = sys.stdout
else:
output = options.output
if os.path.exists(output):
print >> sys.stderr, "ERROR: Output file already exists"
exit(1)
if len(args) == 0:
print "Choose a command: diff, cleanup, filter"
exit(1)
elif len(args) > 1:
cmd_args = args[1:]
else:
cmd_args = None
cmd = args[0]
if cmd == 'diff':
if len(cmd_args) != 2:
print >> sys.stderr, "ERROR: must specify exactly two arguments (left and right)"
exit(1)
diff(cmd_args[0], cmd_args[1], output=output, output_format=options.out_format, filters=options.filters)
elif cmd == 'freeze':
freeze(cmd_args[0], output, options.root,
gaia_branch='v1-train',
gecko_branch='gecko-18',
moz_remotes=['b2g'],
moz_branch='v1-train')
elif cmd == 'cleanup':
if len(cmd_args) != 1:
print >> sys.stderr, "ERROR: you can only filter one file at a time"
exit(1)
cleanup(cmd_args[0], output, options.filters)
elif cmd == 'filter':
if len(cmd_args) != 1:
print >> sys.stderr, "ERROR: you can only filter one file at a time"
exit(1)
if options.filters == None:
print >> sys.stderr, "ERROR: you must specify filters for the filter command"
exit(1)
filter(cmd_args[0], output, options.filters)
def test_transition_should_change_metadata(blank):
bcstate = blank
md_a = bcstate.get_metadata()
hash_a = hash(freeze.freeze(md_a))
md_a_cpy = bcstate.get_metadata()
hash_a_cpy = hash(freeze.freeze(md_a_cpy))
assert hash_a == hash_a_cpy
bcstate.transition()
md_b = bcstate.get_metadata()
hash_b = hash(freeze.freeze(md_b))
assert hash_a != hash_b
def test_conditioned_row_resample_should_not_change_metadata_rejected(blank):
bcstate = blank
# double exponential model
logcf = lambda x: float('-Inf')
md_a = bcstate.get_metadata()
hash_a = hash(freeze.freeze(md_a))
acr = bcstate.conditioned_row_resample(2, logcf, num_samples=10)
assert acr == 0.0
md_b = bcstate.get_metadata()
hash_b = hash(freeze.freeze(md_b))
assert hash_a == hash_b
def test_transition_should_change_metadata(blank):
bcstate = blank
md_a = bcstate.get_metadata()
hash_a = hash(freeze.freeze(md_a))
md_a_cpy = bcstate.get_metadata()
hash_a_cpy = hash(freeze.freeze(md_a_cpy))
assert hash_a == hash_a_cpy
bcstate.transition()
md_b = bcstate.get_metadata()
hash_b = hash(freeze.freeze(md_b))
assert hash_a != hash_b
def cashed_fun(*args, **kwargs):
element = freeze(args + tuple(kwargs.items()))
if element in cash:
print('from cache')
return cash[element]
else:
result = f(*args, **kwargs)
cash[element] = result
return result
def save_imagenet(net_architecture, prefix):
K.set_learning_phase(1)
# Set neural net architecture
if net_architecture == "InceptionV3":
model = InceptionV3(weights="imagenet", include_top=True)
elif net_architecture == "ResNet50":
model = ResNet50(weights="imagenet", include_top=True)
elif net_architecture == "MobileNet":
model = MobileNet(weights="imagenet", include_top=True)
else:
print "[ERROR] Didn't recognize net ", net_architecture
sys.exit(-1)
net.compile(model)
net.save_pb(model, prefix)
freeze.freeze(prefix)
return
def test_conditioned_row_resample_should_change_metadata(blank):
bcstate = blank
# double exponential model
logcf = lambda x: np.sum(-np.abs(x))
md_a = bcstate.get_metadata()
hash_a = hash(freeze.freeze(md_a))
acr = bcstate.conditioned_row_resample(2, logcf, num_samples=1)
while acr == 0:
acr = bcstate.conditioned_row_resample(2, logcf, num_samples=1)
assert acr > 0.0
md_b = bcstate.get_metadata()
hash_b = hash(freeze.freeze(md_b))
assert hash_a != hash_b
def test_freeze_simple_script(self):
script = textwrap.dedent("""
import sys
print('running...')
sys.exit(0)
""")
with os_helper.temp_dir() as outdir:
outdir, scriptfile, python = helper.prepare(script, outdir)
executable = helper.freeze(python, scriptfile, outdir)
text = helper.run(executable)
self.assertEqual(text, 'running...')
def nan_to_string(data):
"""Convert NaNs to strings in order to compare the structure"""
frez = freeze.traverse_frozen_data(
freeze.recursive_sort(
freeze.freeze(data)
)
)
for elem in frez: # pragma: no cover
if isinstance(elem, float) and math.isnan(elem):
yield "nan"
else:
yield elem
model_file_basename = args.class_name
input_binary = False
sys.path.append(os.path.dirname(args.path_to_code_output_file))
basename, ext = os.path.splitext(
os.path.basename(args.path_to_code_output_file))
module = __import__(basename)
print("Imported %s from" % basename)
print(module)
constructor = getattr(module, args.class_name)
# todo pass in network dimensions
x = tf.placeholder(
tf.float32,
shape=[1, args.input_width, args.input_height, args.input_channels],
name="input")
net = constructor({'data': x})
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
net.load(args.path_to_data_output_file, sess)
model_input = x.name.replace(":0", "")
model_output = net.get_output().name.replace(":0", "")
width, height, channels = args.input_width, args.input_height, args.input_channels
# END OF caffe-tensorflow/convert.py specific code ...
freeze(sess, model_file_basename, model_input, width, height, channels,
model_output)
"Attempts to export a keras trained inference or regression model to a tensorflow protocol buffer file"
)
parser.add_argument(
"path_to_keras_model",
type=str,
help=
"path to the trained keras hdf5 model generated by calling model.save(filepath)"
)
args = parser.parse_args()
input_binary = True
K.set_learning_phase(0)
path_to_keras_model_file = args.path_to_keras_model
model_file_basename, file_extension = os.path.splitext(
os.path.basename(path_to_keras_model_file))
model = keras.models.load_model(path_to_keras_model_file,
custom_objects={'relu6': relu6})
model_input = model.input.name.replace(':0', '')
model_output = model.output.name.replace(':0', '')
sess = K.get_session()
width, height, channels = int(model.input.shape[2]), int(
model.input.shape[1]), int(model.input.shape[3])
# END OF keras specific code
freeze(sess, path_to_keras_model_file, model_input, width, height,
channels, model_output)
def freeze(expenses, out):
from freeze import freeze
freeze(expenses, os.path.abspath(out))
