def download_and_extract_model(url,
model_extension='.cntk',
local_folder=None):
"""Downloads file pointed to by `url`. Once downloaded, unzips the downloaded file.
If `local_folder` is not supplied, downloads to the current folder.
"""
def get_model_name(url):
filename = os.path.basename(url)
model_filename, ext = os.path.splitext(filename)
if (not ext == '.zip'):
model_filename = filename
model_name, _ = os.path.splitext(model_filename)
return model_name
# Download the file
filename = download_file(url, local_folder)
_logger.info("Extracting zipped model: " + filename)
# Extract the file if it's a zip
unzip = ziptools.Extractor(filename)
success, model_filename = unzip.extract_file(model_extension)
if success:
_logger.info("Extracted zipped model: " + model_filename)
else:
_logger.info("Non-zipped model: " + filename)
return model_filename
def main(argv):
arg_parser = argparse.ArgumentParser(
"Converts CNTK model to ELL model\n"
"Example:\n"
" cntk_import.py VGG16_ImageNet_Caffe.model\n"
"This outputs 'VGG16_ImageNet_Caffe.ell' and 'VGG16_ImageNet_Caffe_config.json'\n"
)
arg_parser.add_argument(
"cntk_model_file",
help="path to a CNTK model file, or a zip archive of a CNTK model file"
)
arg_parser.add_argument("--zip_ell_model",
help="zips the output ELL model if set",
action="store_true")
model_options = arg_parser.add_argument_group('model_options')
model_options.add_argument(
"--step_interval",
help="produce a steppable ELL model for a millisecond interval",
default=0)
model_options.add_argument(
"--lag_threshold",
help=
"number of step intervals to fall behind before notifying the caller.\n"
"used when step_interval is set\n",
default=5)
args = vars(arg_parser.parse_args(argv))
model_options = args.get('model_options', {})
step_interval = model_options.get('step_interval')
lag_threshold = model_options.get('lag_threshold')
# extract the model if it's in an archive
unzip = ziptools.Extractor(args['cntk_model_file'])
success, filename = unzip.extract_file(".cntk")
if success:
print("extracted: " + filename)
else:
# not a zip archive
filename = args['cntk_model_file']
predictor = cntk_to_ell.predictor_from_cntk_model(filename)
model_file_name = os.path.splitext(filename)[0] + '.ell'
ell_map = ell_utilities.ell_map_from_float_predictor(
predictor, step_interval, lag_threshold)
print("Saving model file: '" + model_file_name + "'")
ell_map.Save(model_file_name)
if args['zip_ell_model']:
print("Zipping model file: '" + model_file_name + ".zip'")
zipper = ziptools.Zipper()
zipper.zip_file(model_file_name, model_file_name + ".zip")
os.remove(model_file_name)
def _ensure_model_file(self):
"Lazy-extracts the ELL model file from zip if it doesn't exist"
try:
filename = self._get_data_filename(".ell", is_suffix=True)
self.model_file = filename
except:
filename = self._get_data_filename(".ell.zip", is_suffix=True)
unzip = ziptools.Extractor(filename)
success, temp = unzip.extract_file(".ell")
if (success):
print("extracted: " + temp)
self.model_file = temp
self.cleanup_model_file = True
else:
# not a zip archive
raise(FileNotFoundError, "{} is not a valid zip archive".format(filename))
def main(argv):
arg_parser = argparse.ArgumentParser(
"Converts CNTK model to ELL model\n"
"Example:\n"
" cntk_import.py VGG16_ImageNet_Caffe.model\n"
"This outputs 'VGG16_ImageNet_Caffe.ell' and 'VGG16_ImageNet_Caffe_config.json'\n"
)
arg_parser.add_argument(
"cntk_model_file",
help="path to a CNTK model file, or a zip archive of a CNTK model file"
)
arg_parser.add_argument("--zip_ell_model",
help="zips the output ELL model if set",
action="store_true")
args = arg_parser.parse_args(argv)
# extract the model if it's in an archive
unzip = ziptools.Extractor(args.cntk_model_file)
success, filename = unzip.extract_file(".cntk")
if (success):
print("extracted: " + filename)
else:
# not a zip archive
filename = args.cntk_model_file
predictor = cntk_to_ell.predictor_from_cntk_model(filename)
input_shape = predictor.GetInputShape()
output_shape = predictor.GetOutputShape()
model_file_name = os.path.splitext(filename)[0] + '.ell'
head, tail = os.path.split(model_file_name)
ell_map = ell_utilities.ell_map_from_float_predictor(predictor)
print("Saving model file: '" + model_file_name + "'")
ell_map.Save(model_file_name)
if (args.zip_ell_model):
print("Zipping model file: '" + model_file_name + ".zip'")
zipper = ziptools.Zipper()
zipper.zip_file(model_file_name, model_file_name + ".zip")
os.remove(model_file_name)
def download_and_extract_model(url, model_extension='.cntk'):
"""Downloads file pointed to by `url`. Once downloaded, unzips the
downloaded file.
"""
modelName = get_model_name(url)
# Download the file
fileName = download_file(url)
# Extract the file if it's a zip
unzip = ziptools.Extractor(fileName)
success, modelFileName = unzip.extract_file(model_extension)
if success:
print("extracted zipped model: " + modelFileName)
else:
print("non-zipped model: " + fileName)
modelFileName = fileName
return modelName
def main(argv):
arg_parser = argparse.ArgumentParser(
description="Converts CNTK model to ELL model\n"
"Example:\n"
" cntk_import.py model.cntk\n"
"This outputs 'model.ell' which can be compiled with ELL's 'wrap' tool\n"
)
arg_parser.add_argument(
"cntk_model_file",
help="path to a CNTK model file, or a zip archive of a CNTK model file"
)
arg_parser.add_argument("--zip_ell_model",
help="zips the output ELL model if set",
action="store_true")
arg_parser.add_argument(
"--use_legacy_importer",
help=
"specifies whether to use the new importer engine or the legacy importer",
action="store_true")
arg_parser.add_argument(
"--plot_model",
help=
"specifies whether to plot the model using SVG to cntk_model_file.svg",
action="store_true")
arg_parser.add_argument(
"--verify_vision_model",
help=
"verifies the imported vision ELL model produces the same output as the original CNTK model",
action="store_true")
arg_parser.add_argument(
"--verify_audio_model",
help=
"verifies the imported audio ELL model produces the same output as the original CNTK model",
action="store_true")
model_options = arg_parser.add_argument_group('model_options')
model_options.add_argument(
"--step_interval",
help="produce a steppable ELL model for a millisecond interval",
default=0)
model_options.add_argument(
"--lag_threshold",
help="millisecond time lag before notifying the caller.\n"
"used when step_interval is set\n",
default=5)
args = vars(arg_parser.parse_args(argv))
model_options = args.get('model_options', {})
step_interval = model_options.get('step_interval', 0)
lag_threshold = model_options.get('lag_threshold', 0)
plot_model = args["plot_model"]
verify_model = {
"vision": args["verify_vision_model"],
"audio": args["verify_audio_model"]
}
# extract the model if it's in an archive
unzip = ziptools.Extractor(args['cntk_model_file'])
success, filename = unzip.extract_file(".cntk")
if success:
_logger.info("Extracted: " + filename)
else:
# not a zip archive
filename = args['cntk_model_file']
if not args["use_legacy_importer"]:
_logger.info("-- Using new importer engine --")
ell_map = cntk_to_ell.map_from_cntk_model_using_new_engine(
filename, step_interval, lag_threshold, plot_model, verify_model)
else:
_logger.info("-- Using legacy importer --")
predictor = cntk_to_ell.predictor_from_cntk_model(filename)
ell_map = ell.neural.utilities.ell_map_from_float_predictor(
predictor, step_interval, lag_threshold)
model_file_name = os.path.splitext(filename)[0] + ".ell"
_logger.info("\nSaving model file: '" + model_file_name + "'")
ell_map.Save(model_file_name)
if args["zip_ell_model"]:
_logger.info("Zipping model file: '" + model_file_name + ".zip'")
zipper = ziptools.Zipper()
zipper.zip_file(model_file_name, model_file_name + ".zip")
os.remove(model_file_name)
