def build_model_dir(model_dir, objects, config, check_integrity_fn):
"""Prepares the model directory based on the model package."""
if os.path.exists(model_dir):
raise ValueError("model directory %s already exists" % model_dir)
else:
logger.info("Building model package in %s", model_dir)
os.mkdir(model_dir)
for target, source in six.iteritems(objects):
if os.path.isdir(source):
shutil.copytree(source, os.path.join(model_dir, target))
else:
shutil.copyfile(source, os.path.join(model_dir, target))
config_path = os.path.join(model_dir, 'config.json')
with open(config_path, 'w') as config_file:
json.dump(config, config_file)
objects['config.json'] = config_path
if "description" in config:
readme_path = os.path.join(model_dir, 'README.md')
with open(readme_path, 'w') as readme_file:
readme_file.write(config['description'])
objects['README.md'] = readme_path
md5 = md5files(
(k, v) for k, v in six.iteritems(objects) if check_integrity_fn(k))
with open(os.path.join(model_dir, "checksum.md5"), "w") as f:
f.write(md5)
def check_model_dir(model_dir):
"""Compares model package MD5."""
logger.info("Checking integrity of model package %s", model_dir)
md5ref = None
with open(os.path.join(model_dir, "checksum.md5"), "r") as f:
md5ref = f.read().strip()
files = os.listdir(model_dir)
md5check = md5files([(f, os.path.join(model_dir, f)) for f in files
if f != "checksum.md5"])
return md5check == md5ref
def check_model_dir(model_dir, check_integrity_fn):
"""Compares model package MD5."""
logger.info("Checking integrity of model package %s", model_dir)
md5_file = os.path.join(model_dir, "checksum.md5")
if not os.path.exists(md5_file):
return True
md5ref = None
with open(md5_file, "r") as f:
md5ref = f.read().strip()
files = os.listdir(model_dir)
md5check = md5files([(f, os.path.join(model_dir, f)) for f in files if check_integrity_fn(f)])
return md5check == md5ref
def build_model_dir(model_dir, objects, config):
"""Prepares the model directory based on the model package."""
if os.path.exists(model_dir):
raise ValueError("model directory %s already exists" % model_dir)
else:
logger.info("Building model package in %s", model_dir)
os.mkdir(model_dir)
config_path = os.path.join(model_dir, 'config.json')
with open(config_path, 'w') as config_file:
json.dump(config, config_file)
for target, source in six.iteritems(objects):
shutil.copyfile(source, os.path.join(model_dir, target))
objects['config.json'] = config_path
md5 = md5files(six.iteritems(objects))
with open(os.path.join(model_dir, "checksum.md5"), "w") as f:
f.write(md5)
def build_model_dir(model_dir, objects, config, should_check_integrity_fn):
"""Prepares the model directory based on the model package."""
if os.path.exists(model_dir):
raise ValueError("model directory %s already exists" % model_dir)
else:
logger.info("Building model package in %s", model_dir)
os.mkdir(model_dir)
for target, source in six.iteritems(objects):
if os.path.isdir(source):
shutil.copytree(source, os.path.join(model_dir, target))
else:
shutil.copyfile(source, os.path.join(model_dir, target))
config_path = save_model_config(model_dir, config)
objects[os.path.basename(config_path)] = config_path
if "description" in config:
readme_path = os.path.join(model_dir, "README.md")
with io.open(readme_path, "w", encoding="utf-8") as readme_file:
readme_file.write(config["description"])
objects["README.md"] = readme_path
md5 = md5files((k, v) for k, v in six.iteritems(objects)
if should_check_integrity_fn(k))
with open(os.path.join(model_dir, "checksum.md5"), "w") as f:
f.write(md5)
