def __init__(self):
NEEDED_FILES = ("scripts.js", "style.css", "intel-logo-white.png")
HTML_TEMPLATE_DIR = 'html_static_files'
HTML_STATIC = '/'.join((HtmlFormatter.REPORT_FOLDER, 'static'))
self._check_and_make_path(HTML_STATIC)
try:
import pip
dist = None
for d in pip.get_installed_distributions():
if d.project_name == 'Flask-Bootstrap':
dist = d.location
project_path = "/".join([dist, "flask_bootstrap", "static"])
copytree(project_path, HTML_STATIC)
except ImportError:
print "CTS_ERROR:: %s" % ImportError
try:
for f in NEEDED_FILES:
if not exists('/'.join((HTML_STATIC, f))):
resource_package = __name__
resource_path = '/'.join((HTML_TEMPLATE_DIR, f))
template = pkg_resources.resource_string(resource_package, resource_path)
with open('/'.join((HTML_STATIC, f)), 'w') as resource:
resource.write(template)
except IOError as e:
print("CTS can't create file or directory. Do you've appropriate permissions?")
def __init__(self):
NEEDED_FILES = ("scripts.js", "style.css", "intel-logo-white.png")
HTML_TEMPLATE_DIR = 'html_static_files'
HTML_STATIC = '/'.join((HtmlFormatter.REPORT_FOLDER, 'static'))
self._check_and_make_path(HTML_STATIC)
try:
import pip
dist = None
for d in pip.get_installed_distributions():
if d.project_name == 'Flask-Bootstrap':
dist = d.location
project_path = "/".join([dist, "flask_bootstrap", "static"])
copytree(project_path, HTML_STATIC)
except ImportError:
print "CTS_ERROR:: %s" % ImportError
try:
for f in NEEDED_FILES:
if not exists('/'.join((HTML_STATIC, f))):
resource_package = __name__
resource_path = '/'.join((HTML_TEMPLATE_DIR, f))
template = pkg_resources.resource_string(resource_package, resource_path)
with open('/'.join((HTML_STATIC, f)), 'w') as resource:
resource.write(template)
except IOError as e:
print("CTS can't create file or directory. Do you've appropriate permissions?")
def start_score_evaluation_on_validation_experiment(temp_conf, exp_path, data_path):
os.makedirs(exp_path, exist_ok=True)
copytree(join(gitrepo_path, 'denoiseg'), join(exp_path, 'denoiseg'))
cp(join(gitrepo_path, 'validation_evaluation.py'), exp_path)
with open(join(exp_path, 'temp.json'), 'w') as f:
json.dump(temp_conf, f)
slurm_script = create_slurm_script(exp_path, data_path)
with open(join(exp_path, 'slurm.job'), 'w') as f:
for l in slurm_script:
f.write(l)
os.system('chmod -R 775 ' + exp_path)
os.system('sbatch {}'.format(join(exp_path, "slurm.job")))
def start_experiment(exp_conf, exp_path, data_path):
os.makedirs(exp_path, exist_ok=True)
copytree(resources_path, exp_path)
cp(join(resources_path, 'main.py'), exp_path)
with open(join(exp_path, 'experiment.json'), 'w') as f:
json.dump(exp_conf, f, sort_keys=True, indent=4)
singularity_cmd = 'singularity exec -B {}:/notebooks -B {}:/data {} python3 /notebooks/main.py --exp_config ' \
'/notebooks/experiment.json'.format(exp_path, data_path, singularity_path)
slurm_script = create_slurm_script(singularity_cmd)
with open(join(exp_path, 'slurm.job'), 'w') as f:
for l in slurm_script:
f.write(l)
os.system('chmod -R 775 ' + exp_path)
# Submit the cluster-job via slurm-script
os.system('sbatch {}'.format(join(exp_path, 'slurm.job')))
def copy_http_invoker_func_data(
config,
extra_config,
app_type,
app_path,
):
"""
If the user configured a PULL function using the Invoker variant, this function generates
the necessary files to deploy the invocation function in the same function app as the
main PULL function.
:param config: Configuration dictionary (the one that gets exported to local.settings.json).
:param extra_config: Dictionary configuring the additional options passed by user in generator_config.json.
:param app_type: Application type (Unnecessary, here only for abstraction purposes).
:param app_path: Path to the output folder for the Function App.
:return: The local.settings.json dictionary currently being formed.
"""
# If the user has chosen to enable the invoker with their function, add the necessary files.
if extra_config.get("invoker").lower() == "true":
invoker_path = f"{app_path}http_invoker"
invoker_func_json_path = f"{invoker_path}/function.json"
copytree(INVOKER_FUNC_PATH, invoker_path)
copyfile(
INVOKER_FUNC_JSON,
invoker_func_json_path,
)
with open(invoker_func_json_path, "r") as f_invoker_func_json:
inv_func = json.loads(f_invoker_func_json.read())
inv_func["scriptFile"] = "invoker.py"
with open(invoker_func_json_path, "w") as f_invoker_func_json:
f_invoker_func_json.write(json.dumps(inv_func, indent=JSON_STRING_INDENT))
logging.info(textwrap.indent(
text="COPIED INVOKER DATA",
prefix=' ' * PRINT_LEVEL_INDENT * 8,
))
return config
def app_create(
app_type,
app_name,
output_path,
extra_config,
):
"""
Creates the Function App folder, containing all the necessary files and configuration samples.
:param app_type: Type of Function App (type of Persistor) being generated.
:param app_name: Name of the Function App.
:param output_path: Path to the output folder.
:param extra_config: Dictionary containing any additional parameters the user set in generator_config.json.
:return:
"""
logging.info("CREATING FILES FOR APP: %s", app_name)
app_path = f"{output_path}{app_name}/"
main_func_path = f"{app_path}{app_type}"
if "output_binding" in extra_config and extra_config["output_binding"].lower() == "true":
main_func_origin = "{original_path}_output_binding/".format(
original_path=MAIN_FUNC_PATH[:-1]
)
else:
main_func_origin = MAIN_FUNC_PATH
# Copy the main function.
copytree(main_func_origin, main_func_path)
logging.info(textwrap.indent(
text="COPIED MAIN FUNCTION",
prefix=' ' * PRINT_LEVEL_INDENT * 2,
))
# Copy the function.json specific to the chosen Persistor type.
copyfile(
f"{JSON_PRESETS_PATH}{PERSISTOR_TYPE_FUNC_TYPE[app_type]}.json",
f"{main_func_path}/function.json",
)
logging.info(textwrap.indent(
text="COPIED FUNCTION.JSON",
prefix=' ' * PRINT_LEVEL_INDENT * 2,
))
# Copy all of the necessary utilities.
copy_utils(app_path)
logging.info(textwrap.indent(
text="COPIED ALL UTILITIES",
prefix=' ' * PRINT_LEVEL_INDENT * 2,
))
logging.info(textwrap.indent(
text="CONFIGURING local.settings.json",
prefix=' ' * PRINT_LEVEL_INDENT * 2,
))
# Create a local.settings.json based on the App Type and given configurations.
local_settings_create(app_type, app_path, extra_config)
logging.info(textwrap.indent(
text="COPIED local.settings.json",
prefix=' ' * PRINT_LEVEL_INDENT * 2,
))
logging.info(textwrap.indent(
text="APP FOLDER STRUCTURE CREATED!\n\n",
prefix=' ' * PRINT_LEVEL_INDENT * 2,
))