resource_id)
reify_script = whizzml_script(args, api)
# apply the reify script to the resource
execute_command = [
'execute', '--script', reify_script, '--output-dir', args.output_dir
]
command_args, _, _, exe_session_file, _ = get_context( \
execute_command, EXE_SETTINGS)
command_args.arguments_ = [["res-id", resource_id]]
command_args.inputs = json.dumps(command_args.arguments_)
# process the command
session_file = None
execute_whizzml(command_args, api, session_file)
with open("%s.json" % command_args.output) as file_handler:
exe_output = json.load(file_handler)['result']
if args.language == "nb":
write_nb_output(resource_id, \
exe_output, args.output.replace(".py", ".ipynb"), api)
return
elif args.language == "whizzml":
output = exe_output["source_code"]
args.output = args.output.replace(".py", ".whizzml")
exe_output["source_code"] = args.output
exe_output["kind"] = "script"
with open(os.path.join(os.path.dirname(args.output), "metadata.json"),
"w") as meta_handler:
meta_handler.write(json.dumps(exe_output))
resource_id)
reify_script = whizzml_script(args, api)
# apply the reify script to the resource
execute_command = ['execute',
'--script', reify_script,
'--output-dir', args.output_dir]
command_args, _, _, exe_session_file, _ = get_context( \
execute_command, EXE_SETTINGS)
command_args.arguments_ = [["res-id", resource_id]]
command_args.inputs = json.dumps(command_args.arguments_)
# process the command
session_file = None
execute_whizzml(command_args, api, session_file)
with open("%s.json" % command_args.output) as file_handler:
exe_output = json.load(file_handler)['result']
if args.language == "nb":
write_nb_output(resource_id, \
exe_output, args.output.replace(".py", ".ipynb"), api)
return
elif args.language == "whizzml":
output = exe_output["source_code"]
args.output = args.output.replace(".py", ".whizzml")
exe_output["source_code"] = args.output
exe_output["kind"] = "script"
with open(os.path.join(os.path.dirname(args.output),
"metadata.json"), "w") as meta_handler:
meta_handler.write(json.dumps(exe_output))
def retrain_model(args, api, common_options, session_file=None):
"""Retrieve or create the retrain script for a model and
execute it with the new provided data
"""
# retrieve the modeling resource to be retrained by tag or id
if args.resource_id:
resource_id = args.resource_id
reference_tag = "retrain:%s" % resource_id
else:
for model_type in MODEL_TYPES:
if hasattr(args, "%s_tag" % model_type) and \
getattr(args, "%s_tag" % model_type) is not None:
tag = getattr(args, "%s_tag" % model_type)
query_string = "tags=%s" % tag
resource_id = get_last_resource( \
model_type.replace("_", ""),
api=api,
query_string=query_string)
if resource_id is None:
sys.exit("Failed to find the %s with tag %s. "
"Please, check the tag and"
" the connection info (domain and credentials)." %
(model_type.replace("_", " "), tag))
reference_tag = tag
break
if args.upgrade:
shutil.rmtree(BIGMLER_SCRIPTS_DIRECTORY)
script_id = None
else:
# check for the last script used to retrain the model
query_string = "tags=retrain:%s" % resource_id
script_id = get_last_resource( \
"script",
api=api,
query_string=query_string)
if script_id is None:
# if the script to retrain does not exist:
# check whether the model exists
try:
bigml.api.check_resource(resource_id, raise_on_error=True, api=api)
except Exception, exc:
sys.exit("Failed to find the model %s. Please, check its ID and"
" the connection info (domain and credentials)." %
resource_id)
# look for the script that creates the rebuild script.
retrain_file = os.path.join(BIGMLER_SCRIPTS_DIRECTORY,
"retrain",
"scripts")
reify_script = get_script_id(retrain_file)
if reify_script is None:
# new bigmler command: creating the scriptify scripts
whizzml_command = ['whizzml',
'--package-dir', INCREMENTAL_PACKAGE_PATH,
'--output-dir', BIGMLER_SCRIPTS_DIRECTORY]
whizzml_dispatcher(args=whizzml_command)
reify_script = get_script_id(retrain_file)
# new bigmler command: creating the retrain script
execute_command = ['execute',
'--script', reify_script,
'--output-dir', args.output_dir]
command_args, _, _, exe_session_file, _ = get_context(execute_command,
EXE_SETTINGS)
command_args.arguments_ = [["model-resource", resource_id]]
command_args.inputs = json.dumps(command_args.arguments_)
# process the command
execute_whizzml(command_args, api, session_file)
script_id = extract_retrain_id(command_args, api, session_file)
def retrain_model(args, api, common_options, session_file=None):
"""Retrieve or create the retrain script for a model and
execute it with the new provided data
"""
# retrieve the modeling resource to be retrained by tag or id
if args.resource_id:
resource_id = args.resource_id
reference_tag = "retrain:%s" % resource_id
else:
for model_type in MODEL_TYPES:
if hasattr(args, "%s_tag" % model_type) and \
getattr(args, "%s_tag" % model_type) is not None:
tag = getattr(args, "%s_tag" % model_type)
query_string = "tags=%s" % tag
resource_id = get_last_resource( \
model_type.replace("_", ""),
api=api,
query_string=query_string)
if resource_id is None:
sys.exit("Failed to find the %s with tag %s. "
"Please, check the tag and"
" the connection info (domain and credentials)." %
(model_type.replace("_", " "), tag))
reference_tag = tag
break
if args.upgrade:
shutil.rmtree(BIGMLER_SCRIPTS_DIRECTORY)
script_id = None
else:
# check for the last script used to retrain the model
query_string = "tags=retrain:%s" % resource_id
script_id = get_last_resource( \
"script",
api=api,
query_string=query_string)
if script_id is None:
# if the script to retrain does not exist:
# check whether the model exists
try:
bigml.api.check_resource(resource_id, raise_on_error=True, api=api)
except Exception, exc:
sys.exit("Failed to find the model %s. Please, check its ID and"
" the connection info (domain and credentials)." %
resource_id)
# look for the script that creates the rebuild script.
retrain_file = os.path.join(BIGMLER_SCRIPTS_DIRECTORY, "retrain",
"scripts")
reify_script = get_script_id(retrain_file)
if reify_script is None:
# new bigmler command: creating the scriptify scripts
whizzml_command = [
'whizzml', '--package-dir', INCREMENTAL_PACKAGE_PATH,
'--output-dir', BIGMLER_SCRIPTS_DIRECTORY
]
whizzml_dispatcher(args=whizzml_command)
reify_script = get_script_id(retrain_file)
# new bigmler command: creating the retrain script
execute_command = [
'execute', '--script', reify_script, '--output-dir',
args.output_dir
]
command_args, _, _, exe_session_file, _ = get_context(
execute_command, EXE_SETTINGS)
command_args.arguments_ = [["model-resource", resource_id]]
command_args.inputs = json.dumps(command_args.arguments_)
# process the command
execute_whizzml(command_args, api, session_file)
script_id = extract_retrain_id(command_args, api, session_file)
def retrain_model(args, api, command, session_file=None):
"""Retrieve or create the retrain script for a model and
execute it with the new provided data
"""
retrain_file = os.path.join(BIGMLER_SCRIPTS_DIRECTORY, "retrain",
"scripts")
try:
os.remove(UPGRADE_FILE)
reify_script = None
try:
shutil.rmtree(BIGMLER_SCRIPTS_DIRECTORY)
except OSError:
pass
except OSError:
# look for the script that creates the rebuild script.
reify_script = get_script_id(retrain_file)
if reify_script is None:
# new bigmler command: creating the scriptify scripts
whizzml_command = [
'whizzml', '--package-dir', INCREMENTAL_PACKAGE_PATH,
'--output-dir', BIGMLER_SCRIPTS_DIRECTORY
]
add_api_context(whizzml_command, args)
whizzml_dispatcher(args=whizzml_command)
reify_script = get_script_id(retrain_file)
# retrieve the modeling resource to be retrained by tag or id
if args.resource_id:
resource_id = args.resource_id
reference_tag = "retrain:%s" % resource_id
else:
for model_type in MODEL_TYPES:
if hasattr(args, "%s_tag" % model_type) and \
getattr(args, "%s_tag" % model_type) is not None:
tag = getattr(args, "%s_tag" % model_type)
query_string = "tags=%s" % tag
resource_id = get_first_resource( \
model_type.replace("_", ""),
api=api,
query_string=query_string)
if resource_id is None:
sys.exit("Failed to find the %s with tag %s. "
"Please, check the tag and"
" the connection info (domain and credentials)." %
(model_type.replace("_", " "), tag))
reference_tag = tag
break
# updating the dataset that generated the model with the reference tag
model = api.getters[get_resource_type(resource_id)](resource_id)
dataset_id = model["object"]["dataset"]
dataset = api.get_dataset(dataset_id)
tags = dataset["object"]["tags"]
if reference_tag not in tags:
tags.append(reference_tag)
api.update_dataset(dataset_id, {"tags": tags})
# if --upgrade, we force rebuilding the scriptified script
if args.upgrade:
script_id = None
else:
# check for the last script used to retrain the model
query_string = "tags=%s" % reference_tag
script_id = get_last_resource( \
"script",
api=api,
query_string=query_string)
if script_id is None:
# if the script to retrain does not exist:
# check whether the model exists
try:
bigml.api.check_resource(resource_id, raise_on_error=True, api=api)
except Exception:
sys.exit("Failed to find the model %s. Please, check its ID and"
" the connection info (domain and credentials)." %
resource_id)
# new bigmler command: creating the retrain script
execute_command = [
'execute', '--script', reify_script, '--tag', reference_tag,
'--output-dir', args.output_dir
]
command.propagate(execute_command)
command_args, _, _, exe_session_file, _ = get_context(
execute_command, EXE_SETTINGS)
command_args.arguments_ = [["model-resource", resource_id]]
command_args.inputs = json.dumps(command_args.arguments_)
# process the command
execute_whizzml(command_args, api, session_file)
script_id = extract_retrain_id(command_args, api, session_file)
# apply the retrain script to the new data:
# add new data: depending on the script we will need to use
# a source-url, a source or a dataset
if args.add:
script_inputs = api.get_script(script_id)['object']['inputs']
input_type = script_inputs[0]['type']
command_args, api, exe_session_file = \
create_input(args, api, input_type, script_id, command)
# process the command
execute_whizzml(command_args, api, exe_session_file)
with open("%s.json" % command_args.output) as file_handler:
model_resource_id = json.load(file_handler)['result']
message = (u'The new retrained model is: %s.\n'
u'You can use the\n\n%s\n\nquery to retrieve the latest'
u' retrained model.\n\n') % \
(model_resource_id, last_resource_url( \
resource_id, api, \
"limit=1;full=yes;tags=%s" % reference_tag))
log_message(message, log_file=session_file, console=1)
def reify_resources(args, api):
""" Extracts the properties of the created resources and generates
code to rebuild them
"""
resource_id = get_resource_id(args.resource_id)
if resource_id is None:
sys.exit("Failed to match a valid resource ID. Please, check: %s" %
args.resource_id)
# check whether the resource exists
try:
check_resource(resource_id, raise_on_error=True, api=api)
except Exception:
sys.exit("Failed to find the resource %s. Please, check its ID and"
" the connection info (domain and credentials)." %
resource_id)
reify_script = whizzml_script(args, api)
# apply the reify script to the resource
execute_command = [
'execute', '--script', reify_script, '--output-dir', args.output_dir
]
command_args, _, _, _, _ = get_context( \
execute_command, EXE_SETTINGS)
command_args.arguments_ = [["res-id", resource_id]]
command_args.inputs = json.dumps(command_args.arguments_)
# process the command
session_file = None
execute_whizzml(command_args, api, session_file)
with open("%s.json" % command_args.output) as file_handler:
exe_output = json.load(file_handler)['result']
if args.language == "nb":
write_nb_output(resource_id, \
exe_output, args.output.replace(".py", ".ipynb"), api)
return
elif args.language == "whizzml":
output = exe_output["source_code"]
args.output = args.output.replace(".py", ".whizzml")
exe_output["source_code"] = args.output
exe_output["kind"] = "script"
with open(os.path.join(os.path.dirname(args.output), "metadata.json"),
"w") as meta_handler:
meta_handler.write(json.dumps(exe_output))
else:
output = python_output(exe_output, api)
prefix = u"""\
#!/usr/bin/env python
# -*- coding: utf-8 -*-
\"\"\"Python code to reify %s
Generated by BigMLer
\"\"\"
def main():
""" % resource_id
suffix = u"""\
if __name__ == "__main__":
main()
"""
output = "%s%s\n%s" % (prefix, output, suffix)
write_to_utf8(args.output, output)
sts = os.stat(args.output)
os.chmod(args.output, sts.st_mode | stat.S_IEXEC)