def create_projection(self, pca, input_data=None,
args=None, wait_time=3, retries=10):
"""Creates a new projection.
The pca parameter can be a pca resource or ID
"""
pca_id = None
resource_type = get_resource_type(pca)
if resource_type != PCA_PATH:
raise Exception("A PCA resource id is needed"
" to create a projection. %s found." %
resource_type)
pca_id = get_resource_id(pca)
if pca_id is not None:
check_resource(pca_id,
query_string=TINY_RESOURCE,
wait_time=wait_time, retries=retries,
raise_on_error=True, api=self)
if input_data is None:
input_data = {}
create_args = {}
if args is not None:
create_args.update(args)
create_args.update({
"input_data": input_data})
if pca_id is not None:
create_args.update({
"pca": pca_id})
body = json.dumps(create_args)
return self._create(self.projection_url, body,
verify=self.verify)
def status(self, resource):
"""Maps status code to string.
"""
resource_id = get_resource_id(resource)
if resource_id:
resource = self._get("%s%s" % (self.url, resource_id))
status = get_status(resource)
code = status['code']
return STATUSES.get(code, "UNKNOWN")
else:
status = get_status(resource)
if status['code'] != UPLOADING:
LOGGER.error("Wrong resource id")
return
return STATUSES[UPLOADING]
def reify_resource(self, resource_id):
"""Redirects to the reify method according to the resource type
"""
# first check if this is a valid id
resource_id = get_resource_id(resource_id)
if resource_id is not None:
resource_type = get_resource_type(resource_id)
reify_handler = getattr(self, 'reify_%s' % resource_type)
message = "Analyzing %s.\n" % resource_id
self.logger(message)
reify_handler(resource_id)
if self.delete:
self.delete_stored_resource(resource_id)
def status(self, resource):
"""Maps status code to string.
"""
resource_id = get_resource_id(resource)
if resource_id:
resource = self._get("%s%s" % (self.url, resource_id))
status = get_status(resource)
code = status['code']
return STATUSES.get(code, "UNKNOWN")
else:
status = get_status(resource)
if status['code'] != UPLOADING:
LOGGER.error("Wrong resource id")
return
return STATUSES[UPLOADING]
def reify_resource(self, resource_id):
"""Redirects to the reify method according to the resource type
"""
# first check if this is a valid id
resource_id = get_resource_id(resource_id)
if resource_id is not None:
resource_type = get_resource_type(resource_id)
reify_handler = getattr(self, 'reify_%s' % resource_type)
message = "Analyzing %s.\n" % resource_id
self.logger(message)
reify_handler(resource_id)
if self.delete:
self.delete_stored_resource(resource_id)
def create_prediction(self,
model,
input_data=None,
args=None,
wait_time=3,
retries=10):
"""Creates a new prediction.
The model parameter can be:
- a simple tree model
- a simple logistic regression model
- an ensemble
- a deepnet
. a linear regression
- a fusion
Note that the old `by_name` argument has been deprecated.
"""
model_id = None
resource_type = get_resource_type(model)
if resource_type not in SUPERVISED_PATHS:
raise Exception("A supervised model resource id is needed"
" to create a prediction. %s found." %
resource_type)
model_id = get_resource_id(model)
if model_id is not None:
check_resource(model_id,
query_string=TINY_RESOURCE,
wait_time=wait_time,
retries=retries,
raise_on_error=True,
api=self)
if input_data is None:
input_data = {}
create_args = {}
if args is not None:
create_args.update(args)
create_args.update({"input_data": input_data})
if model_id is not None:
create_args.update({"model": model_id})
body = json.dumps(create_args)
return self._create(self.prediction_url,
body,
verify=self.verify_prediction)
def reify_resources(args, api, logger):
""" Extracts the properties of the created resources and generates
code to rebuild them
"""
resource_id = get_resource_id(args.resource_id)
if resource_id is None:
sys.exit("Failed to match a valid resource ID. Please, check: %s" %
args.resource_id)
# check whether the resource exists
try:
check_resource(resource_id, raise_on_error=True, api=api)
except Exception, exc:
sys.exit("Failed to find the resource %s. Please, check its ID and"
" the connection info (domain and credentials)." %
resource_id)
def reify_resources(args, api, logger):
""" Extracts the properties of the created resources and generates
code to rebuild them
"""
resource_id = get_resource_id(args.resource_id)
if resource_id is None:
sys.exit("Failed to match a valid resource ID. Please, check: %s"
% args.resource_id)
# check whether the resource exists
try:
check_resource(resource_id, raise_on_error=True, api=api)
except Exception, exc:
sys.exit("Failed to find the resource %s. Please, check its ID and"
" the connection info (domain and credentials)." %
resource_id)
def create_prediction(self, model, input_data=None,
args=None, wait_time=3, retries=10):
"""Creates a new prediction.
The model parameter can be:
- a simple tree model
- a simple logistic regression model
- an ensemble
- a deepnet
. a linear regression
- a fusion
Note that the old `by_name` argument has been deprecated.
"""
model_id = None
resource_type = get_resource_type(model)
if resource_type not in SUPERVISED_PATHS:
raise Exception("A supervised model resource id is needed"
" to create a prediction. %s found." %
resource_type)
model_id = get_resource_id(model)
if model_id is not None:
check_resource(model_id,
query_string=TINY_RESOURCE,
wait_time=wait_time, retries=retries,
raise_on_error=True, api=self)
if input_data is None:
input_data = {}
create_args = {}
if args is not None:
create_args.update(args)
create_args.update({
"input_data": input_data})
if model_id is not None:
create_args.update({
"model": model_id})
body = json.dumps(create_args)
return self._create(self.prediction_url, body,
verify=self.verify_prediction)
def reify_resources(args, api, logger):
""" Extracts the properties of the created resources and generates
code to rebuild them
"""
resource_id = get_resource_id(args.resource_id)
if resource_id is None:
sys.exit("Failed to match a valid resource ID. Please, check: %s"
% args.resource_id)
api_calls = RESTChain(api, resource_id, args.add_fields,
logger, args.output_dir)
output = api_calls.reify("python")
if PYTHON3:
with open(args.output, "w", encoding="utf-8") as reify_file:
reify_file.write(output)
else:
with open(args.output, "w") as reify_file:
reify_file.write(output.encode("utf-8"))
def create_projection(self,
pca,
input_data=None,
args=None,
wait_time=3,
retries=10):
"""Creates a new projection.
The pca parameter can be a pca resource or ID
"""
pca_id = None
resource_type = get_resource_type(pca)
if resource_type != PCA_PATH:
raise Exception("A PCA resource id is needed"
" to create a projection. %s found." %
resource_type)
pca_id = get_resource_id(pca)
if pca_id is not None:
check_resource(pca_id,
query_string=TINY_RESOURCE,
wait_time=wait_time,
retries=retries,
raise_on_error=True,
api=self)
if input_data is None:
input_data = {}
create_args = {}
if args is not None:
create_args.update(args)
create_args.update({"input_data": input_data})
if pca_id is not None:
create_args.update({"pca": pca_id})
body = json.dumps(create_args)
return self._create(self.projection_url, body, verify=self.verify)
def reify_resources(args, api):
""" Extracts the properties of the created resources and generates
code to rebuild them
"""
resource_id = get_resource_id(args.resource_id)
if resource_id is None:
sys.exit("Failed to match a valid resource ID. Please, check: %s" %
args.resource_id)
# check whether the resource exists
try:
check_resource(resource_id, raise_on_error=True, api=api)
except Exception:
sys.exit("Failed to find the resource %s. Please, check its ID and"
" the connection info (domain and credentials)." %
resource_id)
reify_script = whizzml_script(args, api)
# apply the reify script to the resource
execute_command = [
'execute', '--script', reify_script, '--output-dir', args.output_dir
]
command_args, _, _, _, _ = get_context( \
execute_command, EXE_SETTINGS)
command_args.arguments_ = [["res-id", resource_id]]
command_args.inputs = json.dumps(command_args.arguments_)
# process the command
session_file = None
execute_whizzml(command_args, api, session_file)
with open("%s.json" % command_args.output) as file_handler:
exe_output = json.load(file_handler)['result']
if args.language == "nb":
write_nb_output(resource_id, \
exe_output, args.output.replace(".py", ".ipynb"), api)
return
elif args.language == "whizzml":
output = exe_output["source_code"]
args.output = args.output.replace(".py", ".whizzml")
exe_output["source_code"] = args.output
exe_output["kind"] = "script"
with open(os.path.join(os.path.dirname(args.output), "metadata.json"),
"w") as meta_handler:
meta_handler.write(json.dumps(exe_output))
else:
output = python_output(exe_output, api)
prefix = u"""\
#!/usr/bin/env python
# -*- coding: utf-8 -*-
\"\"\"Python code to reify %s
Generated by BigMLer
\"\"\"
def main():
""" % resource_id
suffix = u"""\
if __name__ == "__main__":
main()
"""
output = "%s%s\n%s" % (prefix, output, suffix)
write_to_utf8(args.output, output)
sts = os.stat(args.output)
os.chmod(args.output, sts.st_mode | stat.S_IEXEC)
