def delete(api, delete_list):
""" Deletes the resources given in the list.
"""
delete_functions = {bigml.api.SOURCE_RE: api.delete_source,
bigml.api.DATASET_RE: api.delete_dataset,
bigml.api.MODEL_RE: api.delete_model,
bigml.api.PREDICTION_RE: api.delete_prediction,
bigml.api.EVALUATION_RE: api.delete_evaluation,
bigml.api.ENSEMBLE_RE: api.delete_ensemble,
bigml.api.BATCH_PREDICTION_RE:
api.delete_batch_prediction,
bigml.api.CLUSTER_RE: api.delete_cluster,
bigml.api.CENTROID_RE: api.delete_centroid,
bigml.api.BATCH_CENTROID_RE: api.delete_centroid}
for resource_id in delete_list:
resource_type = None
try:
for resource_type in delete_functions:
try:
bigml.api.get_resource(resource_type, resource_id)
break
except ValueError:
pass
delete_functions[resource_type](resource_id)
except ValueError:
console_log("Failed to delete resource %s" % resource_id)
def checkpoint(function, *args, **kwargs):
"""Redirects to each checkpoint function
"""
common_parms = ['debug', 'message', 'log_file', 'console']
debug = kwargs.get('debug', False)
message = kwargs.get('message', None)
log_file = kwargs.get('log_file', None)
console = kwargs.get('console', False)
f_kwargs = {
key: value
for key, value in kwargs.items() if not key in common_parms
}
result = function(*args, **f_kwargs)
if debug:
console_log(
"Checkpoint: checking %s with args:\n%s\n\nResult:\n%s\n" %
(function.__name__, "\n".join([repr(arg)
for arg in args]), repr(result)))
# resume is the first element in the result tuple
if not result[0] and message is not None:
log_message(message, log_file=log_file, console=console)
return result
def draw_progress_bar(param, current, total):
"""Draws a text based progress report.
"""
pct = 100 - ((total - current) * 100) / (total)
console_log("Uploaded %s out of %s bytes [%s%%]" % (
localize(current), localize(total), pct))
def draw_progress_bar(param, current, total):
"""Draws a text based progress report.
"""
pct = 100 - ((total - current) * 100) / (total)
console_log("Uploaded %s out of %s bytes [%s%%]" % (
localize(current), localize(total), pct), reset=True)
def draw_progress_bar(current, total):
"""Draws a text based progress report.
"""
pct = 100 - ((total - current) * 100) / (total)
console_log("Predicted on %s out of %s models [%s%%]" % (
localize(current), localize(total), pct))
def draw_progress_bar(current, total):
"""Draws a text based progress report.
"""
pct = 100 - ((total - current) * 100) / (total)
console_log("Predicted on %s out of %s models [%s%%]" %
(localize(current), localize(total), pct))
def checkpoint(function, *args, **kwargs):
"""Redirects to each checkpoint function
"""
debug = kwargs.get('debug', False)
result = function(*args)
if debug:
console_log("Checkpoint: checking %s with args:\n%s\n\nResult:\n%s\n" %
(function.__name__, "\n".join([repr(arg) for arg in args]),
repr(result)))
return result
def log_message(message, log_file=None, console=False):
"""Logs a message in a file and/or to console
If log_file is set, logs the message in the file.
If console is True, sends the message to console.
"""
if isinstance(message, unicode):
message = message.encode("utf-8")
if console:
console_log(message)
if log_file is not None:
with open(log_file, 'a', 0) as log_file:
log_file.write(message)
def log_message(message, log_file=None, console=False):
"""Logs a message in a file and/or to console
If log_file is set, logs the message in the file.
If console is True, sends the message to console.
"""
if console:
console_log(message)
if log_file is not None:
if isinstance(message, unicode):
message = message.encode("utf-8")
with open(log_file, 'a', 0) as log_file:
log_file.write(message)
def delete(api, delete_list):
""" Deletes the resources given in the list.
"""
for resource_id in delete_list:
resource_type = None
try:
for resource_type, resource_re in bigml.api.RESOURCE_RE.items():
try:
bigml.api.get_resource(resource_re, resource_id)
break
except ValueError:
pass
api.deleters[resource_type](resource_id)
except ValueError:
console_log("Failed to delete resource %s" % resource_id)
def log_message(message, log_file=None, console=False):
"""Logs a message in a file and/or to console
If log_file is set, logs the message in the file.
If console is True, sends the message to console.
"""
if isinstance(message, unicode) and not PYTHON3:
message = message.encode(FILE_ENCODING)
if console:
console_log(message)
if log_file is not None:
if PYTHON3:
message = message.encode(FILE_ENCODING)
with open(log_file, 'ab', 0) as log_handler:
log_handler.write(message)
def delete(api, delete_list):
""" Deletes the resources given in the list.
"""
for resource_id in delete_list:
resource_type = None
try:
for resource_type, resource_re in bigml.api.RESOURCE_RE.items():
try:
bigml.api.get_resource(resource_re, resource_id)
break
except ValueError:
pass
api.deleters[resource_type](resource_id)
except ValueError:
console_log("Failed to delete resource %s" % resource_id)
def log_message(message, log_file=None, console=False):
"""Logs a message in a file and/or to console
If log_file is set, logs the message in the file.
If console is True, sends the message to console.
"""
if isinstance(message, unicode) and not PYTHON3:
message = message.encode(FILE_ENCODING)
if console:
console_log(message)
if log_file is not None:
if PYTHON3:
message = message.encode(FILE_ENCODING)
with open(log_file, 'ab', 0) as log_file:
log_file.write(message)
def checkpoint(function, *args, **kwargs):
"""Redirects to each checkpoint function
"""
debug = kwargs.get('debug', False)
message = kwargs.get('message', None)
log_file = kwargs.get('log_file', None)
console = kwargs.get('console', False)
result = function(*args)
if debug:
console_log("Checkpoint: checking %s with args:\n%s\n\nResult:\n%s\n" %
(function.__name__, "\n".join([repr(arg) for arg in args]),
repr(result)))
# resume is the first element in the result tuple
if not result[0] and message is not None:
log_message(message, log_file=log_file, console=console)
return result
def read_types(path):
"""Types to update source fields types.
A column number and type separated by a comma per line.
For example:
0, 'categorical'
1, 'numeric'
"""
types_dict = {}
for line in fileinput.input([path]):
try:
pair = ast.literal_eval(line)
types_dict.update({pair[0]: {'optype': pair[1]}})
except SyntaxError:
console_log("WARNING: The following line in file %s"
" does not match the expected"
" syntax: %s" % (path, line))
return types_dict
def delete(api, delete_list):
""" Deletes the resources given in the list.
"""
delete_functions = {bigml.api.SOURCE_RE: api.delete_source,
bigml.api.DATASET_RE: api.delete_dataset,
bigml.api.MODEL_RE: api.delete_model,
bigml.api.PREDICTION_RE: api.delete_prediction,
bigml.api.EVALUATION_RE: api.delete_evaluation}
for resource_id in delete_list:
resource_type = None
try:
for resource_type in delete_functions:
try:
bigml.api.get_resource(resource_type, resource_id)
break
except ValueError:
pass
delete_functions[resource_type](resource_id)
except ValueError:
console_log("Failed to delete resource %s" % resource_id)
def delete(api, delete_list, exe_outputs=True):
""" Deletes the resources given in the list. If the exe_outputs is set,
deleting an execution causes the deletion of any outpur resource.
"""
for resource_id in delete_list:
resource_type = None
try:
for resource_type in bigml.api.RESOURCE_RE:
try:
bigml.api.get_resource(resource_type, resource_id)
break
except ValueError:
pass
if resource_type == "execution" and exe_outputs:
query_string = "delete_all=true"
api.deleters[resource_type](resource_id,
query_string=query_string)
else:
api.deleters[resource_type](resource_id)
except ValueError:
console_log("Failed to delete resource %s" % resource_id)
def delete(api, delete_list, exe_outputs=True):
""" Deletes the resources given in the list. If the exe_outputs is set,
deleting an execution causes the deletion of any outpur resource.
"""
for resource_id in delete_list:
resource_type = None
try:
for resource_type in bigml.api.RESOURCE_RE:
try:
bigml.api.get_resource(resource_type, resource_id)
break
except ValueError:
pass
if resource_type == "execution" and exe_outputs:
query_string = "delete_all=true"
api.deleters[resource_type](resource_id,
query_string=query_string)
else:
api.deleters[resource_type](resource_id)
except ValueError:
console_log("Failed to delete resource %s" % resource_id)
def read_types(path):
"""Types to update source fields types.
A column number and type separated by a comma per line.
For example:
0, 'categorical'
1, 'numeric'
"""
types_dict = {}
for line in fileinput.input([path]):
try:
pair = ast.literal_eval(line)
types_dict.update({
pair[0]: {'optype': pair[1]}})
except SyntaxError:
console_log("WARNING: The following line in file %s"
" does not match the expected"
" syntax: %s" % (path, line))
return types_dict
def checkpoint(function, *args, **kwargs):
"""Redirects to each checkpoint function
"""
common_parms = ["debug", "message", "log_file", "console"]
debug = kwargs.get("debug", False)
message = kwargs.get("message", None)
log_file = kwargs.get("log_file", None)
console = kwargs.get("console", False)
f_kwargs = {key: value for key, value in kwargs.items() if not key in common_parms}
result = function(*args, **f_kwargs)
if debug:
console_log(
"Checkpoint: checking %s with args:\n%s\n\nResult:\n%s\n"
% (function.__name__, "\n".join([repr(arg) for arg in args]), repr(result))
)
# resume is the first element in the result tuple
if not result[0] and message is not None:
log_message(message, log_file=log_file, console=console)
return result
def delete(api, delete_list):
""" Deletes the resources given in the list.
"""
delete_functions = {bigml.api.SOURCE_RE: api.delete_source,
bigml.api.DATASET_RE: api.delete_dataset,
bigml.api.MODEL_RE: api.delete_model,
bigml.api.PREDICTION_RE: api.delete_prediction,
bigml.api.EVALUATION_RE: api.delete_evaluation,
bigml.api.ENSEMBLE_RE: api.delete_ensemble}
for resource_id in delete_list:
resource_type = None
try:
for resource_type in delete_functions:
try:
bigml.api.get_resource(resource_type, resource_id)
break
except ValueError:
pass
delete_functions[resource_type](resource_id)
except ValueError:
console_log("Failed to delete resource %s" % resource_id)
def read_fields_map(path):
"""Fields map from evaluated model to test dataset.
The the evaluated model field column and the test dataset field column
separated by a comma per line.
For example:
0, 0
1, 2
2, 1
"""
map_dict = {}
for line in fileinput.input([path]):
try:
pair = ast.literal_eval(line)
map_dict.update({pair[0]: pair[1]})
except SyntaxError:
console_log("WARNING: The following line in file %s"
" does not match the expected"
" syntax: %s" % (path, line))
return map_dict
def read_fields_map(path):
"""Fields map from evaluated model to test dataset.
The the evaluated model field column and the test dataset field column
separated by a comma per line.
For example:
0, 0
1, 2
2, 1
"""
map_dict = {}
for line in fileinput.input([path]):
try:
pair = ast.literal_eval(line)
map_dict.update({
pair[0]: pair[1]})
except SyntaxError:
console_log("WARNING: The following line in file %s"
" does not match the expected"
" syntax: %s" % (path, line))
return map_dict
