def delete(api, delete_list): """ Deletes the resources given in the list. """ delete_functions = {bigml.api.SOURCE_RE: api.delete_source, bigml.api.DATASET_RE: api.delete_dataset, bigml.api.MODEL_RE: api.delete_model, bigml.api.PREDICTION_RE: api.delete_prediction, bigml.api.EVALUATION_RE: api.delete_evaluation, bigml.api.ENSEMBLE_RE: api.delete_ensemble, bigml.api.BATCH_PREDICTION_RE: api.delete_batch_prediction, bigml.api.CLUSTER_RE: api.delete_cluster, bigml.api.CENTROID_RE: api.delete_centroid, bigml.api.BATCH_CENTROID_RE: api.delete_centroid} for resource_id in delete_list: resource_type = None try: for resource_type in delete_functions: try: bigml.api.get_resource(resource_type, resource_id) break except ValueError: pass delete_functions[resource_type](resource_id) except ValueError: console_log("Failed to delete resource %s" % resource_id)
def checkpoint(function, *args, **kwargs): """Redirects to each checkpoint function """ common_parms = ['debug', 'message', 'log_file', 'console'] debug = kwargs.get('debug', False) message = kwargs.get('message', None) log_file = kwargs.get('log_file', None) console = kwargs.get('console', False) f_kwargs = { key: value for key, value in kwargs.items() if not key in common_parms } result = function(*args, **f_kwargs) if debug: console_log( "Checkpoint: checking %s with args:\n%s\n\nResult:\n%s\n" % (function.__name__, "\n".join([repr(arg) for arg in args]), repr(result))) # resume is the first element in the result tuple if not result[0] and message is not None: log_message(message, log_file=log_file, console=console) return result
def draw_progress_bar(param, current, total): """Draws a text based progress report. """ pct = 100 - ((total - current) * 100) / (total) console_log("Uploaded %s out of %s bytes [%s%%]" % ( localize(current), localize(total), pct))
def draw_progress_bar(param, current, total): """Draws a text based progress report. """ pct = 100 - ((total - current) * 100) / (total) console_log("Uploaded %s out of %s bytes [%s%%]" % ( localize(current), localize(total), pct), reset=True)
def draw_progress_bar(current, total): """Draws a text based progress report. """ pct = 100 - ((total - current) * 100) / (total) console_log("Predicted on %s out of %s models [%s%%]" % ( localize(current), localize(total), pct))
def draw_progress_bar(current, total): """Draws a text based progress report. """ pct = 100 - ((total - current) * 100) / (total) console_log("Predicted on %s out of %s models [%s%%]" % (localize(current), localize(total), pct))
def checkpoint(function, *args, **kwargs): """Redirects to each checkpoint function """ debug = kwargs.get('debug', False) result = function(*args) if debug: console_log("Checkpoint: checking %s with args:\n%s\n\nResult:\n%s\n" % (function.__name__, "\n".join([repr(arg) for arg in args]), repr(result))) return result
def log_message(message, log_file=None, console=False): """Logs a message in a file and/or to console If log_file is set, logs the message in the file. If console is True, sends the message to console. """ if isinstance(message, unicode): message = message.encode("utf-8") if console: console_log(message) if log_file is not None: with open(log_file, 'a', 0) as log_file: log_file.write(message)
def log_message(message, log_file=None, console=False): """Logs a message in a file and/or to console If log_file is set, logs the message in the file. If console is True, sends the message to console. """ if console: console_log(message) if log_file is not None: if isinstance(message, unicode): message = message.encode("utf-8") with open(log_file, 'a', 0) as log_file: log_file.write(message)
def delete(api, delete_list): """ Deletes the resources given in the list. """ for resource_id in delete_list: resource_type = None try: for resource_type, resource_re in bigml.api.RESOURCE_RE.items(): try: bigml.api.get_resource(resource_re, resource_id) break except ValueError: pass api.deleters[resource_type](resource_id) except ValueError: console_log("Failed to delete resource %s" % resource_id)
def log_message(message, log_file=None, console=False): """Logs a message in a file and/or to console If log_file is set, logs the message in the file. If console is True, sends the message to console. """ if isinstance(message, unicode) and not PYTHON3: message = message.encode(FILE_ENCODING) if console: console_log(message) if log_file is not None: if PYTHON3: message = message.encode(FILE_ENCODING) with open(log_file, 'ab', 0) as log_handler: log_handler.write(message)
def log_message(message, log_file=None, console=False): """Logs a message in a file and/or to console If log_file is set, logs the message in the file. If console is True, sends the message to console. """ if isinstance(message, unicode) and not PYTHON3: message = message.encode(FILE_ENCODING) if console: console_log(message) if log_file is not None: if PYTHON3: message = message.encode(FILE_ENCODING) with open(log_file, 'ab', 0) as log_file: log_file.write(message)
def checkpoint(function, *args, **kwargs): """Redirects to each checkpoint function """ debug = kwargs.get('debug', False) message = kwargs.get('message', None) log_file = kwargs.get('log_file', None) console = kwargs.get('console', False) result = function(*args) if debug: console_log("Checkpoint: checking %s with args:\n%s\n\nResult:\n%s\n" % (function.__name__, "\n".join([repr(arg) for arg in args]), repr(result))) # resume is the first element in the result tuple if not result[0] and message is not None: log_message(message, log_file=log_file, console=console) return result
def read_types(path): """Types to update source fields types. A column number and type separated by a comma per line. For example: 0, 'categorical' 1, 'numeric' """ types_dict = {} for line in fileinput.input([path]): try: pair = ast.literal_eval(line) types_dict.update({pair[0]: {'optype': pair[1]}}) except SyntaxError: console_log("WARNING: The following line in file %s" " does not match the expected" " syntax: %s" % (path, line)) return types_dict
def delete(api, delete_list): """ Deletes the resources given in the list. """ delete_functions = {bigml.api.SOURCE_RE: api.delete_source, bigml.api.DATASET_RE: api.delete_dataset, bigml.api.MODEL_RE: api.delete_model, bigml.api.PREDICTION_RE: api.delete_prediction, bigml.api.EVALUATION_RE: api.delete_evaluation} for resource_id in delete_list: resource_type = None try: for resource_type in delete_functions: try: bigml.api.get_resource(resource_type, resource_id) break except ValueError: pass delete_functions[resource_type](resource_id) except ValueError: console_log("Failed to delete resource %s" % resource_id)
def delete(api, delete_list, exe_outputs=True): """ Deletes the resources given in the list. If the exe_outputs is set, deleting an execution causes the deletion of any outpur resource. """ for resource_id in delete_list: resource_type = None try: for resource_type in bigml.api.RESOURCE_RE: try: bigml.api.get_resource(resource_type, resource_id) break except ValueError: pass if resource_type == "execution" and exe_outputs: query_string = "delete_all=true" api.deleters[resource_type](resource_id, query_string=query_string) else: api.deleters[resource_type](resource_id) except ValueError: console_log("Failed to delete resource %s" % resource_id)
def read_types(path): """Types to update source fields types. A column number and type separated by a comma per line. For example: 0, 'categorical' 1, 'numeric' """ types_dict = {} for line in fileinput.input([path]): try: pair = ast.literal_eval(line) types_dict.update({ pair[0]: {'optype': pair[1]}}) except SyntaxError: console_log("WARNING: The following line in file %s" " does not match the expected" " syntax: %s" % (path, line)) return types_dict
def checkpoint(function, *args, **kwargs): """Redirects to each checkpoint function """ common_parms = ["debug", "message", "log_file", "console"] debug = kwargs.get("debug", False) message = kwargs.get("message", None) log_file = kwargs.get("log_file", None) console = kwargs.get("console", False) f_kwargs = {key: value for key, value in kwargs.items() if not key in common_parms} result = function(*args, **f_kwargs) if debug: console_log( "Checkpoint: checking %s with args:\n%s\n\nResult:\n%s\n" % (function.__name__, "\n".join([repr(arg) for arg in args]), repr(result)) ) # resume is the first element in the result tuple if not result[0] and message is not None: log_message(message, log_file=log_file, console=console) return result
def delete(api, delete_list): """ Deletes the resources given in the list. """ delete_functions = {bigml.api.SOURCE_RE: api.delete_source, bigml.api.DATASET_RE: api.delete_dataset, bigml.api.MODEL_RE: api.delete_model, bigml.api.PREDICTION_RE: api.delete_prediction, bigml.api.EVALUATION_RE: api.delete_evaluation, bigml.api.ENSEMBLE_RE: api.delete_ensemble} for resource_id in delete_list: resource_type = None try: for resource_type in delete_functions: try: bigml.api.get_resource(resource_type, resource_id) break except ValueError: pass delete_functions[resource_type](resource_id) except ValueError: console_log("Failed to delete resource %s" % resource_id)
def read_fields_map(path): """Fields map from evaluated model to test dataset. The the evaluated model field column and the test dataset field column separated by a comma per line. For example: 0, 0 1, 2 2, 1 """ map_dict = {} for line in fileinput.input([path]): try: pair = ast.literal_eval(line) map_dict.update({pair[0]: pair[1]}) except SyntaxError: console_log("WARNING: The following line in file %s" " does not match the expected" " syntax: %s" % (path, line)) return map_dict
def read_fields_map(path): """Fields map from evaluated model to test dataset. The the evaluated model field column and the test dataset field column separated by a comma per line. For example: 0, 0 1, 2 2, 1 """ map_dict = {} for line in fileinput.input([path]): try: pair = ast.literal_eval(line) map_dict.update({ pair[0]: pair[1]}) except SyntaxError: console_log("WARNING: The following line in file %s" " does not match the expected" " syntax: %s" % (path, line)) return map_dict