def remote_forecast(time_series,
forecast_args, args,
api, resume, prediction_file=None, session_file=None,
path=None, log=None):
"""Computes a remote forecast.
"""
time_series_id = bigml.api.get_time_series_id( \
time_series)
# if resuming, try to extract dataset form log files
if resume:
message = u.dated("Forecast not found. Resuming.\n")
resume, forecast = c.checkpoint(
c.is_forecast_created, path, debug=args.debug,
message=message, log_file=session_file, console=args.verbosity)
if not resume:
local_time_series = TimeSeries(time_series,
api=args.retrieve_api_)
output = args.predictions
if args.test_set is not None:
input_data = u.read_json(args.test_set)
elif args.horizon is not None:
input_data = {local_time_series.objective_id: { \
"horizon": args.horizon}}
forecast = create_forecast(
time_series_id, input_data, forecast_args,
args, api, session_file=session_file, path=path, log=log)
write_forecasts(forecast["object"]["forecast"]["result"], output)
def attribute_args(command_args):
"""Reads the attributes in JSON files
"""
# Parses attributes in json format if provided
command_args.json_args = {}
for resource_type in RESOURCE_TYPES:
attributes_file = getattr(command_args,
"%s_attributes" % resource_type, None)
if attributes_file is not None:
command_args.json_args[resource_type] = u.read_json(
attributes_file)
else:
command_args.json_args[resource_type] = {}
def attribute_args(command_args):
"""Reads the attributes in JSON files
"""
# Parses attributes in json format if provided
command_args.json_args = {}
for resource_type in RESOURCE_TYPES:
attributes_file = getattr(command_args,
"%s_attributes" % resource_type, None)
if attributes_file is not None:
command_args.json_args[resource_type] = u.read_json(
attributes_file)
else:
command_args.json_args[resource_type] = {}
def forecast(time_series, args, session_file=None):
"""Computes a time-series forecast
"""
local_time_series = TimeSeries(time_series, api=args.retrieve_api_)
output = args.predictions
# Local forecasts: Forecasts are computed locally
message = u.dated("Creating local forecasts.\n")
u.log_message(message, log_file=session_file, console=args.verbosity)
input_data = []
if args.test_set is not None:
input_data = [u.read_json(args.test_set)]
elif args.horizon is not None:
input_data = [{local_time_series.objective_id: { \
"horizon": args.horizon}}]
write_forecasts(local_time_series.forecast(*input_data), output)
def remote_forecast(time_series,
forecast_args,
args,
api,
resume,
prediction_file=None,
session_file=None,
path=None,
log=None):
"""Computes a remote forecast.
"""
time_series_id = bigml.api.get_time_series_id( \
time_series)
# if resuming, try to extract dataset form log files
if resume:
message = u.dated("Forecast not found. Resuming.\n")
resume, forecast = c.checkpoint(c.is_forecast_created,
path,
debug=args.debug,
message=message,
log_file=session_file,
console=args.verbosity)
if not resume:
local_time_series = TimeSeries(time_series, api=args.retrieve_api_)
output = args.predictions
if args.test_set is not None:
input_data = u.read_json(args.test_set)
elif args.horizon is not None:
input_data = {local_time_series.objective_id: { \
"horizon": args.horizon}}
forecast = create_forecast(time_series_id,
input_data,
forecast_args,
args,
api,
session_file=session_file,
path=path,
log=log)
write_forecasts(forecast["object"]["forecast"]["result"], output)
def forecast(time_series, args, session_file=None):
"""Computes a time-series forecast
"""
local_time_series = TimeSeries(time_series,
api=args.retrieve_api_)
output = args.predictions
# Local forecasts: Forecasts are computed locally
message = u.dated("Creating local forecasts.\n")
u.log_message(message, log_file=session_file, console=args.verbosity)
input_data = []
if args.test_set is not None:
input_data = [u.read_json(args.test_set)]
elif args.horizon is not None:
input_data = [{local_time_series.objective_id: { \
"horizon": args.horizon}}]
write_forecasts(local_time_series.forecast(*input_data),
output)
def transform_args(command_args, flags, api, user_defaults):
"""Transforms the formatted argument strings into structured arguments
"""
# Parses attributes in json format if provided
command_args.json_args = {}
for resource_type in RESOURCE_TYPES:
attributes_file = getattr(command_args,
"%s_attributes" % resource_type, None)
if attributes_file is not None:
command_args.json_args[resource_type] = u.read_json(
attributes_file)
else:
command_args.json_args[resource_type] = {}
# Parses dataset generators in json format if provided
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
# Parses multi-dataset attributes in json such as field maps
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
dataset_ids = None
command_args.dataset_ids = []
# Parses dataset/id if provided.
if command_args.datasets:
dataset_ids = u.read_datasets(command_args.datasets)
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Reading test dataset ids is delayed till the very moment of use to ensure
# that the newly generated resources files can be used there too
command_args.test_dataset_ids = []
# Retrieve dataset/ids if provided.
if command_args.dataset_tag:
dataset_ids = dataset_ids.extend(
u.list_ids(api.list_datasets,
"tags__in=%s" % command_args.dataset_tag))
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Reads a json filter if provided.
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
# Reads a lisp filter if provided.
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
# Adds default tags unless that it is requested not to do so.
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
# Checks combined votes method
try:
if (command_args.method and command_args.method != COMBINATION_LABEL
and not (command_args.method in COMBINATION_WEIGHTS.keys())):
command_args.method = 0
else:
combiner_methods = dict([[value, key]
for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
except AttributeError:
pass
# Checks missing_strategy
try:
if (command_args.missing_strategy
and not (command_args.missing_strategy
in MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
except AttributeError:
pass
# Adds replacement=True if creating ensemble and nothing is specified
try:
if (command_args.number_of_models > 1 and not command_args.replacement
and not '--no-replacement' in flags
and not 'replacement' in user_defaults
and not '--no-randomize' in flags
and not 'randomize' in user_defaults
and not '--sample-rate' in flags
and not 'sample_rate' in user_defaults):
command_args.replacement = True
except AttributeError:
pass
try:
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print("WARNING: 'full data' is a deprecated value. Use"
" 'full' instead")
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
# Parses class, weight pairs for objective weight
try:
if command_args.objective_weights:
objective_weights = (u.read_objective_weights(
command_args.objective_weights))
command_args.objective_weights_json = objective_weights
except AttributeError:
pass
try:
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(
command_args.args_separator)
except AttributeError:
pass
# Sets shared_flag if --shared or --unshared has been used
if '--shared' in flags or '--unshared' in flags:
command_args.shared_flag = True
else:
command_args.shared_flag = False
# Set remote on if scoring a trainind dataset in bigmler anomaly
try:
if command_args.score:
command_args.remote = True
if not "--prediction-info" in flags:
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
command_args.has_models_ = (
(hasattr(command_args, 'model') and command_args.model)
or (hasattr(command_args, 'models') and command_args.models)
or (hasattr(command_args, 'ensemble') and command_args.ensemble)
or (hasattr(command_args, 'ensembles') and command_args.ensembles)
or (hasattr(command_args, 'cluster') and command_args.cluster)
or (hasattr(command_args, 'clusters') and command_args.clusters)
or (hasattr(command_args, 'model_tag') and command_args.model_tag)
or (hasattr(command_args, 'anomaly') and command_args.anomaly)
or (hasattr(command_args, 'anomalies') and command_args.anomalies) or
(hasattr(command_args, 'ensemble_tag') and command_args.ensemble_tag)
or (hasattr(command_args, 'cluster_tag') and command_args.cluster_tag)
or (hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag))
command_args.has_datasets_ = (
(hasattr(command_args, 'dataset') and command_args.dataset)
or (hasattr(command_args, 'datasets') and command_args.datasets)
or (hasattr(command_args, 'dataset_tag') and command_args.dataset_tag))
command_args.has_test_datasets_ = (
(hasattr(command_args, 'test_dataset') and command_args.test_dataset)
or
(hasattr(command_args, 'test_datasets') and command_args.test_datasets)
or (hasattr(command_args, 'test_dataset_tag')
and command_args.test_dataset_tag))
def get_output_args(api, command_args, resume):
"""Returns the output args needed for the main bigmler computation process
"""
try:
if command_args.train_stdin:
if command_args.test_stdin:
sys.exit("The standard input can't be used both for training "
"and testing. Choose one of them")
command_args.training_set = StringIO(sys.stdin.read())
elif command_args.test_stdin:
command_args.test_set = StringIO(sys.stdin.read())
except AttributeError:
pass
try:
if command_args.objective_field:
objective = command_args.objective_field
try:
command_args.objective_field = int(objective)
except ValueError:
if not command_args.train_header:
sys.exit("The %s has been set as objective field but"
" the file has not been marked as containing"
" headers.\nPlease set the --train-header flag if"
" the file has headers or use a column number"
" to set the objective field." % objective)
except AttributeError:
pass
command_args.resume_ = resume
command_args.predictions = command_args.output
command_args.projections = command_args.output
# Reads description if provided.
try:
if command_args.description:
description_arg = u.read_description(command_args.description)
command_args.description_ = description_arg
else:
command_args.description_ = DEFAULT_DESCRIPTION
except AttributeError:
pass
# Parses fields if provided.
try:
if command_args.field_attributes:
field_attributes_arg = (u.read_field_attributes(
command_args.field_attributes))
command_args.field_attributes_ = field_attributes_arg
else:
command_args.field_attributes_ = []
except AttributeError:
pass
try:
if command_args.test_field_attributes:
field_attributes_arg = (u.read_field_attributes(
command_args.test_field_attributes))
command_args.test_field_attributes_ = field_attributes_arg
else:
command_args.test_field_attributes_ = []
except AttributeError:
pass
# Parses types if provided.
try:
if command_args.types:
types_arg = u.read_types(command_args.types)
command_args.types_ = types_arg
else:
command_args.types_ = None
if command_args.test_types:
types_arg = u.read_types(command_args.test_types)
command_args.test_types_ = types_arg
else:
command_args.test_types_ = None
except AttributeError:
pass
# Parses dataset fields if provided.
try:
if command_args.dataset_fields:
dataset_fields_arg = [
field.strip() for field in command_args.dataset_fields.split(
command_args.args_separator)
]
command_args.dataset_fields_ = dataset_fields_arg
else:
command_args.dataset_fields_ = []
except AttributeError:
pass
# Parses model input fields if provided.
try:
if command_args.model_fields:
model_fields_arg = [
field.strip() for field in command_args.model_fields.split(
command_args.args_separator)
]
command_args.model_fields_ = model_fields_arg
else:
command_args.model_fields_ = []
except AttributeError:
pass
# Parses cluster input fields if provided.
try:
if command_args.cluster_fields:
cluster_fields_arg = [
field.strip() for field in command_args.cluster_fields.split(
command_args.args_separator)
]
command_args.cluster_fields_ = cluster_fields_arg
else:
command_args.cluster_fields_ = []
except AttributeError:
pass
# Parses association input fields if provided.
try:
if command_args.association_fields:
association_fields_arg = [
field.strip() for field in \
command_args.association_fields.split( \
command_args.args_separator)]
command_args.association_fields_ = association_fields_arg
else:
command_args.association_fields_ = []
except AttributeError:
pass
# Parses anomaly input fields if provided.
try:
if command_args.anomaly_fields:
anomaly_fields_arg = [
field.strip() for field in command_args.anomaly_fields.split(
command_args.args_separator)
]
command_args.anomaly_fields_ = anomaly_fields_arg
else:
command_args.anomaly_fields_ = []
except AttributeError:
pass
# Parses logistic regression input fields if provided.
try:
if command_args.logistic_fields:
logistic_fields_arg = [
field.strip() for field in command_args.logistic_fields.split(
command_args.args_separator)
]
command_args.logistic_fields_ = logistic_fields_arg
else:
command_args.logistic_fields_ = []
except AttributeError:
pass
# Parses linear regression input fields if provided.
try:
if command_args.linear_fields:
linear_fields_arg = [
field.strip() for field in command_args.linear_fields.split(
command_args.args_separator)
]
command_args.linear_fields_ = linear_fields_arg
else:
command_args.linear_fields_ = []
except AttributeError:
pass
# Parses deepnet input fields if provided.
try:
if command_args.deepnet_fields:
deepnet_fields_arg = [
field.strip() for field in command_args.deepnet_fields.split(
command_args.args_separator)
]
command_args.deepnet_fields_ = deepnet_fields_arg
else:
command_args.deepnet_fields_ = []
except AttributeError:
pass
# Parses topic model fields if provided.
try:
if command_args.topic_fields:
topic_fields_arg = [
field.strip() for field in command_args.topic_fields.split(
command_args.args_separator)
]
command_args.topic_model_fields_ = topic_fields_arg
else:
command_args.topic_model_fields_ = []
except AttributeError:
pass
# Parses pca fields if provided.
try:
if command_args.pca_fields:
pca_fields_arg = [
field.strip() for field in command_args.pca_fields.split(
command_args.args_separator)
]
command_args.pca_fields_ = pca_fields_arg
else:
command_args.pca_fields_ = []
except AttributeError:
pass
# Parses field_codings for deepnet
try:
if command_args.field_codings:
command_args.field_codings_ = u.read_json(
command_args.field_codings)
else:
command_args.field_codings_ = []
except AttributeError:
pass
# Parses imports for scripts and libraries.
try:
if command_args.imports:
imports_arg = [
field.strip() for field in command_args.imports.split(
command_args.args_separator)
]
command_args.imports_ = imports_arg
else:
command_args.imports_ = []
except AttributeError:
pass
# Parses objective fields for time-series.
try:
if command_args.objectives:
objective_fields_arg = [
field.strip() for field in command_args.objectives.split(
command_args.args_separator)
]
command_args.objective_fields_ = objective_fields_arg
else:
command_args.objective_fields_ = []
except AttributeError:
pass
# Parses range.
try:
if command_args.range:
range_arg = [
value.strip() for value in command_args.range.split(
command_args.args_separator)
]
command_args.range_ = range_arg
else:
command_args.range_ = []
except AttributeError:
pass
# Parses parameters for scripts.
try:
if command_args.declare_inputs:
command_args.parameters_ = u.read_json(command_args.declare_inputs)
else:
command_args.parameters_ = []
except AttributeError:
pass
# Parses creation_defaults for executions.
try:
if command_args.creation_defaults:
command_args.creation_defaults_ = u.read_json(
command_args.creation_defaults)
else:
command_args.creation_defaults_ = {}
except AttributeError:
pass
# Parses arguments for executions.
try:
if command_args.inputs:
command_args.arguments_ = u.read_json(command_args.inputs)
else:
command_args.arguments_ = []
except AttributeError:
pass
# Parses input maps for executions.
try:
if command_args.input_maps:
command_args.input_maps_ = u.read_json(command_args.input_maps)
else:
command_args.input_maps_ = []
except AttributeError:
pass
# Parses outputs for executions.
try:
if command_args.outputs:
command_args.outputs_ = u.read_json(command_args.outputs)
else:
command_args.outputs_ = []
except AttributeError:
pass
# Parses outputs for scripts.
try:
if command_args.declare_outputs:
command_args.declare_outputs_ = \
u.read_json(command_args.declare_outputs)
else:
command_args.declare_outputs_ = []
except AttributeError:
pass
model_ids = []
try:
# Parses model/ids if provided.
if command_args.models:
model_ids = u.read_resources(command_args.models)
command_args.model_ids_ = model_ids
except AttributeError:
pass
# Retrieve model/ids if provided.
try:
if command_args.model_tag:
model_ids = (model_ids + u.list_ids(
api.list_models, "tags__in=%s" % command_args.model_tag))
command_args.model_ids_ = model_ids
except AttributeError:
pass
# Reads votes files in the provided directories.
try:
if command_args.votes_dirs:
dirs = [
directory.strip() for directory in
command_args.votes_dirs.split(command_args.args_separator)
]
votes_path = os.path.dirname(command_args.predictions)
votes_files = u.read_votes_files(dirs, votes_path)
command_args.votes_files_ = votes_files
else:
command_args.votes_files_ = []
except AttributeError:
pass
# Parses fields map if provided.
try:
if command_args.fields_map:
fields_map_arg = u.read_fields_map(command_args.fields_map)
command_args.fields_map_ = fields_map_arg
else:
command_args.fields_map_ = None
except AttributeError:
pass
cluster_ids = []
try:
# Parses cluster/ids if provided.
if command_args.clusters:
cluster_ids = u.read_resources(command_args.clusters)
command_args.cluster_ids_ = cluster_ids
except AttributeError:
pass
# Retrieve cluster/ids if provided.
try:
if command_args.cluster_tag:
cluster_ids = (cluster_ids + u.list_ids(
api.list_clusters, "tags__in=%s" % command_args.cluster_tag))
command_args.cluster_ids_ = cluster_ids
except AttributeError:
pass
association_ids = []
try:
# Parses association/ids if provided.
if command_args.associations:
association_ids = u.read_resources(command_args.associations)
command_args.association_ids_ = association_ids
except AttributeError:
pass
# Retrieve association/ids if provided.
try:
if command_args.association_tag:
association_ids = (
association_ids +
u.list_ids(api.list_associations,
"tags__in=%s" % command_args.association_tag))
command_args.association_ids_ = association_ids
except AttributeError:
pass
logistic_regression_ids = []
try:
# Parses logisticregression/ids if provided.
if command_args.logistic_regressions:
logistic_regression_ids = u.read_resources( \
command_args.logistic_regressions)
command_args.logistic_regression_ids_ = logistic_regression_ids
except AttributeError:
pass
# Retrieve logsticregression/ids if provided.
try:
if command_args.logistic_regression_tag:
logistic_regression_ids = (logistic_regression_ids + \
u.list_ids(api.list_logistic_regressions,
"tags__in=%s" %
command_args.logistic_regression_tag))
command_args.logistic_regression_ids_ = logistic_regression_ids
except AttributeError:
pass
linear_regression_ids = []
try:
# Parses linearregression/ids if provided.
if command_args.linear_regressions:
linear_regression_ids = u.read_resources( \
command_args.linear_regressions)
command_args.linear_regression_ids_ = linear_regression_ids
except AttributeError:
pass
# Retrieve linearregression/ids if provided.
try:
if command_args.linear_regression_tag:
linear_regression_ids = (linear_regression_ids + \
u.list_ids(api.list_linear_regressions,
"tags__in=%s" %
command_args.linear_regression_tag))
command_args.linear_regression_ids_ = linear_regression_ids
except AttributeError:
pass
deepnet_ids = []
try:
# Parses deepnet/ids if provided.
if command_args.deepnets:
deepnet_ids = u.read_resources( \
command_args.deepnets)
command_args.deepnet_ids_ = deepnet_ids
except AttributeError:
pass
# Retrieve deepnet/ids if provided.
try:
if command_args.deepnet_tag:
deepnet_ids = (deepnet_ids + \
u.list_ids(api.list_deepnets,
"tags__in=%s" % command_args.deepnet_tag))
command_args.deepnet_ids_ = deepnet_ids
except AttributeError:
pass
topic_model_ids = []
try:
# Parses topicmodel/ids if provided.
if command_args.topic_models:
topic_model_ids = u.read_resources(command_args.topic_models)
command_args.topic_model_ids_ = topic_model_ids
except AttributeError:
pass
# Retrieve topicmodel/ids if provided.
try:
if command_args.topic_model_tag:
topic_model_ids = (
topic_model_ids +
u.list_ids(api.list_topic_models,
"tags__in=%s" % command_args.topic_model_tag))
command_args.topic_model_ids_ = topic_model_ids
except AttributeError:
pass
time_series_ids = []
try:
# Parses timeseries/ids if provided.
if command_args.time_series_set:
time_series_ids = u.read_resources(command_args.time_series)
command_args.time_series_ids_ = time_series_ids
except AttributeError:
pass
# Retrieve timeseries/ids if provided.
try:
if command_args.time_series_tag:
time_series_ids = (
time_series_ids +
u.list_ids(api.list_time_series,
"tags__in=%s" % command_args.time_series_tag))
command_args.time_series_ids_ = time_series_ids
except AttributeError:
pass
pca_ids = []
try:
# Parses pca/ids if provided.
if command_args.pcas:
pca_ids = u.read_resources(command_args.pcas)
command_args.pca_ids_ = pca_ids
except AttributeError:
pass
# Retrieve pca/ids if provided.
try:
if command_args.pca_tag:
pca_ids = (pca_ids + u.list_ids(
api.pca_series, "tags__in=%s" % command_args.pca_tag))
command_args.pca_ids_ = pca_ids
except AttributeError:
pass
# Parses models list for fusions if provided.
try:
if command_args.fusion_models:
fusion_models_arg = [
model.strip() for model in command_args.fusion_models.split(
command_args.args_separator)
]
command_args.fusion_models_ = fusion_models_arg
else:
command_args.fusion_models_ = []
except AttributeError:
pass
# Parses models list for fusions if provided.
if not has_value(command_args, "fusion_models_"):
try:
if command_args.fusion_models_file:
fusion_models_arg = u.read_json(
command_args.fusion_models_file)
command_args.fusion_models_ = fusion_models_arg
else:
command_args.fusion_models_ = []
except AttributeError:
pass
fusion_ids = []
try:
# Parses fusion/ids if provided.
if command_args.fusions:
fusion_ids = u.read_resources(command_args.fusions)
command_args.fusion_ids_ = fusion_ids
except AttributeError:
pass
# Retrieve fusion/ids if provided.
try:
if command_args.fusion_tag:
fusion_ids = (fusion_ids + u.list_ids(
api.fusion_series, "tags__in=%s" % command_args.fusion_tag))
command_args.fusion_ids_ = fusion_ids
except AttributeError:
pass
# Parses cluster names to generate datasets if provided
try:
if command_args.cluster_datasets:
cluster_datasets_arg = [
dataset.strip()
for dataset in command_args.cluster_datasets.split(
command_args.args_separator)
]
command_args.cluster_datasets_ = cluster_datasets_arg
else:
command_args.cluster_datasets_ = []
except AttributeError:
pass
# Parses cluster names to generate models if provided
try:
if command_args.cluster_models:
cluster_models_arg = [
model.strip() for model in command_args.cluster_models.split(
command_args.args_separator)
]
command_args.cluster_models_ = cluster_models_arg
else:
command_args.cluster_models_ = []
except AttributeError:
pass
# Parses summary_fields to exclude from the clustering algorithm
try:
if command_args.summary_fields:
summary_fields_arg = [
field.strip() for field in command_args.summary_fields.split(
command_args.args_separator)
]
command_args.summary_fields_ = summary_fields_arg
else:
command_args.summary_fields_ = []
except AttributeError:
pass
anomaly_ids = []
try:
# Parses anomaly/ids if provided.
if command_args.anomalies:
anomaly_ids = u.read_resources(command_args.anomalies)
command_args.anomaly_ids_ = anomaly_ids
except AttributeError:
pass
# Retrieve anomaly/ids if provided.
try:
if command_args.anomaly_tag:
anomaly_ids = (anomaly_ids + u.list_ids(
api.list_anomalies, "tags__in=%s" % command_args.anomaly_tag))
command_args.anomaly_ids_ = anomaly_ids
except AttributeError:
pass
sample_ids = []
try:
# Parses sample/ids if provided.
if command_args.samples:
sample_ids = u.read_resources(command_args.samples)
command_args.sample_ids_ = sample_ids
except AttributeError:
pass
# Retrieve sample/ids if provided.
try:
if command_args.sample_tag:
sample_ids = (sample_ids + u.list_ids(
api.list_samples, "tags__in=%s" % command_args.sample_tag))
command_args.sample_ids_ = sample_ids
except AttributeError:
pass
# Parses sample row fields
try:
if command_args.row_fields:
row_fields_arg = [
field.strip() for field in command_args.row_fields.split(
command_args.args_separator)
]
command_args.row_fields_ = row_fields_arg
else:
command_args.row_fields_ = []
except AttributeError:
pass
# Parses sample stat_fields
try:
if command_args.stat_fields:
stat_fields_arg = [
field.strip() for field in command_args.stat_fields.split(
command_args.args_separator)
]
command_args.stat_fields_ = stat_fields_arg
else:
command_args.stat_fields_ = []
except AttributeError:
pass
# if boosting arguments are used, set on boosting
try:
if command_args.iterations or command_args.learning_rate \
or command_args.early_holdout:
command_args.boosting = True
except AttributeError:
pass
# Extracts the imports from the JSON metadata file
try:
if command_args.embedded_imports:
command_args.embedded_imports_ = u.read_resources( \
command_args.embedded_imports)
else:
command_args.embedded_imports_ = []
except AttributeError:
pass
# Parses hidden_layers for deepnets.
try:
if command_args.hidden_layers:
command_args.hidden_layers_ = u.read_json(
command_args.hidden_layers)
else:
command_args.hidden_layers_ = []
except AttributeError:
pass
# Parses operating_point for predictions.
try:
if command_args.operating_point:
command_args.operating_point_ = u.read_json(
command_args.operating_point)
else:
command_args.operating_point_ = []
except AttributeError:
pass
# Parses the json_query
try:
if command_args.json_query:
command_args.json_query_ = u.read_json(command_args.json_query)
else:
command_args.json_query_ = None
except AttributeError:
pass
# Parses the models_file
try:
if command_args.models_file:
command_args.models_file_ = u.read_json(command_args.models_file)
else:
command_args.models_file_ = None
except AttributeError:
pass
# Parses the sql_output_fields
try:
if command_args.sql_output_fields:
command_args.sql_output_fields_ = u.read_json( \
command_args.sql_output_fields)
else:
command_args.sql_output_fields_ = None
except AttributeError:
pass
# Parses connection info for external connectors
try:
if command_args.connection_json:
command_args.connection_json_ = u.read_json(
command_args.connection_json)
else:
command_args.connection_json_ = {}
except AttributeError:
pass
return {"api": api, "args": command_args}
def transform_args(command_args, flags, api):
"""Transforms the formatted argument strings into structured arguments
"""
attribute_args(command_args)
# Parses dataset generators in json format if provided
try:
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
except AttributeError:
pass
# Parses multi-dataset attributes in json such as field maps
try:
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(
command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
except AttributeError:
pass
transform_dataset_options(command_args, api)
script_ids = None
command_args.script_ids = []
# Parses script/id if provided.
try:
if command_args.scripts:
script_ids = u.read_resources(command_args.scripts)
if len(script_ids) == 1:
command_args.script = script_ids[0]
command_args.script_ids = script_ids
except AttributeError:
pass
# Retrieve script/ids if provided.
try:
if command_args.script_tag:
script_ids = script_ids.extend(
u.list_ids(api.list_scripts,
"tags__in=%s" % command_args.script_tag))
if len(script_ids) == 1:
command_args.script = script_ids[0]
command_args.script_ids = script_ids
except AttributeError:
pass
# Reads a json filter if provided.
try:
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
except AttributeError:
pass
# Reads a lisp filter if provided.
try:
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
except AttributeError:
pass
# Adds default tags unless that it is requested not to do so.
try:
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
except AttributeError:
pass
# Checks combined votes method
try:
if (command_args.method and command_args.method != COMBINATION_LABEL
and not command_args.method in COMBINATION_WEIGHTS.keys()):
command_args.method = 0
else:
combiner_methods = dict([[value, key]
for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
except AttributeError:
pass
# Checks missing_strategy
try:
if (command_args.missing_strategy
and not (command_args.missing_strategy
in MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
except AttributeError:
pass
try:
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print("WARNING: 'full data' is a deprecated value. Use"
" 'full' instead")
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
# Parses class, weight pairs for objective weight
try:
if command_args.objective_weights:
objective_weights = (u.read_objective_weights(
command_args.objective_weights))
command_args.objective_weights_json = objective_weights
except AttributeError:
pass
try:
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(
command_args.args_separator)
except AttributeError:
pass
# Sets shared_flag if --shared or --unshared has been used
command_args.shared_flag = '--shared' in flags or '--unshared' in flags
# Set remote on if scoring a trainind dataset in bigmler anomaly
try:
if command_args.score:
command_args.remote = True
if not "--prediction-info" in flags:
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
command_args.has_supervised_ = (
(hasattr(command_args, 'model') and command_args.model)
or (hasattr(command_args, 'models') and command_args.models)
or (hasattr(command_args, 'ensemble') and command_args.ensemble)
or (hasattr(command_args, 'ensembles') and command_args.ensembles)
or (hasattr(command_args, 'model_tag') and command_args.model_tag)
or (hasattr(command_args, 'logistic_regression')
and command_args.logistic_regression)
or (hasattr(command_args, 'logistic_regressions')
and command_args.logistic_regressions)
or (hasattr(command_args, 'logistic_regression_tag')
and command_args.logistic_regression_tag)
or (hasattr(command_args, 'deepnet') and command_args.deepnet)
or (hasattr(command_args, 'deepnets') and command_args.deepnets) or
(hasattr(command_args, 'deepnet_tag') and command_args.deepnet_tag) or
(hasattr(command_args, 'ensemble_tag') and command_args.ensemble_tag))
command_args.has_models_ = (
command_args.has_supervised_
or (hasattr(command_args, 'cluster') and command_args.cluster)
or (hasattr(command_args, 'clusters') and command_args.clusters)
or (hasattr(command_args, 'anomaly') and command_args.anomaly)
or (hasattr(command_args, 'anomalies') and command_args.anomalies)
or (hasattr(command_args, 'cluster_tag') and command_args.cluster_tag)
or (hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag))
command_args.has_datasets_ = (
(hasattr(command_args, 'dataset') and command_args.dataset)
or (hasattr(command_args, 'datasets') and command_args.datasets)
or (hasattr(command_args, 'dataset_ids') and command_args.dataset_ids)
or (hasattr(command_args, 'dataset_tag') and command_args.dataset_tag))
command_args.has_test_datasets_ = (
(hasattr(command_args, 'test_dataset') and command_args.test_dataset)
or
(hasattr(command_args, 'test_datasets') and command_args.test_datasets)
or (hasattr(command_args, 'test_dataset_tag')
and command_args.test_dataset_tag))
command_args.new_dataset = (
(hasattr(command_args, 'datasets_json') and command_args.datasets_json)
or
(hasattr(command_args, 'multi_dataset') and command_args.multi_dataset)
or (hasattr(command_args, 'juxtapose') and command_args.juxtapose)
or (hasattr(command_args, 'sql_query') and command_args.sql_query)
or (hasattr(command_args, 'sql_output_fields')
and command_args.sql_output_fields)
or (hasattr(command_args, 'json_query') and command_args.json_query))
def transform_args(command_args, flags, api, user_defaults):
"""Transforms the formatted argument strings into structured arguments
"""
# Parses attributes in json format if provided
command_args.json_args = {}
json_attribute_options = {
'source': command_args.source_attributes,
'dataset': command_args.dataset_attributes,
'model': command_args.model_attributes,
'ensemble': command_args.ensemble_attributes,
'evaluation': command_args.evaluation_attributes,
'batch_prediction': command_args.batch_prediction_attributes}
for resource_type, attributes_file in json_attribute_options.items():
if attributes_file is not None:
command_args.json_args[resource_type] = u.read_json(
attributes_file)
else:
command_args.json_args[resource_type] = {}
# Parses dataset generators in json format if provided
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
# Parses multi-dataset attributes in json such as field maps
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(command_args.multi_dataset_attributes)
command_args.multi_dataset_json= multi_dataset_json
else:
command_args.multi_dataset_json = {}
dataset_ids = None
command_args.dataset_ids = []
# Parses dataset/id if provided.
if command_args.datasets:
dataset_ids = u.read_datasets(command_args.datasets)
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
test_dataset_ids = None
command_args.test_dataset_ids = []
# Parses dataset/id if provided.
if command_args.test_datasets:
test_dataset_ids = u.read_datasets(command_args.test_datasets)
command_args.test_dataset_ids = test_dataset_ids
# Retrieve dataset/ids if provided.
if command_args.dataset_tag:
dataset_ids = dataset_ids.extend(
u.list_ids(api.list_datasets,
"tags__in=%s" % command_args.dataset_tag))
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Reads a json filter if provided.
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
# Reads a lisp filter if provided.
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
# Adds default tags unless that it is requested not to do so.
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
# Checks combined votes method
if (command_args.method and command_args.method != COMBINATION_LABEL and
not (command_args.method in COMBINATION_WEIGHTS.keys())):
command_args.method = 0
else:
combiner_methods = dict([[value, key]
for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
# Checks missing_strategy
if (command_args.missing_strategy and
not (command_args.missing_strategy in MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
# Adds replacement=True if creating ensemble and nothing is specified
if (command_args.number_of_models > 1 and
not command_args.replacement and
not '--no-replacement' in flags and
not 'replacement' in user_defaults and
not '--no-randomize' in flags and
not 'randomize' in user_defaults and
not '--sample-rate' in flags and
not 'sample_rate' in user_defaults):
command_args.replacement = True
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print "WARNING: 'full data' is a deprecated value. Use 'full' instead"
command_args.prediction_info = FULL_FORMAT
# Parses class, weight pairs for objective weight
if command_args.objective_weights:
objective_weights = (
u.read_objective_weights(command_args.objective_weights))
command_args.objective_weights_json = objective_weights
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(
command_args.args_separator)
# Sets shared_flag if --shared or --unshared has been used
if '--shared' in flags or '--unshared' in flags:
command_args.shared_flag = True
else:
command_args.shared_flag = False
def transform_args(command_args, flags, api, user_defaults):
"""Transforms the formatted argument strings into structured arguments
"""
attribute_args(command_args)
# Parses dataset generators in json format if provided
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
# Parses multi-dataset attributes in json such as field maps
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
transform_dataset_options(command_args, api)
# Reads a json filter if provided.
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
# Reads a lisp filter if provided.
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
# Adds default tags unless it is requested not to do so.
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
# Checks combined votes method
try:
if (command_args.method and command_args.method != COMBINATION_LABEL
and not (command_args.method in COMBINATION_WEIGHTS.keys())):
command_args.method = 0
else:
combiner_methods = dict(
[[value, key] for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
except AttributeError:
pass
# Checks missing_strategy
try:
if (command_args.missing_strategy and
not (command_args.missing_strategy in
MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
except AttributeError:
pass
# Adds replacement=True if creating ensemble and nothing is specified
try:
if (command_args.number_of_models > 1 and
not command_args.replacement and
not '--no-replacement' in flags and
not 'replacement' in user_defaults and
not '--no-randomize' in flags and
not 'randomize' in user_defaults and
not '--sample-rate' in flags and
not 'sample_rate' in user_defaults):
command_args.replacement = True
except AttributeError:
pass
try:
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print ("WARNING: 'full data' is a deprecated value. Use"
" 'full' instead")
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
# Parses class, weight pairs for objective weight
try:
if command_args.objective_weights:
objective_weights = (
u.read_objective_weights(command_args.objective_weights))
command_args.objective_weights_json = objective_weights
except AttributeError:
pass
try:
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(
command_args.args_separator)
except AttributeError:
pass
# Sets shared_flag if --shared or --unshared has been used
if '--shared' in flags or '--unshared' in flags:
command_args.shared_flag = True
else:
command_args.shared_flag = False
# Set remote on if scoring a trainind dataset in bigmler anomaly
try:
if command_args.score:
command_args.remote = True
if not "--prediction-info" in flags:
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
command_args.has_models_ = (
(hasattr(command_args, 'model') and command_args.model) or
(hasattr(command_args, 'models') and command_args.models) or
(hasattr(command_args, 'ensemble') and command_args.ensemble) or
(hasattr(command_args, 'ensembles') and command_args.ensembles) or
(hasattr(command_args, 'cluster') and command_args.cluster) or
(hasattr(command_args, 'clusters') and command_args.clusters) or
(hasattr(command_args, 'model_tag') and command_args.model_tag) or
(hasattr(command_args, 'anomaly') and command_args.anomaly) or
(hasattr(command_args, 'anomalies') and command_args.anomalies) or
(hasattr(command_args, 'ensemble_tag')
and command_args.ensemble_tag) or
(hasattr(command_args, 'cluster_tag') and command_args.cluster_tag) or
(hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag))
command_args.has_datasets_ = (
(hasattr(command_args, 'dataset') and command_args.dataset) or
(hasattr(command_args, 'datasets') and command_args.datasets) or
(hasattr(command_args, 'dataset_tag') and command_args.dataset_tag))
command_args.has_test_datasets_ = (
(hasattr(command_args, 'test_dataset') and
command_args.test_dataset) or
(hasattr(command_args, 'test_datasets') and
command_args.test_datasets) or
(hasattr(command_args, 'test_dataset_tag') and
command_args.test_dataset_tag))
def transform_args(command_args, flags, api, user_defaults):
"""Transforms the formatted argument strings into structured arguments
"""
# Parses attributes in json format if provided
command_args.json_args = {}
json_attribute_options = {
'source': command_args.source_attributes,
'dataset': command_args.dataset_attributes,
'model': command_args.model_attributes,
'ensemble': command_args.ensemble_attributes,
'evaluation': command_args.evaluation_attributes,
'batch_prediction': command_args.batch_prediction_attributes
}
for resource_type, attributes_file in json_attribute_options.items():
if attributes_file is not None:
command_args.json_args[resource_type] = u.read_json(
attributes_file)
else:
command_args.json_args[resource_type] = {}
# Parses dataset generators in json format if provided
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
# Parses multi-dataset attributes in json such as field maps
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
dataset_ids = None
command_args.dataset_ids = []
# Parses dataset/id if provided.
if command_args.datasets:
dataset_ids = u.read_datasets(command_args.datasets)
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Retrieve dataset/ids if provided.
if command_args.dataset_tag:
dataset_ids = dataset_ids.extend(
u.list_ids(api.list_datasets,
"tags__in=%s" % command_args.dataset_tag))
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Reads a json filter if provided.
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
# Reads a lisp filter if provided.
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
# Adds default tags unless that it is requested not to do so.
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
# Checks combined votes method
if (command_args.method and command_args.method != COMBINATION_LABEL
and not (command_args.method in COMBINATION_WEIGHTS.keys())):
command_args.method = 0
else:
combiner_methods = dict([[value, key]
for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
# Checks missing_strategy
if (command_args.missing_strategy and
not (command_args.missing_strategy in MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
# Adds replacement=True if creating ensemble and nothing is specified
if (command_args.number_of_models > 1 and not command_args.replacement
and not '--no-replacement' in flags
and not 'replacement' in user_defaults
and not '--no-randomize' in flags
and not 'randomize' in user_defaults
and not '--sample-rate' in flags
and not 'sample_rate' in user_defaults):
command_args.replacement = True
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print "WARNING: 'full data' is a deprecated value. Use 'full' instead"
command_args.prediction_info = FULL_FORMAT
# Parses class, weight pairs for objective weight
if command_args.objective_weights:
objective_weights = (u.read_objective_weights(
command_args.objective_weights))
command_args.objective_weights_json = objective_weights
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(',')
def transform_args(command_args, flags, api):
"""Transforms the formatted argument strings into structured arguments
"""
attribute_args(command_args)
# Parses dataset generators in json format if provided
try:
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
except AttributeError:
pass
# Parses multi-dataset attributes in json such as field maps
try:
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(
command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
except AttributeError:
pass
transform_dataset_options(command_args, api)
script_ids = None
command_args.script_ids = []
# Parses script/id if provided.
try:
if command_args.scripts:
script_ids = u.read_resources(command_args.scripts)
if len(script_ids) == 1:
command_args.script = script_ids[0]
command_args.script_ids = script_ids
except AttributeError:
pass
# Retrieve script/ids if provided.
try:
if command_args.script_tag:
script_ids = script_ids.extend(
u.list_ids(api.list_scripts,
"tags__in=%s" % command_args.script_tag))
if len(script_ids) == 1:
command_args.script = script_ids[0]
command_args.script_ids = script_ids
except AttributeError:
pass
# Reads a json filter if provided.
try:
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
except AttributeError:
pass
# Reads a lisp filter if provided.
try:
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
except AttributeError:
pass
# Adds default tags unless that it is requested not to do so.
try:
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
except AttributeError:
pass
# Checks combined votes method
try:
if (command_args.method and command_args.method != COMBINATION_LABEL
and not command_args.method in COMBINATION_WEIGHTS.keys()):
command_args.method = 0
else:
combiner_methods = dict(
[[value, key] for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
except AttributeError:
pass
# Checks missing_strategy
try:
if (command_args.missing_strategy and
not (command_args.missing_strategy in
MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
except AttributeError:
pass
try:
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print ("WARNING: 'full data' is a deprecated value. Use"
" 'full' instead")
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
# Parses class, weight pairs for objective weight
try:
if command_args.objective_weights:
objective_weights = (
u.read_objective_weights(command_args.objective_weights))
command_args.objective_weights_json = objective_weights
except AttributeError:
pass
try:
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(
command_args.args_separator)
except AttributeError:
pass
# Sets shared_flag if --shared or --unshared has been used
command_args.shared_flag = '--shared' in flags or '--unshared' in flags
# Set remote on if scoring a trainind dataset in bigmler anomaly
try:
if command_args.score:
command_args.remote = True
if not "--prediction-info" in flags:
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
command_args.has_supervised_ = (
(hasattr(command_args, 'model') and command_args.model) or
(hasattr(command_args, 'models') and command_args.models) or
(hasattr(command_args, 'ensemble') and command_args.ensemble) or
(hasattr(command_args, 'ensembles') and command_args.ensembles) or
(hasattr(command_args, 'model_tag') and command_args.model_tag) or
(hasattr(command_args, 'logistic_regression') and
command_args.logistic_regression) or
(hasattr(command_args, 'logistic_regressions') and
command_args.logistic_regressions) or
(hasattr(command_args, 'logistic_regression_tag') and
command_args.logistic_regression_tag) or
(hasattr(command_args, 'deepnet') and
command_args.deepnet) or
(hasattr(command_args, 'deepnets') and
command_args.deepnets) or
(hasattr(command_args, 'deepnet_tag') and
command_args.deepnet_tag) or
(hasattr(command_args, 'ensemble_tag')
and command_args.ensemble_tag))
command_args.has_models_ = (
command_args.has_supervised_ or
(hasattr(command_args, 'cluster') and command_args.cluster) or
(hasattr(command_args, 'clusters') and command_args.clusters) or
(hasattr(command_args, 'anomaly') and command_args.anomaly) or
(hasattr(command_args, 'anomalies') and command_args.anomalies) or
(hasattr(command_args, 'cluster_tag') and command_args.cluster_tag) or
(hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag))
command_args.has_datasets_ = (
(hasattr(command_args, 'dataset') and command_args.dataset) or
(hasattr(command_args, 'datasets') and command_args.datasets) or
(hasattr(command_args, 'dataset_tag') and command_args.dataset_tag))
command_args.has_test_datasets_ = (
(hasattr(command_args, 'test_dataset') and
command_args.test_dataset) or
(hasattr(command_args, 'test_datasets') and
command_args.test_datasets) or
(hasattr(command_args, 'test_dataset_tag') and
command_args.test_dataset_tag))
def get_output_args(api, command_args, resume):
"""Returns the output args needed for the main bigmler computation process
"""
try:
if command_args.train_stdin:
if command_args.test_stdin:
sys.exit("The standard input can't be used both for training "
"and testing. Choose one of them")
command_args.training_set = StringIO(sys.stdin.read())
elif command_args.test_stdin:
command_args.test_set = StringIO(sys.stdin.read())
except AttributeError:
pass
try:
if command_args.objective_field:
objective = command_args.objective_field
try:
command_args.objective_field = int(objective)
except ValueError:
if not command_args.train_header:
sys.exit("The %s has been set as objective field but"
" the file has not been marked as containing"
" headers.\nPlease set the --train-header flag if"
" the file has headers or use a column number"
" to set the objective field." % objective)
except AttributeError:
pass
command_args.resume_ = resume
command_args.predictions = command_args.output
# Reads description if provided.
try:
if command_args.description:
description_arg = u.read_description(command_args.description)
command_args.description_ = description_arg
else:
command_args.description_ = DEFAULT_DESCRIPTION
except AttributeError:
pass
# Parses fields if provided.
try:
if command_args.field_attributes:
field_attributes_arg = (
u.read_field_attributes(command_args.field_attributes))
command_args.field_attributes_ = field_attributes_arg
else:
command_args.field_attributes_ = []
except AttributeError:
pass
try:
if command_args.test_field_attributes:
field_attributes_arg = (
u.read_field_attributes(command_args.test_field_attributes))
command_args.test_field_attributes_ = field_attributes_arg
else:
command_args.test_field_attributes_ = []
except AttributeError:
pass
# Parses types if provided.
try:
if command_args.types:
types_arg = u.read_types(command_args.types)
command_args.types_ = types_arg
else:
command_args.types_ = None
if command_args.test_types:
types_arg = u.read_types(command_args.test_types)
command_args.test_types_ = types_arg
else:
command_args.test_types_ = None
except AttributeError:
pass
# Parses dataset fields if provided.
try:
if command_args.dataset_fields:
dataset_fields_arg = [
field.strip() for field in command_args.dataset_fields.split(
command_args.args_separator)]
command_args.dataset_fields_ = dataset_fields_arg
else:
command_args.dataset_fields_ = []
except AttributeError:
pass
# Parses model input fields if provided.
try:
if command_args.model_fields:
model_fields_arg = [
field.strip() for field in command_args.model_fields.split(
command_args.args_separator)]
command_args.model_fields_ = model_fields_arg
else:
command_args.model_fields_ = []
except AttributeError:
pass
# Parses cluster input fields if provided.
try:
if command_args.cluster_fields:
cluster_fields_arg = [
field.strip() for field in command_args.cluster_fields.split(
command_args.args_separator)]
command_args.cluster_fields_ = cluster_fields_arg
else:
command_args.cluster_fields_ = []
except AttributeError:
pass
# Parses association input fields if provided.
try:
if command_args.association_fields:
association_fields_arg = [
field.strip() for field in \
command_args.association_fields.split( \
command_args.args_separator)]
command_args.association_fields_ = association_fields_arg
else:
command_args.association_fields_ = []
except AttributeError:
pass
# Parses anomaly input fields if provided.
try:
if command_args.anomaly_fields:
anomaly_fields_arg = [
field.strip() for field in command_args.anomaly_fields.split(
command_args.args_separator)]
command_args.anomaly_fields_ = anomaly_fields_arg
else:
command_args.anomaly_fields_ = []
except AttributeError:
pass
# Parses logistic regression input fields if provided.
try:
if command_args.logistic_fields:
logistic_fields_arg = [
field.strip() for field in command_args.logistic_fields.split(
command_args.args_separator)]
command_args.logistic_fields_ = logistic_fields_arg
else:
command_args.logistic_fields_ = []
except AttributeError:
pass
# Parses deepnet input fields if provided.
try:
if command_args.deepnet_fields:
deepnet_fields_arg = [
field.strip() for field in command_args.deepnet_fields.split(
command_args.args_separator)]
command_args.deepnet_fields_ = deepnet_fields_arg
else:
command_args.deepnet_fields_ = []
except AttributeError:
pass
# Parses topic model fields if provided.
try:
if command_args.topic_fields:
topic_fields_arg = [
field.strip() for field in command_args.topic_fields.split(
command_args.args_separator)]
command_args.topic_model_fields_ = topic_fields_arg
else:
command_args.topic_model_fields_ = []
except AttributeError:
pass
# Parses field_codings for deepnet
try:
if command_args.field_codings:
command_args.field_codings_ = u.read_json(
command_args.field_codings)
else:
command_args.field_codings_ = []
except AttributeError:
pass
# Parses imports for scripts and libraries.
try:
if command_args.imports:
imports_arg = [
field.strip() for field in command_args.imports.split(
command_args.args_separator)]
command_args.imports_ = imports_arg
else:
command_args.imports_ = []
except AttributeError:
pass
# Parses objective fields for time-series.
try:
if command_args.objectives:
objective_fields_arg = [
field.strip() for field in command_args.objectives.split(
command_args.args_separator)]
command_args.objective_fields_ = objective_fields_arg
else:
command_args.objective_fields_ = []
except AttributeError:
pass
# Parses range.
try:
if command_args.range:
range_arg = [
value.strip() for value in command_args.range.split(
command_args.args_separator)]
command_args.range_ = range_arg
else:
command_args.range_ = []
except AttributeError:
pass
# Parses parameters for scripts.
try:
if command_args.declare_inputs:
command_args.parameters_ = u.read_json(command_args.declare_inputs)
else:
command_args.parameters_ = []
except AttributeError:
pass
# Parses creation_defaults for executions.
try:
if command_args.creation_defaults:
command_args.creation_defaults_ = u.read_json(
command_args.creation_defaults)
else:
command_args.creation_defaults_ = {}
except AttributeError:
pass
# Parses arguments for executions.
try:
if command_args.inputs:
command_args.arguments_ = u.read_json(command_args.inputs)
else:
command_args.arguments_ = []
except AttributeError:
pass
# Parses input maps for executions.
try:
if command_args.input_maps:
command_args.input_maps_ = u.read_json(command_args.input_maps)
else:
command_args.input_maps_ = []
except AttributeError:
pass
# Parses outputs for executions.
try:
if command_args.outputs:
command_args.outputs_ = u.read_json(command_args.outputs)
else:
command_args.outputs_ = []
except AttributeError:
pass
# Parses outputs for scripts.
try:
if command_args.declare_outputs:
command_args.declare_outputs_ = \
u.read_json(command_args.declare_outputs)
else:
command_args.declare_outputs_ = []
except AttributeError:
pass
model_ids = []
try:
# Parses model/ids if provided.
if command_args.models:
model_ids = u.read_resources(command_args.models)
command_args.model_ids_ = model_ids
except AttributeError:
pass
# Retrieve model/ids if provided.
try:
if command_args.model_tag:
model_ids = (model_ids +
u.list_ids(api.list_models,
"tags__in=%s" % command_args.model_tag))
command_args.model_ids_ = model_ids
except AttributeError:
pass
# Reads votes files in the provided directories.
try:
if command_args.votes_dirs:
dirs = [
directory.strip() for directory in
command_args.votes_dirs.split(
command_args.args_separator)]
votes_path = os.path.dirname(command_args.predictions)
votes_files = u.read_votes_files(dirs, votes_path)
command_args.votes_files_ = votes_files
else:
command_args.votes_files_ = []
except AttributeError:
pass
# Parses fields map if provided.
try:
if command_args.fields_map:
fields_map_arg = u.read_fields_map(command_args.fields_map)
command_args.fields_map_ = fields_map_arg
else:
command_args.fields_map_ = None
except AttributeError:
pass
cluster_ids = []
try:
# Parses cluster/ids if provided.
if command_args.clusters:
cluster_ids = u.read_resources(command_args.clusters)
command_args.cluster_ids_ = cluster_ids
except AttributeError:
pass
# Retrieve cluster/ids if provided.
try:
if command_args.cluster_tag:
cluster_ids = (cluster_ids +
u.list_ids(api.list_clusters,
"tags__in=%s" %
command_args.cluster_tag))
command_args.cluster_ids_ = cluster_ids
except AttributeError:
pass
association_ids = []
try:
# Parses association/ids if provided.
if command_args.associations:
association_ids = u.read_resources(command_args.associations)
command_args.association_ids_ = association_ids
except AttributeError:
pass
# Retrieve association/ids if provided.
try:
if command_args.association_tag:
association_ids = (association_ids +
u.list_ids(api.list_associations,
"tags__in=%s" %
command_args.association_tag))
command_args.association_ids_ = association_ids
except AttributeError:
pass
logistic_regression_ids = []
try:
# Parses logisticregression/ids if provided.
if command_args.logistic_regressions:
logistic_regression_ids = u.read_resources( \
command_args.logistic_regressions)
command_args.logistic_regression_ids_ = logistic_regression_ids
except AttributeError:
pass
# Retrieve logsticregression/ids if provided.
try:
if command_args.logistic_regression_tag:
logistic_regression_ids = (logistic_regression_ids + \
u.list_ids(api.list_logistic_regressions,
"tags__in=%s" %
command_args.logistic_regression_tag))
command_args.logistic_regression_ids_ = logistic_regression_ids
except AttributeError:
pass
deepnet_ids = []
try:
# Parses deepnet/ids if provided.
if command_args.deepnets:
deepnet_ids = u.read_resources( \
command_args.deepnets)
command_args.deepnet_ids_ = deepnet_ids
except AttributeError:
pass
# Retrieve deepnet/ids if provided.
try:
if command_args.deepnet_tag:
deepnet_regression_ids = (deepnet_ids + \
u.list_ids(api.list_deepnets,
"tags__in=%s" % command_args.deepnet_tag))
command_args.deepnet_ids_ = deepnet_ids
except AttributeError:
pass
topic_model_ids = []
try:
# Parses topicmodel/ids if provided.
if command_args.topic_models:
topic_model_ids = u.read_resources(command_args.topic_models)
command_args.topic_model_ids_ = topic_model_ids
except AttributeError:
pass
# Retrieve topicmodel/ids if provided.
try:
if command_args.topic_model_tag:
topic_model_ids = (topic_model_ids +
u.list_ids(api.list_topic_models,
"tags__in=%s" %
command_args.topic_model_tag))
command_args.topic_model_ids_ = topic_model_ids
except AttributeError:
pass
time_series_ids = []
try:
# Parses timeseries/ids if provided.
if command_args.time_series_set:
time_series_ids = u.read_resources(command_args.time_series)
command_args.time_series_ids_ = time_series_ids
except AttributeError:
pass
# Retrieve timeseries/ids if provided.
try:
if command_args.time_series_tag:
time_series_ids = (time_series_ids +
u.list_ids(api.list_time_series,
"tags__in=%s" %
command_args.time_series_tag))
command_args.time_series_ids_ = time_series_ids
except AttributeError:
pass
# Parses cluster names to generate datasets if provided
try:
if command_args.cluster_datasets:
cluster_datasets_arg = [
dataset.strip() for dataset in
command_args.cluster_datasets.split(
command_args.args_separator)]
command_args.cluster_datasets_ = cluster_datasets_arg
else:
command_args.cluster_datasets_ = []
except AttributeError:
pass
# Parses cluster names to generate models if provided
try:
if command_args.cluster_models:
cluster_models_arg = [
model.strip() for model in
command_args.cluster_models.split(
command_args.args_separator)]
command_args.cluster_models_ = cluster_models_arg
else:
command_args.cluster_models_ = []
except AttributeError:
pass
# Parses summary_fields to exclude from the clustering algorithm
try:
if command_args.summary_fields:
summary_fields_arg = [
field.strip() for field in
command_args.summary_fields.split(
command_args.args_separator)]
command_args.summary_fields_ = summary_fields_arg
else:
command_args.summary_fields_ = []
except AttributeError:
pass
anomaly_ids = []
try:
# Parses anomaly/ids if provided.
if command_args.anomalies:
anomaly_ids = u.read_resources(command_args.anomalies)
command_args.anomaly_ids_ = anomaly_ids
except AttributeError:
pass
# Retrieve anomaly/ids if provided.
try:
if command_args.anomaly_tag:
anomaly_ids = (anomaly_ids +
u.list_ids(api.list_anomalies,
"tags__in=%s" %
command_args.anomaly_tag))
command_args.anomaly_ids_ = anomaly_ids
except AttributeError:
pass
sample_ids = []
try:
# Parses sample/ids if provided.
if command_args.samples:
sample_ids = u.read_resources(command_args.samples)
command_args.sample_ids_ = sample_ids
except AttributeError:
pass
# Retrieve sample/ids if provided.
try:
if command_args.sample_tag:
sample_ids = (
sample_ids + u.list_ids(api.list_samples,
"tags__in=%s" %
command_args.sample_tag))
command_args.sample_ids_ = sample_ids
except AttributeError:
pass
# Parses sample row fields
try:
if command_args.row_fields:
row_fields_arg = [field.strip() for field in
command_args.row_fields.split(
command_args.args_separator)]
command_args.row_fields_ = row_fields_arg
else:
command_args.row_fields_ = []
except AttributeError:
pass
# Parses sample stat_fields
try:
if command_args.stat_fields:
stat_fields_arg = [field.strip() for field in
command_args.stat_fields.split(
command_args.args_separator)]
command_args.stat_fields_ = stat_fields_arg
else:
command_args.stat_fields_ = []
except AttributeError:
pass
# if boosting arguments are used, set on boosting
try:
if command_args.iterations or command_args.learning_rate \
or command_args.early_holdout:
command_args.boosting = True
except AttributeError:
pass
# Extracts the imports from the JSON metadata file
try:
if command_args.embedded_imports:
command_args.embedded_imports_ = u.read_resources( \
command_args.embedded_imports)
else:
command_args.embedded_imports_ = []
except AttributeError:
pass
# Parses hidden_layers for deepnets.
try:
if command_args.hidden_layers:
command_args.hidden_layers_ = u.read_json(
command_args.hidden_layers)
else:
command_args.hidden_layers_ = []
except AttributeError:
pass
# Parses operating_point for predictions.
try:
if command_args.operating_point:
command_args.operating_point_ = u.read_json(
command_args.operating_point)
else:
command_args.operating_point_ = []
except AttributeError:
pass
return {"api": api, "args": command_args}
def get_output_args(api, command_args, resume):
"""Returns the output args needed for the main bigmler computation process
"""
try:
if command_args.train_stdin:
if command_args.test_stdin:
sys.exit("The standard input can't be used both for training "
"and testing. Choose one of them")
command_args.training_set = StringIO(sys.stdin.read())
elif command_args.test_stdin:
command_args.test_set = StringIO(sys.stdin.read())
except AttributeError:
pass
try:
if command_args.objective_field:
objective = command_args.objective_field
try:
command_args.objective_field = int(objective)
except ValueError:
if not command_args.train_header:
sys.exit("The %s has been set as objective field but"
" the file has not been marked as containing"
" headers.\nPlease set the --train-header flag if"
" the file has headers or use a column number"
" to set the objective field." % objective)
except AttributeError:
pass
command_args.resume_ = resume
# Reads description if provided.
try:
if command_args.description:
description_arg = u.read_description(command_args.description)
command_args.description_ = description_arg
else:
command_args.description_ = DEFAULT_DESCRIPTION
except AttributeError:
pass
# Parses fields if provided.
try:
if command_args.field_attributes:
field_attributes_arg = (u.read_field_attributes(
command_args.field_attributes))
command_args.field_attributes_ = field_attributes_arg
else:
command_args.field_attributes_ = []
except AttributeError:
pass
try:
if command_args.test_field_attributes:
field_attributes_arg = (u.read_field_attributes(
command_args.test_field_attributes))
command_args.test_field_attributes_ = field_attributes_arg
else:
command_args.test_field_attributes_ = []
except AttributeError:
pass
# Parses types if provided.
try:
if command_args.types:
types_arg = u.read_types(command_args.types)
command_args.types_ = types_arg
else:
command_args.types_ = None
if command_args.test_types:
types_arg = u.read_types(command_args.test_types)
command_args.test_types_ = types_arg
else:
command_args.test_types_ = None
except AttributeError:
pass
# Parses dataset fields if provided.
try:
if command_args.dataset_fields:
dataset_fields_arg = [
field.strip() for field in command_args.dataset_fields.split(
command_args.args_separator)
]
command_args.dataset_fields_ = dataset_fields_arg
else:
command_args.dataset_fields_ = []
except AttributeError:
pass
# Parses model input fields if provided.
try:
if command_args.model_fields:
model_fields_arg = [
field.strip() for field in command_args.model_fields.split(
command_args.args_separator)
]
command_args.model_fields_ = model_fields_arg
else:
command_args.model_fields_ = []
except AttributeError:
pass
# Parses cluster input fields if provided.
try:
if command_args.cluster_fields:
cluster_fields_arg = [
field.strip() for field in command_args.cluster_fields.split(
command_args.args_separator)
]
command_args.cluster_fields_ = cluster_fields_arg
else:
command_args.cluster_fields_ = []
except AttributeError:
pass
# Parses association input fields if provided.
try:
if command_args.association_fields:
association_fields_arg = [
field.strip() for field in \
command_args.association_fields.split( \
command_args.args_separator)]
command_args.association_fields_ = association_fields_arg
else:
command_args.association_fields_ = []
except AttributeError:
pass
# Parses anomaly input fields if provided.
try:
if command_args.anomaly_fields:
anomaly_fields_arg = [
field.strip() for field in command_args.anomaly_fields.split(
command_args.args_separator)
]
command_args.anomaly_fields_ = anomaly_fields_arg
else:
command_args.anomaly_fields_ = []
except AttributeError:
pass
# Parses logistic regression input fields if provided.
try:
if command_args.logistic_fields:
logistic_fields_arg = [
field.strip() for field in command_args.logistic_fields.split(
command_args.args_separator)
]
command_args.logistic_fields_ = logistic_fields_arg
else:
command_args.logistic_fields_ = []
except AttributeError:
pass
# Parses field_codings for logistic regressions
try:
if command_args.field_codings:
command_args.field_codings_ = u.read_json(
command_args.field_codings)
else:
command_args.field_codings_ = []
except AttributeError:
pass
# Parses imports for scripts and libraries.
try:
if command_args.imports:
imports_arg = [
field.strip() for field in command_args.imports.split(
command_args.args_separator)
]
command_args.imports_ = imports_arg
else:
command_args.imports_ = []
except AttributeError:
pass
# Parses parameters for scripts.
try:
if command_args.declare_inputs:
command_args.parameters_ = u.read_json(command_args.declare_inputs)
else:
command_args.parameters_ = []
except AttributeError:
pass
# Parses creation_defaults for executions.
try:
if command_args.creation_defaults:
command_args.creation_defaults_ = u.read_json(
command_args.creation_defaults)
else:
command_args.creation_defaults_ = {}
except AttributeError:
pass
# Parses arguments for executions.
try:
if command_args.inputs:
command_args.arguments_ = u.read_json(command_args.inputs)
else:
command_args.arguments_ = []
except AttributeError:
pass
# Parses input maps for executions.
try:
if command_args.input_maps:
command_args.input_maps_ = u.read_json(command_args.input_maps)
else:
command_args.input_maps_ = []
except AttributeError:
pass
# Parses outputs for executions.
try:
if command_args.outputs:
command_args.outputs_ = u.read_json(command_args.outputs)
else:
command_args.outputs_ = []
except AttributeError:
pass
# Parses outputs for scripts.
try:
if command_args.declare_outputs:
command_args.declare_outputs_ = \
u.read_json(command_args.declare_outputs)
else:
command_args.declare_outputs_ = []
except AttributeError:
pass
model_ids = []
try:
# Parses model/ids if provided.
if command_args.models:
model_ids = u.read_resources(command_args.models)
command_args.model_ids_ = model_ids
except AttributeError:
pass
# Retrieve model/ids if provided.
try:
if command_args.model_tag:
model_ids = (model_ids + u.list_ids(
api.list_models, "tags__in=%s" % command_args.model_tag))
command_args.model_ids_ = model_ids
except AttributeError:
pass
# Reads votes files in the provided directories.
try:
if command_args.votes_dirs:
dirs = [
directory.strip() for directory in
command_args.votes_dirs.split(command_args.args_separator)
]
votes_path = os.path.dirname(command_args.predictions)
votes_files = u.read_votes_files(dirs, votes_path)
command_args.votes_files_ = votes_files
else:
command_args.votes_files_ = []
except AttributeError:
pass
# Parses fields map if provided.
try:
if command_args.fields_map:
fields_map_arg = u.read_fields_map(command_args.fields_map)
command_args.fields_map_ = fields_map_arg
else:
command_args.fields_map_ = None
except AttributeError:
pass
cluster_ids = []
try:
# Parses cluster/ids if provided.
if command_args.clusters:
cluster_ids = u.read_resources(command_args.clusters)
command_args.cluster_ids_ = cluster_ids
except AttributeError:
pass
# Retrieve cluster/ids if provided.
try:
if command_args.cluster_tag:
cluster_ids = (cluster_ids + u.list_ids(
api.list_clusters, "tags__in=%s" % command_args.cluster_tag))
command_args.cluster_ids_ = cluster_ids
except AttributeError:
pass
association_ids = []
try:
# Parses association/ids if provided.
if command_args.associations:
association_ids = u.read_resources(command_args.associations)
command_args.association_ids_ = association_ids
except AttributeError:
pass
# Retrieve cluster/ids if provided.
try:
if command_args.association_tag:
association_ids = (
association_ids +
u.list_ids(api.list_associations,
"tags__in=%s" % command_args.association_tag))
command_args.association_ids_ = association_ids
except AttributeError:
pass
logistic_regression_ids = []
try:
# Parses logisticregression/ids if provided.
if command_args.logistic_regressions:
logistic_regression_ids = u.read_resources( \
command_args.logistic_regressions)
command_args.logistic_regression_ids_ = logistic_regression_ids
except AttributeError:
pass
# Retrieve logisticregression/ids if provided.
try:
if command_args.logistic_tag:
logistic_regression_ids = (
logistic_ids +
u.list_ids(api.list_logistic_regressions,
"tags__in=%s" % command_args.logistic_tag))
command_args.logistic_regression_ids_ = logistic_regression_ids
except AttributeError:
pass
# Parses cluster names to generate datasets if provided
try:
if command_args.cluster_datasets:
cluster_datasets_arg = [
dataset.strip()
for dataset in command_args.cluster_datasets.split(
command_args.args_separator)
]
command_args.cluster_datasets_ = cluster_datasets_arg
else:
command_args.cluster_datasets_ = []
except AttributeError:
pass
# Parses cluster names to generate models if provided
try:
if command_args.cluster_models:
cluster_models_arg = [
model.strip() for model in command_args.cluster_models.split(
command_args.args_separator)
]
command_args.cluster_models_ = cluster_models_arg
else:
command_args.cluster_models_ = []
except AttributeError:
pass
# Parses summary_fields to exclude from the clustering algorithm
try:
if command_args.summary_fields:
summary_fields_arg = [
field.strip() for field in command_args.summary_fields.split(
command_args.args_separator)
]
command_args.summary_fields_ = summary_fields_arg
else:
command_args.summary_fields_ = []
except AttributeError:
pass
anomaly_ids = []
try:
# Parses anomaly/ids if provided.
if command_args.anomalies:
anomaly_ids = u.read_resources(command_args.anomalies)
command_args.anomaly_ids_ = anomaly_ids
except AttributeError:
pass
# Retrieve anomaly/ids if provided.
try:
if command_args.anomaly_tag:
anomaly_ids = (anomaly_ids + u.list_ids(
api.list_anomalies, "tags__in=%s" % command_args.anomaly_tag))
command_args.anomaly_ids_ = anomaly_ids
except AttributeError:
pass
sample_ids = []
try:
# Parses sample/ids if provided.
if command_args.samples:
sample_ids = u.read_resources(command_args.samples)
command_args.sample_ids_ = sample_ids
except AttributeError:
pass
# Retrieve sample/ids if provided.
try:
if command_args.sample_tag:
sample_ids = (sample_ids + u.list_ids(
api.list_samples, "tags__in=%s" % command_args.sample_tag))
command_args.sample_ids_ = sample_ids
except AttributeError:
pass
# Parses sample row fields
try:
if command_args.row_fields:
row_fields_arg = [
field.strip() for field in command_args.row_fields.split(
command_args.args_separator)
]
command_args.row_fields_ = row_fields_arg
else:
command_args.row_fields_ = []
except AttributeError:
pass
# Parses sample stat_fields
try:
if command_args.stat_fields:
stat_fields_arg = [
field.strip() for field in command_args.stat_fields.split(
command_args.args_separator)
]
command_args.stat_fields_ = stat_fields_arg
else:
command_args.stat_fields_ = []
except AttributeError:
pass
return {"api": api, "args": command_args}
def transform_args(command_args, flags, api, user_defaults):
"""Transforms the formatted argument strings into structured arguments
"""
# Parses attributes in json format if provided
command_args.json_args = {}
for resource_type in RESOURCE_TYPES:
attributes_file = getattr(command_args, "%s_attributes" % resource_type, None)
if attributes_file is not None:
command_args.json_args[resource_type] = u.read_json(attributes_file)
else:
command_args.json_args[resource_type] = {}
# Parses dataset generators in json format if provided
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
# Parses multi-dataset attributes in json such as field maps
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
dataset_ids = None
command_args.dataset_ids = []
# Parses dataset/id if provided.
if command_args.datasets:
dataset_ids = u.read_datasets(command_args.datasets)
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
test_dataset_ids = None
command_args.test_dataset_ids = []
# Parses dataset/id if provided.
if command_args.test_datasets:
test_dataset_ids = u.read_datasets(command_args.test_datasets)
command_args.test_dataset_ids = test_dataset_ids
# Retrieve dataset/ids if provided.
if command_args.dataset_tag:
dataset_ids = dataset_ids.extend(u.list_ids(api.list_datasets, "tags__in=%s" % command_args.dataset_tag))
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Reads a json filter if provided.
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
# Reads a lisp filter if provided.
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
# Adds default tags unless that it is requested not to do so.
if command_args.no_tag:
command_args.tag.append("BigMLer")
command_args.tag.append("BigMLer_%s" % NOW)
# Checks combined votes method
try:
if (
command_args.method
and command_args.method != COMBINATION_LABEL
and not (command_args.method in COMBINATION_WEIGHTS.keys())
):
command_args.method = 0
else:
combiner_methods = dict([[value, key] for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
except AttributeError:
pass
# Checks missing_strategy
try:
if command_args.missing_strategy and not (command_args.missing_strategy in MISSING_STRATEGIES.keys()):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(command_args.missing_strategy, 0)
except AttributeError:
pass
# Adds replacement=True if creating ensemble and nothing is specified
try:
if (
command_args.number_of_models > 1
and not command_args.replacement
and not "--no-replacement" in flags
and not "replacement" in user_defaults
and not "--no-randomize" in flags
and not "randomize" in user_defaults
and not "--sample-rate" in flags
and not "sample_rate" in user_defaults
):
command_args.replacement = True
except AttributeError:
pass
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == "full data":
print "WARNING: 'full data' is a deprecated value. Use 'full' instead"
command_args.prediction_info = FULL_FORMAT
# Parses class, weight pairs for objective weight
try:
if command_args.objective_weights:
objective_weights = u.read_objective_weights(command_args.objective_weights)
command_args.objective_weights_json = objective_weights
except AttributeError:
pass
try:
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(command_args.args_separator)
except AttributeError:
pass
# Sets shared_flag if --shared or --unshared has been used
if "--shared" in flags or "--unshared" in flags:
command_args.shared_flag = True
else:
command_args.shared_flag = False
command_args.has_models_ = (
(hasattr(command_args, "model") and command_args.model)
or (hasattr(command_args, "models") and command_args.models)
or (hasattr(command_args, "ensemble") and command_args.ensemble)
or (hasattr(command_args, "ensembles") and command_args.ensembles)
or (hasattr(command_args, "cluster") and command_args.cluster)
or (hasattr(command_args, "clusters") and command_args.clusters)
or (hasattr(command_args, "model_tag") and command_args.model_tag)
or (hasattr(command_args, "anomaly") and command_args.anomaly)
or (hasattr(command_args, "anomalies") and command_args.anomalies)
or (hasattr(command_args, "ensemble_tag") and command_args.ensemble_tag)
or (hasattr(command_args, "cluster_tag") and command_args.cluster_tag)
or (hasattr(command_args, "anomaly_tag") and command_args.anomaly_tag)
)
command_args.has_datasets_ = (
(hasattr(command_args, "dataset") and command_args.dataset)
or (hasattr(command_args, "datasets") and command_args.datasets)
or (hasattr(command_args, "dataset_tag") and command_args.dataset_tag)
)
