def transform_args(command_args, flags, api):
"""Transforms the formatted argument strings into structured arguments
"""
attribute_args(command_args)
# Parses dataset generators in json format if provided
try:
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
except AttributeError:
pass
# Parses multi-dataset attributes in json such as field maps
try:
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(
command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
except AttributeError:
pass
transform_dataset_options(command_args, api)
script_ids = None
command_args.script_ids = []
# Parses script/id if provided.
try:
if command_args.scripts:
script_ids = u.read_resources(command_args.scripts)
if len(script_ids) == 1:
command_args.script = script_ids[0]
command_args.script_ids = script_ids
except AttributeError:
pass
# Retrieve script/ids if provided.
try:
if command_args.script_tag:
script_ids = script_ids.extend(
u.list_ids(api.list_scripts,
"tags__in=%s" % command_args.script_tag))
if len(script_ids) == 1:
command_args.script = script_ids[0]
command_args.script_ids = script_ids
except AttributeError:
pass
# Reads a json filter if provided.
try:
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
except AttributeError:
pass
# Reads a lisp filter if provided.
try:
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
except AttributeError:
pass
# Adds default tags unless that it is requested not to do so.
try:
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
except AttributeError:
pass
# Checks combined votes method
try:
if (command_args.method and command_args.method != COMBINATION_LABEL
and not command_args.method in COMBINATION_WEIGHTS.keys()):
command_args.method = 0
else:
combiner_methods = dict([[value, key]
for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
except AttributeError:
pass
# Checks missing_strategy
try:
if (command_args.missing_strategy
and not (command_args.missing_strategy
in MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
except AttributeError:
pass
try:
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print("WARNING: 'full data' is a deprecated value. Use"
" 'full' instead")
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
# Parses class, weight pairs for objective weight
try:
if command_args.objective_weights:
objective_weights = (u.read_objective_weights(
command_args.objective_weights))
command_args.objective_weights_json = objective_weights
except AttributeError:
pass
try:
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(
command_args.args_separator)
except AttributeError:
pass
# Sets shared_flag if --shared or --unshared has been used
command_args.shared_flag = '--shared' in flags or '--unshared' in flags
# Set remote on if scoring a trainind dataset in bigmler anomaly
try:
if command_args.score:
command_args.remote = True
if not "--prediction-info" in flags:
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
command_args.has_supervised_ = (
(hasattr(command_args, 'model') and command_args.model)
or (hasattr(command_args, 'models') and command_args.models)
or (hasattr(command_args, 'ensemble') and command_args.ensemble)
or (hasattr(command_args, 'ensembles') and command_args.ensembles)
or (hasattr(command_args, 'model_tag') and command_args.model_tag)
or (hasattr(command_args, 'logistic_regression')
and command_args.logistic_regression)
or (hasattr(command_args, 'logistic_regressions')
and command_args.logistic_regressions)
or (hasattr(command_args, 'logistic_regression_tag')
and command_args.logistic_regression_tag)
or (hasattr(command_args, 'deepnet') and command_args.deepnet)
or (hasattr(command_args, 'deepnets') and command_args.deepnets) or
(hasattr(command_args, 'deepnet_tag') and command_args.deepnet_tag) or
(hasattr(command_args, 'ensemble_tag') and command_args.ensemble_tag))
command_args.has_models_ = (
command_args.has_supervised_
or (hasattr(command_args, 'cluster') and command_args.cluster)
or (hasattr(command_args, 'clusters') and command_args.clusters)
or (hasattr(command_args, 'anomaly') and command_args.anomaly)
or (hasattr(command_args, 'anomalies') and command_args.anomalies)
or (hasattr(command_args, 'cluster_tag') and command_args.cluster_tag)
or (hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag))
command_args.has_datasets_ = (
(hasattr(command_args, 'dataset') and command_args.dataset)
or (hasattr(command_args, 'datasets') and command_args.datasets)
or (hasattr(command_args, 'dataset_ids') and command_args.dataset_ids)
or (hasattr(command_args, 'dataset_tag') and command_args.dataset_tag))
command_args.has_test_datasets_ = (
(hasattr(command_args, 'test_dataset') and command_args.test_dataset)
or
(hasattr(command_args, 'test_datasets') and command_args.test_datasets)
or (hasattr(command_args, 'test_dataset_tag')
and command_args.test_dataset_tag))
command_args.new_dataset = (
(hasattr(command_args, 'datasets_json') and command_args.datasets_json)
or
(hasattr(command_args, 'multi_dataset') and command_args.multi_dataset)
or (hasattr(command_args, 'juxtapose') and command_args.juxtapose)
or (hasattr(command_args, 'sql_query') and command_args.sql_query)
or (hasattr(command_args, 'sql_output_fields')
and command_args.sql_output_fields)
or (hasattr(command_args, 'json_query') and command_args.json_query))
def transform_args(command_args, flags, api, user_defaults):
"""Transforms the formatted argument strings into structured arguments
"""
# Parses attributes in json format if provided
command_args.json_args = {}
for resource_type in RESOURCE_TYPES:
attributes_file = getattr(command_args,
"%s_attributes" % resource_type, None)
if attributes_file is not None:
command_args.json_args[resource_type] = u.read_json(
attributes_file)
else:
command_args.json_args[resource_type] = {}
# Parses dataset generators in json format if provided
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
# Parses multi-dataset attributes in json such as field maps
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
dataset_ids = None
command_args.dataset_ids = []
# Parses dataset/id if provided.
if command_args.datasets:
dataset_ids = u.read_datasets(command_args.datasets)
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Reading test dataset ids is delayed till the very moment of use to ensure
# that the newly generated resources files can be used there too
command_args.test_dataset_ids = []
# Retrieve dataset/ids if provided.
if command_args.dataset_tag:
dataset_ids = dataset_ids.extend(
u.list_ids(api.list_datasets,
"tags__in=%s" % command_args.dataset_tag))
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Reads a json filter if provided.
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
# Reads a lisp filter if provided.
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
# Adds default tags unless that it is requested not to do so.
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
# Checks combined votes method
try:
if (command_args.method and command_args.method != COMBINATION_LABEL
and not (command_args.method in COMBINATION_WEIGHTS.keys())):
command_args.method = 0
else:
combiner_methods = dict([[value, key]
for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
except AttributeError:
pass
# Checks missing_strategy
try:
if (command_args.missing_strategy
and not (command_args.missing_strategy
in MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
except AttributeError:
pass
# Adds replacement=True if creating ensemble and nothing is specified
try:
if (command_args.number_of_models > 1 and not command_args.replacement
and not '--no-replacement' in flags
and not 'replacement' in user_defaults
and not '--no-randomize' in flags
and not 'randomize' in user_defaults
and not '--sample-rate' in flags
and not 'sample_rate' in user_defaults):
command_args.replacement = True
except AttributeError:
pass
try:
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print("WARNING: 'full data' is a deprecated value. Use"
" 'full' instead")
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
# Parses class, weight pairs for objective weight
try:
if command_args.objective_weights:
objective_weights = (u.read_objective_weights(
command_args.objective_weights))
command_args.objective_weights_json = objective_weights
except AttributeError:
pass
try:
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(
command_args.args_separator)
except AttributeError:
pass
# Sets shared_flag if --shared or --unshared has been used
if '--shared' in flags or '--unshared' in flags:
command_args.shared_flag = True
else:
command_args.shared_flag = False
# Set remote on if scoring a trainind dataset in bigmler anomaly
try:
if command_args.score:
command_args.remote = True
if not "--prediction-info" in flags:
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
command_args.has_models_ = (
(hasattr(command_args, 'model') and command_args.model)
or (hasattr(command_args, 'models') and command_args.models)
or (hasattr(command_args, 'ensemble') and command_args.ensemble)
or (hasattr(command_args, 'ensembles') and command_args.ensembles)
or (hasattr(command_args, 'cluster') and command_args.cluster)
or (hasattr(command_args, 'clusters') and command_args.clusters)
or (hasattr(command_args, 'model_tag') and command_args.model_tag)
or (hasattr(command_args, 'anomaly') and command_args.anomaly)
or (hasattr(command_args, 'anomalies') and command_args.anomalies) or
(hasattr(command_args, 'ensemble_tag') and command_args.ensemble_tag)
or (hasattr(command_args, 'cluster_tag') and command_args.cluster_tag)
or (hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag))
command_args.has_datasets_ = (
(hasattr(command_args, 'dataset') and command_args.dataset)
or (hasattr(command_args, 'datasets') and command_args.datasets)
or (hasattr(command_args, 'dataset_tag') and command_args.dataset_tag))
command_args.has_test_datasets_ = (
(hasattr(command_args, 'test_dataset') and command_args.test_dataset)
or
(hasattr(command_args, 'test_datasets') and command_args.test_datasets)
or (hasattr(command_args, 'test_dataset_tag')
and command_args.test_dataset_tag))
def transform_args(command_args, flags, api, user_defaults):
"""Transforms the formatted argument strings into structured arguments
"""
# Parses attributes in json format if provided
command_args.json_args = {}
json_attribute_options = {
'source': command_args.source_attributes,
'dataset': command_args.dataset_attributes,
'model': command_args.model_attributes,
'ensemble': command_args.ensemble_attributes,
'evaluation': command_args.evaluation_attributes,
'batch_prediction': command_args.batch_prediction_attributes}
for resource_type, attributes_file in json_attribute_options.items():
if attributes_file is not None:
command_args.json_args[resource_type] = u.read_json(
attributes_file)
else:
command_args.json_args[resource_type] = {}
# Parses dataset generators in json format if provided
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
# Parses multi-dataset attributes in json such as field maps
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(command_args.multi_dataset_attributes)
command_args.multi_dataset_json= multi_dataset_json
else:
command_args.multi_dataset_json = {}
dataset_ids = None
command_args.dataset_ids = []
# Parses dataset/id if provided.
if command_args.datasets:
dataset_ids = u.read_datasets(command_args.datasets)
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
test_dataset_ids = None
command_args.test_dataset_ids = []
# Parses dataset/id if provided.
if command_args.test_datasets:
test_dataset_ids = u.read_datasets(command_args.test_datasets)
command_args.test_dataset_ids = test_dataset_ids
# Retrieve dataset/ids if provided.
if command_args.dataset_tag:
dataset_ids = dataset_ids.extend(
u.list_ids(api.list_datasets,
"tags__in=%s" % command_args.dataset_tag))
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Reads a json filter if provided.
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
# Reads a lisp filter if provided.
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
# Adds default tags unless that it is requested not to do so.
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
# Checks combined votes method
if (command_args.method and command_args.method != COMBINATION_LABEL and
not (command_args.method in COMBINATION_WEIGHTS.keys())):
command_args.method = 0
else:
combiner_methods = dict([[value, key]
for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
# Checks missing_strategy
if (command_args.missing_strategy and
not (command_args.missing_strategy in MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
# Adds replacement=True if creating ensemble and nothing is specified
if (command_args.number_of_models > 1 and
not command_args.replacement and
not '--no-replacement' in flags and
not 'replacement' in user_defaults and
not '--no-randomize' in flags and
not 'randomize' in user_defaults and
not '--sample-rate' in flags and
not 'sample_rate' in user_defaults):
command_args.replacement = True
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print "WARNING: 'full data' is a deprecated value. Use 'full' instead"
command_args.prediction_info = FULL_FORMAT
# Parses class, weight pairs for objective weight
if command_args.objective_weights:
objective_weights = (
u.read_objective_weights(command_args.objective_weights))
command_args.objective_weights_json = objective_weights
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(
command_args.args_separator)
# Sets shared_flag if --shared or --unshared has been used
if '--shared' in flags or '--unshared' in flags:
command_args.shared_flag = True
else:
command_args.shared_flag = False
def transform_args(command_args, flags, api, user_defaults):
"""Transforms the formatted argument strings into structured arguments
"""
attribute_args(command_args)
# Parses dataset generators in json format if provided
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
# Parses multi-dataset attributes in json such as field maps
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
transform_dataset_options(command_args, api)
# Reads a json filter if provided.
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
# Reads a lisp filter if provided.
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
# Adds default tags unless it is requested not to do so.
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
# Checks combined votes method
try:
if (command_args.method and command_args.method != COMBINATION_LABEL
and not (command_args.method in COMBINATION_WEIGHTS.keys())):
command_args.method = 0
else:
combiner_methods = dict(
[[value, key] for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
except AttributeError:
pass
# Checks missing_strategy
try:
if (command_args.missing_strategy and
not (command_args.missing_strategy in
MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
except AttributeError:
pass
# Adds replacement=True if creating ensemble and nothing is specified
try:
if (command_args.number_of_models > 1 and
not command_args.replacement and
not '--no-replacement' in flags and
not 'replacement' in user_defaults and
not '--no-randomize' in flags and
not 'randomize' in user_defaults and
not '--sample-rate' in flags and
not 'sample_rate' in user_defaults):
command_args.replacement = True
except AttributeError:
pass
try:
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print ("WARNING: 'full data' is a deprecated value. Use"
" 'full' instead")
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
# Parses class, weight pairs for objective weight
try:
if command_args.objective_weights:
objective_weights = (
u.read_objective_weights(command_args.objective_weights))
command_args.objective_weights_json = objective_weights
except AttributeError:
pass
try:
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(
command_args.args_separator)
except AttributeError:
pass
# Sets shared_flag if --shared or --unshared has been used
if '--shared' in flags or '--unshared' in flags:
command_args.shared_flag = True
else:
command_args.shared_flag = False
# Set remote on if scoring a trainind dataset in bigmler anomaly
try:
if command_args.score:
command_args.remote = True
if not "--prediction-info" in flags:
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
command_args.has_models_ = (
(hasattr(command_args, 'model') and command_args.model) or
(hasattr(command_args, 'models') and command_args.models) or
(hasattr(command_args, 'ensemble') and command_args.ensemble) or
(hasattr(command_args, 'ensembles') and command_args.ensembles) or
(hasattr(command_args, 'cluster') and command_args.cluster) or
(hasattr(command_args, 'clusters') and command_args.clusters) or
(hasattr(command_args, 'model_tag') and command_args.model_tag) or
(hasattr(command_args, 'anomaly') and command_args.anomaly) or
(hasattr(command_args, 'anomalies') and command_args.anomalies) or
(hasattr(command_args, 'ensemble_tag')
and command_args.ensemble_tag) or
(hasattr(command_args, 'cluster_tag') and command_args.cluster_tag) or
(hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag))
command_args.has_datasets_ = (
(hasattr(command_args, 'dataset') and command_args.dataset) or
(hasattr(command_args, 'datasets') and command_args.datasets) or
(hasattr(command_args, 'dataset_tag') and command_args.dataset_tag))
command_args.has_test_datasets_ = (
(hasattr(command_args, 'test_dataset') and
command_args.test_dataset) or
(hasattr(command_args, 'test_datasets') and
command_args.test_datasets) or
(hasattr(command_args, 'test_dataset_tag') and
command_args.test_dataset_tag))
def transform_args(command_args, flags, api, user_defaults):
"""Transforms the formatted argument strings into structured arguments
"""
# Parses attributes in json format if provided
command_args.json_args = {}
json_attribute_options = {
'source': command_args.source_attributes,
'dataset': command_args.dataset_attributes,
'model': command_args.model_attributes,
'ensemble': command_args.ensemble_attributes,
'evaluation': command_args.evaluation_attributes,
'batch_prediction': command_args.batch_prediction_attributes
}
for resource_type, attributes_file in json_attribute_options.items():
if attributes_file is not None:
command_args.json_args[resource_type] = u.read_json(
attributes_file)
else:
command_args.json_args[resource_type] = {}
# Parses dataset generators in json format if provided
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
# Parses multi-dataset attributes in json such as field maps
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
dataset_ids = None
command_args.dataset_ids = []
# Parses dataset/id if provided.
if command_args.datasets:
dataset_ids = u.read_datasets(command_args.datasets)
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Retrieve dataset/ids if provided.
if command_args.dataset_tag:
dataset_ids = dataset_ids.extend(
u.list_ids(api.list_datasets,
"tags__in=%s" % command_args.dataset_tag))
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Reads a json filter if provided.
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
# Reads a lisp filter if provided.
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
# Adds default tags unless that it is requested not to do so.
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
# Checks combined votes method
if (command_args.method and command_args.method != COMBINATION_LABEL
and not (command_args.method in COMBINATION_WEIGHTS.keys())):
command_args.method = 0
else:
combiner_methods = dict([[value, key]
for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
# Checks missing_strategy
if (command_args.missing_strategy and
not (command_args.missing_strategy in MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
# Adds replacement=True if creating ensemble and nothing is specified
if (command_args.number_of_models > 1 and not command_args.replacement
and not '--no-replacement' in flags
and not 'replacement' in user_defaults
and not '--no-randomize' in flags
and not 'randomize' in user_defaults
and not '--sample-rate' in flags
and not 'sample_rate' in user_defaults):
command_args.replacement = True
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print "WARNING: 'full data' is a deprecated value. Use 'full' instead"
command_args.prediction_info = FULL_FORMAT
# Parses class, weight pairs for objective weight
if command_args.objective_weights:
objective_weights = (u.read_objective_weights(
command_args.objective_weights))
command_args.objective_weights_json = objective_weights
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(',')
def transform_args(command_args, flags, api):
"""Transforms the formatted argument strings into structured arguments
"""
attribute_args(command_args)
# Parses dataset generators in json format if provided
try:
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
except AttributeError:
pass
# Parses multi-dataset attributes in json such as field maps
try:
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(
command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
except AttributeError:
pass
transform_dataset_options(command_args, api)
script_ids = None
command_args.script_ids = []
# Parses script/id if provided.
try:
if command_args.scripts:
script_ids = u.read_resources(command_args.scripts)
if len(script_ids) == 1:
command_args.script = script_ids[0]
command_args.script_ids = script_ids
except AttributeError:
pass
# Retrieve script/ids if provided.
try:
if command_args.script_tag:
script_ids = script_ids.extend(
u.list_ids(api.list_scripts,
"tags__in=%s" % command_args.script_tag))
if len(script_ids) == 1:
command_args.script = script_ids[0]
command_args.script_ids = script_ids
except AttributeError:
pass
# Reads a json filter if provided.
try:
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
except AttributeError:
pass
# Reads a lisp filter if provided.
try:
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
except AttributeError:
pass
# Adds default tags unless that it is requested not to do so.
try:
if command_args.no_tag:
command_args.tag.append('BigMLer')
command_args.tag.append('BigMLer_%s' % NOW)
except AttributeError:
pass
# Checks combined votes method
try:
if (command_args.method and command_args.method != COMBINATION_LABEL
and not command_args.method in COMBINATION_WEIGHTS.keys()):
command_args.method = 0
else:
combiner_methods = dict(
[[value, key] for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
except AttributeError:
pass
# Checks missing_strategy
try:
if (command_args.missing_strategy and
not (command_args.missing_strategy in
MISSING_STRATEGIES.keys())):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(
command_args.missing_strategy, 0)
except AttributeError:
pass
try:
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == 'full data':
print ("WARNING: 'full data' is a deprecated value. Use"
" 'full' instead")
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
# Parses class, weight pairs for objective weight
try:
if command_args.objective_weights:
objective_weights = (
u.read_objective_weights(command_args.objective_weights))
command_args.objective_weights_json = objective_weights
except AttributeError:
pass
try:
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(
command_args.args_separator)
except AttributeError:
pass
# Sets shared_flag if --shared or --unshared has been used
command_args.shared_flag = '--shared' in flags or '--unshared' in flags
# Set remote on if scoring a trainind dataset in bigmler anomaly
try:
if command_args.score:
command_args.remote = True
if not "--prediction-info" in flags:
command_args.prediction_info = FULL_FORMAT
except AttributeError:
pass
command_args.has_supervised_ = (
(hasattr(command_args, 'model') and command_args.model) or
(hasattr(command_args, 'models') and command_args.models) or
(hasattr(command_args, 'ensemble') and command_args.ensemble) or
(hasattr(command_args, 'ensembles') and command_args.ensembles) or
(hasattr(command_args, 'model_tag') and command_args.model_tag) or
(hasattr(command_args, 'logistic_regression') and
command_args.logistic_regression) or
(hasattr(command_args, 'logistic_regressions') and
command_args.logistic_regressions) or
(hasattr(command_args, 'logistic_regression_tag') and
command_args.logistic_regression_tag) or
(hasattr(command_args, 'deepnet') and
command_args.deepnet) or
(hasattr(command_args, 'deepnets') and
command_args.deepnets) or
(hasattr(command_args, 'deepnet_tag') and
command_args.deepnet_tag) or
(hasattr(command_args, 'ensemble_tag')
and command_args.ensemble_tag))
command_args.has_models_ = (
command_args.has_supervised_ or
(hasattr(command_args, 'cluster') and command_args.cluster) or
(hasattr(command_args, 'clusters') and command_args.clusters) or
(hasattr(command_args, 'anomaly') and command_args.anomaly) or
(hasattr(command_args, 'anomalies') and command_args.anomalies) or
(hasattr(command_args, 'cluster_tag') and command_args.cluster_tag) or
(hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag))
command_args.has_datasets_ = (
(hasattr(command_args, 'dataset') and command_args.dataset) or
(hasattr(command_args, 'datasets') and command_args.datasets) or
(hasattr(command_args, 'dataset_tag') and command_args.dataset_tag))
command_args.has_test_datasets_ = (
(hasattr(command_args, 'test_dataset') and
command_args.test_dataset) or
(hasattr(command_args, 'test_datasets') and
command_args.test_datasets) or
(hasattr(command_args, 'test_dataset_tag') and
command_args.test_dataset_tag))
def transform_args(command_args, flags, api, user_defaults):
"""Transforms the formatted argument strings into structured arguments
"""
# Parses attributes in json format if provided
command_args.json_args = {}
for resource_type in RESOURCE_TYPES:
attributes_file = getattr(command_args, "%s_attributes" % resource_type, None)
if attributes_file is not None:
command_args.json_args[resource_type] = u.read_json(attributes_file)
else:
command_args.json_args[resource_type] = {}
# Parses dataset generators in json format if provided
if command_args.new_fields:
json_generators = u.read_json(command_args.new_fields)
command_args.dataset_json_generators = json_generators
else:
command_args.dataset_json_generators = {}
# Parses multi-dataset attributes in json such as field maps
if command_args.multi_dataset_attributes:
multi_dataset_json = u.read_json(command_args.multi_dataset_attributes)
command_args.multi_dataset_json = multi_dataset_json
else:
command_args.multi_dataset_json = {}
dataset_ids = None
command_args.dataset_ids = []
# Parses dataset/id if provided.
if command_args.datasets:
dataset_ids = u.read_datasets(command_args.datasets)
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
test_dataset_ids = None
command_args.test_dataset_ids = []
# Parses dataset/id if provided.
if command_args.test_datasets:
test_dataset_ids = u.read_datasets(command_args.test_datasets)
command_args.test_dataset_ids = test_dataset_ids
# Retrieve dataset/ids if provided.
if command_args.dataset_tag:
dataset_ids = dataset_ids.extend(u.list_ids(api.list_datasets, "tags__in=%s" % command_args.dataset_tag))
if len(dataset_ids) == 1:
command_args.dataset = dataset_ids[0]
command_args.dataset_ids = dataset_ids
# Reads a json filter if provided.
if command_args.json_filter:
json_filter = u.read_json_filter(command_args.json_filter)
command_args.json_filter = json_filter
# Reads a lisp filter if provided.
if command_args.lisp_filter:
lisp_filter = u.read_lisp_filter(command_args.lisp_filter)
command_args.lisp_filter = lisp_filter
# Adds default tags unless that it is requested not to do so.
if command_args.no_tag:
command_args.tag.append("BigMLer")
command_args.tag.append("BigMLer_%s" % NOW)
# Checks combined votes method
try:
if (
command_args.method
and command_args.method != COMBINATION_LABEL
and not (command_args.method in COMBINATION_WEIGHTS.keys())
):
command_args.method = 0
else:
combiner_methods = dict([[value, key] for key, value in COMBINER_MAP.items()])
combiner_methods[COMBINATION_LABEL] = COMBINATION
command_args.method = combiner_methods.get(command_args.method, 0)
except AttributeError:
pass
# Checks missing_strategy
try:
if command_args.missing_strategy and not (command_args.missing_strategy in MISSING_STRATEGIES.keys()):
command_args.missing_strategy = 0
else:
command_args.missing_strategy = MISSING_STRATEGIES.get(command_args.missing_strategy, 0)
except AttributeError:
pass
# Adds replacement=True if creating ensemble and nothing is specified
try:
if (
command_args.number_of_models > 1
and not command_args.replacement
and not "--no-replacement" in flags
and not "replacement" in user_defaults
and not "--no-randomize" in flags
and not "randomize" in user_defaults
and not "--sample-rate" in flags
and not "sample_rate" in user_defaults
):
command_args.replacement = True
except AttributeError:
pass
# Old value for --prediction-info='full data' maps to 'full'
if command_args.prediction_info == "full data":
print "WARNING: 'full data' is a deprecated value. Use 'full' instead"
command_args.prediction_info = FULL_FORMAT
# Parses class, weight pairs for objective weight
try:
if command_args.objective_weights:
objective_weights = u.read_objective_weights(command_args.objective_weights)
command_args.objective_weights_json = objective_weights
except AttributeError:
pass
try:
command_args.multi_label_fields_list = []
if command_args.multi_label_fields is not None:
multi_label_fields = command_args.multi_label_fields.strip()
command_args.multi_label_fields_list = multi_label_fields.split(command_args.args_separator)
except AttributeError:
pass
# Sets shared_flag if --shared or --unshared has been used
if "--shared" in flags or "--unshared" in flags:
command_args.shared_flag = True
else:
command_args.shared_flag = False
command_args.has_models_ = (
(hasattr(command_args, "model") and command_args.model)
or (hasattr(command_args, "models") and command_args.models)
or (hasattr(command_args, "ensemble") and command_args.ensemble)
or (hasattr(command_args, "ensembles") and command_args.ensembles)
or (hasattr(command_args, "cluster") and command_args.cluster)
or (hasattr(command_args, "clusters") and command_args.clusters)
or (hasattr(command_args, "model_tag") and command_args.model_tag)
or (hasattr(command_args, "anomaly") and command_args.anomaly)
or (hasattr(command_args, "anomalies") and command_args.anomalies)
or (hasattr(command_args, "ensemble_tag") and command_args.ensemble_tag)
or (hasattr(command_args, "cluster_tag") and command_args.cluster_tag)
or (hasattr(command_args, "anomaly_tag") and command_args.anomaly_tag)
)
command_args.has_datasets_ = (
(hasattr(command_args, "dataset") and command_args.dataset)
or (hasattr(command_args, "datasets") and command_args.datasets)
or (hasattr(command_args, "dataset_tag") and command_args.dataset_tag)
)
