def set_pca_args(args, name=None, fields=None, pca_fields=None):
"""Return pca arguments dict
"""
if name is None:
name = args.name
if pca_fields is None:
pca_fields = args.pca_fields_
pca_args = set_basic_args(args, name)
pca_args.update({
"seed": SEED if args.seed is None else args.seed,
"pca_seed": SEED if args.seed is None else args.seed
})
pca_args.update({"sample_rate": args.sample_rate})
pca_args = update_sample_parameters_args( \
pca_args, args)
if fields is not None:
input_fields = fields.fields.keys()
if pca_fields and fields is not None:
input_fields = configure_input_fields(fields, pca_fields)
if args.exclude_objective:
input_fields = [field for field in input_fields \
if field not in args.exclude_fields]
pca_args.update(input_fields=input_fields)
if 'pca' in args.json_args:
update_json_args(pca_args, args.json_args.get('pca'), fields)
return pca_args
def set_anomaly_args(args, name=None, fields=None, anomaly_fields=None):
"""Return anomaly arguments dict
"""
if name is None:
name = args.name
if anomaly_fields is None:
anomaly_fields = args.anomaly_fields_
anomaly_args = set_basic_model_args(args, name)
anomaly_args.update({
"seed":
SEED if args.seed is None else args.seed,
"anomaly_seed":
(SEED if args.anomaly_seed is None else args.anomaly_seed)
})
if anomaly_fields and fields is not None:
input_fields = configure_input_fields(fields, anomaly_fields)
anomaly_args.update(input_fields=input_fields)
if args.top_n > 0:
anomaly_args.update(top_n=args.top_n)
if args.forest_size > 0:
anomaly_args.update(forest_size=args.forest_size)
anomaly_args = update_sample_parameters_args(anomaly_args, args)
if 'anomaly' in args.json_args:
update_json_args(anomaly_args, args.json_args.get('anomaly'), fields)
return anomaly_args
def set_association_args(args, name=None, fields=None,
association_fields=None):
"""Return association arguments dict
"""
if name is None:
name = args.name
if association_fields is None:
association_fields = args.association_fields_
association_args = set_basic_model_args(args, name)
if association_fields and fields is not None:
input_fields = configure_input_fields(fields, association_fields)
association_args.update(input_fields=input_fields)
if args.association_k:
association_args.update({"max_k": args.association_k})
if args.search_strategy:
association_args.update({"search_strategy": args.search_strategy})
association_args = update_sample_parameters_args(association_args, args)
if 'association' in args.json_args:
update_json_args(association_args,
args.json_args.get('association'), fields)
return association_args
def set_cluster_args(args, name=None, fields=None,
cluster_fields=None):
"""Return cluster arguments dict
"""
if name is None:
name = args.name
if cluster_fields is None:
cluster_fields = args.cluster_fields_
cluster_args = set_basic_model_args(args, name)
cluster_args.update({
"seed": SEED if args.seed is None else args.seed,
"cluster_seed": (SEED if args.cluster_seed is None
else args.cluster_seed)
})
if args.cluster_models is not None:
cluster_args.update({"model_clusters": True})
if args.cluster_k:
cluster_args.update({"k": args.cluster_k})
if cluster_fields and fields is not None:
input_fields = configure_input_fields(fields, cluster_fields)
cluster_args.update(input_fields=input_fields)
if args.summary_fields is not None:
cluster_args.update({"summary_fields": args.summary_fields_})
cluster_args = update_sample_parameters_args(cluster_args, args)
if 'cluster' in args.json_args:
update_json_args(cluster_args, args.json_args.get('cluster'), fields)
return cluster_args
def set_logistic_regression_args(args,
name=None,
fields=None,
objective_id=None,
logistic_regression_fields=None):
"""Return logistic regression arguments dict
"""
if name is None:
name = args.name
if logistic_regression_fields is None:
logistic_regression_fields = args.logistic_regression_fields_
if objective_id is None:
objective_id = args.objective_id_
logistic_regression_args = set_basic_model_args(args, name)
logistic_regression_args.update(
{"seed": SEED if args.seed is None else args.seed})
if objective_id is not None and fields is not None:
logistic_regression_args.update({"objective_field": objective_id})
if logistic_regression_fields and fields is not None:
input_fields = configure_input_fields(fields,
logistic_regression_fields)
logistic_regression_args.update(input_fields=input_fields)
if ((args.evaluate and args.test_split == 0 and args.test_datasets is None)
or args.cross_validation_rate > 0):
logistic_regression_args.update(seed=SEED)
if args.cross_validation_rate > 0:
args.sample_rate = 1 - args.cross_validation_rate
args.replacement = False
elif (args.sample_rate == 1 and args.test_datasets is None
and not args.dataset_off):
args.sample_rate = EVALUATE_SAMPLE_RATE
logistic_regression_args.update({"sample_rate": args.sample_rate})
if args.lr_c:
logistic_regression_args.update({"c": args.lr_c})
logistic_regression_args.update({"bias": args.bias})
logistic_regression_args.update( \
{"balance_fields": args.balance_fields})
if args.eps:
logistic_regression_args.update({"eps": args.eps})
if args.normalize is not None:
logistic_regression_args.update({"normalize": args.normalize})
if args.missing_numerics is not None:
logistic_regression_args.update( \
{"missing_numerics": args.missing_numerics})
if args.field_codings is not None:
logistic_regression_args.update(\
{"field_codings": args.field_codings_})
logistic_regression_args = update_sample_parameters_args( \
logistic_regression_args, args)
if 'logistic_regression' in args.json_args:
update_json_args(logistic_regression_args,
args.json_args.get('logistic_regression'), fields)
return logistic_regression_args
def set_topic_model_args(args,
name=None,
fields=None,
topic_model_fields=None):
"""Return topic_model arguments dict
"""
if name is None:
name = args.name
if topic_model_fields is None:
topic_model_fields = args.topic_model_fields_
topic_model_args = set_basic_args(args, name)
topic_model_args.update({
"seed":
SEED if args.seed is None else args.seed,
"topicmodel_seed":
SEED if args.seed is None else args.seed
})
if topic_model_fields and fields is not None:
input_fields = configure_input_fields(fields, topic_model_fields)
topic_model_args.update(input_fields=input_fields)
topic_model_args.update({"sample_rate": args.sample_rate})
topic_model_args.update({"bigrams": args.bigrams})
topic_model_args.update({"case_sensitive": args.case_sensitive})
if args.number_of_topics is not None:
topic_model_args.update({"number_of_topics": args.number_of_topics})
if args.term_limit is not None:
topic_model_args.update({"term_limit": args.term_limit})
if args.top_n_terms is not None:
topic_model_args.update({"top_n_terms": args.top_n_terms})
if args.minimum_name_terms is not None:
topic_model_args.update(
{"minimum_name_terms": args.minimum_name_terms})
if args.excluded_terms:
topic_model_args.update({"excluded_terms": args.excluded_terms_})
topic_model_args = update_sample_parameters_args( \
topic_model_args, args)
if 'topic_model' in args.json_args:
update_json_args(topic_model_args, args.json_args.get('topic_model'),
fields)
return topic_model_args
if args.weight_field:
try:
weight_field = fields.field_id(args.weight_field)
except ValueError, exc:
sys.exit(exc)
model_args.update(weight_field=weight_field)
if args.objective_weights:
model_args.update(objective_weights=args.objective_weights_json)
if args.max_categories > 0:
model_args.update(
user_metadata={'other_label': other_label,
'max_categories': args.max_categories})
model_args = update_sample_parameters_args(model_args, args)
if 'model' in args.json_args:
update_json_args(model_args, args.json_args.get('model'), fields)
return model_args
def set_label_model_args(args, fields, labels, multi_label_data):
"""Set of args needed to build a model per label
"""
objective_field = args.objective_field
if not args.model_fields_:
model_fields = []
if args.balance:
ensemble_args.update(balance_objective=True)
if args.weight_field:
try:
weight_field = fields.field_id(args.weight_field)
except ValueError, exc:
sys.exit(exc)
ensemble_args.update(weight_field=weight_field)
if args.objective_weights:
ensemble_args.update(objective_weights=args.objective_weights_json)
if args.random_candidates:
ensemble_args.update(random_candidates=args.random_candidates)
update_attributes(ensemble_args, args.json_args.get('model'))
ensemble_args = update_sample_parameters_args(ensemble_args, args)
ensemble_args["ensemble_sample"].update( \
{"rate": args.ensemble_sample_rate,
"replacement": args.ensemble_sample_replacement})
if 'ensemble' in args.json_args:
update_json_args(ensemble_args, args.json_args.get('ensemble'), fields)
return ensemble_args
def create_ensembles(datasets, ensemble_ids, ensemble_args, args,
number_of_ensembles=1,
api=None, path=None, session_file=None, log=None):
"""Create ensembles from input data
def set_deepnet_args(args,
name=None,
fields=None,
objective_id=None,
deepnet_fields=None):
"""Return deepnet arguments dict
"""
if name is None:
name = args.name
if deepnet_fields is None:
deepnet_fields = args.deepnet_fields_
if objective_id is None:
objective_id = args.objective_id_
deepnet_args = set_basic_model_args(args, name)
deepnet_args.update({"seed": SEED if args.seed is None else args.seed})
if objective_id is not None and fields is not None:
deepnet_args.update({"objective_field": objective_id})
if deepnet_fields and fields is not None:
input_fields = configure_input_fields(fields, deepnet_fields)
deepnet_args.update(input_fields=input_fields)
if ((args.evaluate and args.test_split == 0 and args.test_datasets is None)
or args.cross_validation_rate > 0):
deepnet_args.update(seed=SEED)
if args.cross_validation_rate > 0:
args.sample_rate = 1 - args.cross_validation_rate
args.replacement = False
elif (args.sample_rate == 1 and args.test_datasets is None
and not args.dataset_off):
args.sample_rate = EVALUATE_SAMPLE_RATE
deepnet_args.update({"sample_rate": args.sample_rate})
if args.batch_normalization is not None:
deepnet_args.update({"batch_normalization": args.batch_normalization})
if args.dropout_rate:
deepnet_args.update({"dropout_rate": args.dropout_rate})
if args.hidden_layers is not None:
deepnet_args.update({"hidden_layers": args.hidden_layers_})
if args.learn_residuals is not None:
deepnet_args.update( \
{"learn_residuals": args.learn_residuals})
if args.max_iterations is not None:
deepnet_args.update(\
{"learning_rate": args.learning_rate})
if args.max_training_time is not None:
deepnet_args.update(\
{"max_training_time": args.max_training_time})
if args.number_of_hidden_layers is not None:
deepnet_args.update(\
{"number_of_hidden_layers": args.number_of_hidden_layers})
if args.number_of_model_candidates is not None:
deepnet_args.update(\
{"number_of_model_candidates": args.number_of_model_candidates})
if args.search is not None:
deepnet_args.update(\
{"search": args.search})
if args.suggest_structure is not None:
deepnet_args.update(\
{"suggest_structure": args.suggest_structure})
if not args.missing_numerics:
deepnet_args.update(\
{"missing_numerics": args.missing_numerics})
if args.tree_embedding:
deepnet_args.update(\
{"tree_embedding": args.tree_embedding})
deepnet_args = update_sample_parameters_args( \
deepnet_args, args)
if 'deepnet' in args.json_args:
update_json_args(deepnet_args, args.json_args.get('deepnet'), fields)
return deepnet_args