def create_kfold_datasets_file(args, api, common_options, resume=False):
"""Create the kfold dataset resources and store their ids in a file
one per line
"""
message = ('Creating the kfold datasets............\n')
u.log_message(message, log_file=session_file, console=args.verbosity)
if args.output_dir is None:
args.output_dir = a.NOW
# retrieve dataset
dataset_id = bigml.api.get_dataset_id(args.dataset)
if dataset_id:
dataset = api.check_resource(dataset_id, api.get_dataset)
# check that kfold_field is unique
fields = Fields(dataset, {"objective_field": args.objective_field,
"objective_field_present": True})
objective_id = fields.field_id(fields.objective_field)
kfold_field_name = avoid_duplicates(DEFAULT_KFOLD_FIELD, fields)
# create jsons to generate partial datasets
selecting_file_list, resume = create_kfold_json(args, kfold_field_name,
objective_id,
resume=resume)
# generate test datasets
datasets_file, resume = create_kfold_datasets(dataset_id, args,
selecting_file_list,
fields.objective_field,
kfold_field_name,
common_options,
resume=resume)
return datasets_file, fields.field_column_number(objective_id), resume
return None, None, None
def update_with_summary_file(step, resource, summary_file):
fields = Fields(resource)
changes = fields.filter_fields_update( \
fields.new_fields_structure(res_filename(summary_file)))
resource_type = get_resource_type(resource)
resource = world.api.updaters[resource_type](resource, changes)
world.api.ok(resource)
setattr(world, resource_type, resource)
def best_first_search(datasets_file,
api,
args,
common_options,
staleness=None,
penalty=None,
objective_name=None,
resume=False):
"""Selecting the fields to be used in the model construction
"""
counter = 0
loop_counter = 0
features_file = os.path.normpath(
os.path.join(args.output_dir, FEATURES_LOG))
with open(features_file, u.open_mode("w")) as features_handler:
features_writer = csv.writer(features_handler, lineterminator="\n")
features_writer.writerow(
["step", "state", "score", "metric_value", "best_score"])
features_handler.flush()
if staleness is None:
staleness = DEFAULT_STALENESS
if penalty is None:
penalty = DEFAULT_PENALTY
# retrieving the first dataset in the file
try:
with open(datasets_file, u.open_mode("r")) as datasets_handler:
dataset_id = datasets_handler.readline().strip()
except IOError, exc:
sys.exit("Could not read the generated datasets file: %s" %
str(exc))
try:
stored_dataset = u.storage_file_name(args.output_dir, dataset_id)
with open(stored_dataset, u.open_mode("r")) as dataset_handler:
dataset = json.loads(dataset_handler.read())
except IOError:
dataset = api.check_resource(dataset_id,
query_string=ALL_FIELDS_QS)
# initial feature set
fields = Fields(dataset)
excluded_features = ([] if args.exclude_features is None else
args.exclude_features.split(args.args_separator))
try:
excluded_ids = [
fields.field_id(feature) for feature in excluded_features
]
objective_id = fields.field_id(objective_name)
except ValueError, exc:
sys.exit(exc)
def pca_processing(datasets, pca, \
pca_ids, api, args, resume, fields=None, \
session_file=None, path=None, log=None):
"""Creates or retrieves pca from the input data
"""
# If we have a dataset but not a model, we create the model if the no_model
# flag hasn't been set up.
if datasets and not (has_pca(args) or \
args.no_pca):
pca_ids = []
pcas = []
# Only 1 pca per bigmler command at present
number_of_pcas = 1
if resume:
resume, pca_ids = c.checkpoint( \
c.are_pcas_created, path, \
number_of_pcas, debug=args.debug)
if not resume:
message = u.dated("Found %s pcas out of %s."
" Resuming.\n" %
(len(pca_ids), number_of_pcas))
u.log_message(message,
log_file=session_file,
console=args.verbosity)
pcas = pca_ids
number_of_pcas -= len(pca_ids)
args.exclude_fields = []
if args.exclude_objective:
dataset = datasets[0]
fields = Fields(dataset)
objective_id = \
fields.fields_by_column_number[fields.objective_field]
args.exclude_fields = [objective_id]
pca_args = r.set_pca_args( \
args, fields=fields, \
pca_fields=args.pca_fields_)
pca = \
r.create_pca( \
datasets, pca, pca_args, \
args, api, path, session_file, log)
# If a pca is provided, we use it.
elif args.pca:
pca_ids = [args.pca]
pca = pca_ids[0]
elif args.pca or args.pca_tag:
pca = pca_ids[0]
# If we are going to create projections, we must retrieve the pca
if pca_ids and (args.test_set or args.export_fields):
pca = \
r.get_pca(pca, args, api, session_file)
return pca, resume
def read_local_resource(path, csv_properties=None):
"""Read the JSON resource structure information from the given file.
"""
resource = empty_resource()
if csv_properties is None:
csv_properties = {}
fields = None
open_mode = "rt" if PYTHON3 else "rb"
with open(path, open_mode) as resource_file:
try:
resource = json.loads(resource_file.read())
except IOError:
pass
resource_id = resource.get('resource')
if resource_id is None:
sys.exit("Failed to extract a BigML resource structure from the"
" contents of file %s." % path)
if resource.get('object') is None:
resource = {
'resource': resource_id,
'object': resource,
'error': None,
'code': bigml.api.HTTP_OK
}
fields, resource_locale, missing_tokens = get_fields_structure(resource)
if missing_tokens:
csv_properties['missing_tokens'] = missing_tokens
if resource_locale:
csv_properties['data_locale'] = resource_locale
if fields:
fields = Fields(resource, **csv_properties)
return resource, csv_properties, fields
def get_input_fields(resource, referrer=None):
"""New list of input fields
"""
if referrer is None:
referrer = {}
input_fields_ids = resource.get('input_fields', [])
if referrer:
referrer_fields = Fields({
'resource': referrer['resource'],
'object': referrer
})
referrer_fields_ids = referrer_fields.fields.keys()
# case where objective field is not in input fields
# check whether the resource has an objective field not included in
# the input fields list
resource_type = get_resource_type(resource)
if resource_type == 'model':
objective_id = resource.get('objective_field')
try:
objective_id = objective_id.get('id')
except AttributeError:
pass
if objective_id not in input_fields_ids:
input_fields_ids.append(objective_id)
if input_fields_ids.sort() == referrer_fields_ids.sort():
return []
return input_fields_ids
def get_model_fields(model, model_fields, csv_properties, args):
"""Retrieves fields info from model resource
"""
if not csv_properties:
csv_properties = {}
csv_properties.update(verbose=True)
if args.user_locale is None:
args.user_locale = model['object'].get('locale', None)
csv_properties.update(data_locale=args.user_locale)
if 'model_fields' in model['object']['model']:
model_fields = model['object']['model']['model_fields'].keys()
csv_properties.update(include=model_fields)
if 'missing_tokens' in model['object']['model']:
missing_tokens = model['object']['model']['missing_tokens']
else:
missing_tokens = MISSING_TOKENS
csv_properties.update(missing_tokens=missing_tokens)
objective_field = model['object']['objective_fields']
if isinstance(objective_field, list):
objective_field = objective_field[0]
csv_properties.update(objective_field=objective_field)
fields = Fields(model['object']['model']['fields'], **csv_properties)
return fields, objective_field
def get_fields_structure(resource, csv_properties):
"""Builds a Fields object from the fields information in the resource
"""
if not csv_properties and 'locale' in resource['object']:
csv_properties = {'data_locale': resource['object']['locale']}
fields = Fields(resource['object']['fields'], **csv_properties)
return fields
def fusion_processing(fusion, \
fusion_ids, api, args, resume, fields=None, \
session_file=None, path=None, log=None):
"""Creates or retrieves fusion from the input data
"""
# If we have a models' list but not a fusion,
# we create the model if the no_model
# flag hasn't been set up.
if args.fusion_models_ is not None and not has_fusion(args):
fusion_ids = []
# Only 1 fusion per bigmler command at present
number_of_fusions = 1
if resume:
resume, fusion_ids = c.checkpoint( \
c.are_fusions_created, path, \
number_of_fusions, debug=args.debug)
if not resume:
message = u.dated("Found %s fusions out of %s."
" Resuming.\n" %
(len(fusion_ids), number_of_fusions))
u.log_message(message,
log_file=session_file,
console=args.verbosity)
fusion = fusion_ids[0]
first_model_id = api.get_fusion(fusion)[ \
"object"]["fusion"]["models"][0]["id"]
first_model_kind = api.get_fusion(fusion)[ \
"object"]["fusion"]["models"][0]["kind"]
first_model = api.getters[first_model_kind](first_model_id)
fields = Fields(first_model)
number_of_fusions -= len(fusion_ids)
fusion_args = r.set_fusion_args( \
args, fields)
fusion = \
r.create_fusion( \
args.fusion_models_, fusion, fusion_args, \
args, api, path, session_file, log)
# If a fusion is provided, we use it.
elif args.fusion:
fusion_ids = [args.fusion]
fusion = fusion_ids[0]
elif args.fusion or args.fusion_tag:
fusion = fusion_ids[0]
# If we are going to create predictions, we must retrieve the fusion
if fusion_ids and args.test_set:
fusion = \
r.get_fusion(fusion, args, api, session_file)
args.objective_field = fusion['object']['objective_field_name']
return fusion, resume
def create_kfold_datasets_file(args, api, common_options, resume=False):
"""Create the kfold dataset resources and store their ids in a file
one per line
"""
message = ('Creating the kfold datasets............\n')
u.log_message(message, log_file=session_file, console=args.verbosity)
if args.output_dir is None:
args.output_dir = a.NOW
# retrieve dataset
dataset_id = bigml.api.get_dataset_id(args.dataset)
if dataset_id:
dataset = api.check_resource(dataset_id)
try:
args.objective_field = int(args.objective_field)
except (TypeError, ValueError):
pass
# if the user provided no objective field, try to use the one in the
# dataset
if args.objective_field is None:
try:
args.objective_field = dataset['object'][
'objective_field']['column_number']
except KeyError:
pass
# check that kfold_field is unique
fields = Fields(dataset, objective_field=args.objective_field,
objective_field_present=True)
try:
objective_id = fields.field_id(fields.objective_field)
objective_name = fields.field_name(objective_id)
except ValueError, exc:
sys.exit(exc)
kfold_field_name = avoid_duplicates(DEFAULT_KFOLD_FIELD, fields)
# create jsons to generate partial datasets
selecting_file_list, resume = create_kfold_json(args, kfold_field_name,
objective_id,
resume=resume)
# generate test datasets
datasets_file, resume = create_kfold_datasets(dataset_id, args,
selecting_file_list,
objective_name,
common_options,
resume=resume)
return datasets_file, objective_name, resume
def get_input_fields(resource, referrer=None):
"""New list of input fields
"""
if referrer is None:
referrer = {}
input_fields_ids = resource.get('input_fields', [])
if referrer:
referrer_input_fields = [[]]
# compare fields by name
resource_fields = Fields({
'resource': resource['resource'],
'object': resource
})
referrer_fields = Fields({
'resource': referrer['resource'],
'object': referrer
})
input_fields = [
resource_fields.field_name(field_id)
for field_id in input_fields_ids
]
input_fields = sorted(input_fields)
referrer_type = get_resource_type(referrer)
if referrer_type == 'dataset':
referrer_fields = Fields(referrer_fields.preferred_fields())
referrer_fields_names = sorted( \
[field['name'] for _, field in referrer_fields.fields.items()])
else:
referrer_fields_names = sorted( \
referrer_fields.fields_by_name.keys())
# check referrer input fields to see if they are equal
referrer_input_fields.append(referrer_fields_names)
# check whether the resource has an objective field not included in
# the input fields list
resource_type = get_resource_type(resource)
if resource_type == 'model':
objective_id = resource.get('objective_field')
try:
objective_id = objective_id.get('id')
except AttributeError:
pass
referrer_objective = resource_fields.field_name(objective_id)
referrer_input_fields.append([
name for name in referrer_fields_names
if name != referrer_objective
])
if input_fields in referrer_input_fields:
return []
return referrer_fields.fields.keys()
def best_first_search(datasets_file, api, args, common_options,
staleness=None, penalty=None, objective_name=None,
resume=False):
"""Selecting the fields to be used in the model construction
"""
counter = 0
loop_counter = 0
features_file = os.path.normpath(os.path.join(args.output_dir,
FEATURES_LOG))
with open(features_file, u.open_mode("w")) as features_handler:
features_writer = csv.writer(features_handler, lineterminator="\n")
features_writer.writerow([
"step", "state", "score", "metric_value", "best_score"])
features_handler.flush()
if staleness is None:
staleness = DEFAULT_STALENESS
if penalty is None:
penalty = DEFAULT_PENALTY
# retrieving the first dataset in the file
try:
with open(datasets_file, u.open_mode("r")) as datasets_handler:
dataset_id = datasets_handler.readline().strip()
except IOError, exc:
sys.exit("Could not read the generated datasets file: %s" %
str(exc))
try:
stored_dataset = u.storage_file_name(args.output_dir, dataset_id)
with open(stored_dataset, u.open_mode("r")) as dataset_handler:
dataset = json.loads(dataset_handler.read())
except IOError:
dataset = api.check_resource(dataset_id,
query_string=ALL_FIELDS_QS)
# initial feature set
fields = Fields(dataset)
excluded_features = ([] if args.exclude_features is None else
args.exclude_features.split(
args.args_separator))
try:
excluded_ids = [fields.field_id(feature) for
feature in excluded_features]
objective_id = fields.field_id(objective_name)
except ValueError, exc:
sys.exit(exc)
def dataset_processing(source,
training_set,
test_set,
model_ids,
name,
description,
fields,
dataset_fields,
api,
args,
resume,
csv_properties=None,
session_file=None,
path=None,
log=None):
"""Creating or retrieving dataset from input arguments
"""
dataset = None
if (training_set or args.source or (args.evaluate and test_set)):
# if resuming, try to extract args.dataset form log files
if resume:
message = u.dated("Dataset not found. Resuming.\n")
resume, args.dataset = c.checkpoint(c.is_dataset_created,
path,
debug=args.debug,
message=message,
log_file=session_file,
console=args.verbosity)
# If we have a source but no dataset or model has been provided, we
# create a new dataset if the no_dataset option isn't set up. Also
# if evaluate is set and test_set has been provided.
if ((source and not args.dataset and not args.model and not model_ids
and not args.no_dataset)
or (args.evaluate and args.test_set and not args.dataset)):
dataset_args = r.set_dataset_args(name, description, args, fields,
dataset_fields)
dataset = r.create_dataset(source, dataset_args, args, api, path,
session_file, log)
# If a dataset is provided, let's retrieve it.
elif args.dataset:
dataset = bigml.api.get_dataset_id(args.dataset)
# If we already have a dataset, we check the status and get the fields if
# we hadn't them yet.
if dataset:
dataset = r.get_dataset(dataset, api, args.verbosity, session_file)
if not csv_properties and 'locale' in dataset['object']:
csv_properties = {'data_locale': dataset['object']['locale']}
fields = Fields(dataset['object']['fields'], **csv_properties)
if args.public_dataset:
r.publish_dataset(dataset, api, args, session_file)
return dataset, resume, csv_properties, fields
def get_fields_changes(resource, referrer=None,
updatable_attrs=DEFAULT_UPDATABLE):
"""Changed field attributes
"""
if referrer is None:
referrer = {}
fields_attributes = {}
resource_fields = Fields(
{'resource': resource['resource'], 'object': resource}).fields
resource_type = get_resource_type(resource)
# for sources, extract all the updatable attributes
if resource_type == 'source':
updatable_attrs = SOURCE_UPDATABLE
for field_id in resource_fields.keys():
field_opts = {}
field = resource_fields[field_id]
for attribute in updatable_attrs:
if field.get(attribute):
field_opts.update({attribute: field[attribute]})
if field_opts != {}:
fields_attributes.update({field_id: field_opts})
return fields_attributes
# for the rest of resources, check which attributes changed
if referrer:
referrer_fields = Fields(
{'resource': referrer['resource'], 'object': referrer}).fields
for field_id in resource_fields.keys():
field_opts = {}
if not field_id in referrer_fields.keys():
continue
field = resource_fields[field_id]
for attribute in updatable_attrs:
ref_values = ["", referrer_fields[field_id].get(attribute, "")]
if not field.get(attribute, "") in ref_values:
field_opts.update({attribute: field[attribute]})
if field_opts != {}:
fields_attributes.update({field_id: field_opts})
return fields_attributes
def get_fusion_fields(fusion, csv_properties, args):
"""Retrieves fields info from Fusion resource
"""
args.retrieve_api_.ok(fusion)
if not csv_properties:
csv_properties = {}
csv_properties.update(verbose=True)
csv_properties.update(missing_tokens=DEFAULT_MISSING_TOKENS)
return Fields(fusion['object']['fusion']['fields'], \
**csv_properties)
def test_ensemble(self,test_file):
assert self.authenticated, 'Not authenticated!'
# download a local copy of the ensemble
self.logger.info('Creating local ensemble')
local_ensemble = Ensemble(self.ensemble_res,api=self.api)
# make the Fields object
source = self.api.get_source(self.source_res)
fields = Fields(source['object']['fields'])
self.logger.info('Reading test data and generating predictions')
true_labels = []
predict_labels = []
pr = Profile()
pr.enable()
with open(test_file) as fid:
test_reader = csv.reader(fid)
# skip the header line
test_reader.next()
for row in test_reader:
row_list = [val for val in row]
true_labels.append(row_list.pop())
instance = fields.pair(row_list)
predict_labels.append(local_ensemble.predict(instance,
by_name=False,
method=1))
pr.disable()
ps = Stats(pr)
self.predict_time = ps.total_tt
# eval_args = {'combiner':1}
# evaluation = self.api.create_evaluation(self.ensemble_res,test_data,eval_args)
# check_resource(evaluation['resource'],self.api.get_evaluation)
# evaluation = self.api.get_evaluation(evaluation['resource'])
# matrix = evaluation['object']['result']['model']['confusion_matrix']
# self.predict_time = evaluation['object']['status']['elapsed']/1000
if self.regression:
self.results = (predict_labels,true_labels)
else:
self.results = make_confusion_matrix(true_labels,predict_labels)
def get_cluster_fields(cluster, csv_properties, args):
"""Retrieves fields info from cluster resource
"""
if not csv_properties:
csv_properties = {}
csv_properties.update(verbose=True)
if args.user_locale is None:
args.user_locale = cluster['object'].get('locale', None)
csv_properties.update(data_locale=args.user_locale)
csv_properties.update(missing_tokens=DEFAULT_MISSING_TOKENS)
return Fields(cluster['object']['clusters']['fields'], **csv_properties)
def get_topic_model_fields(topic_model, csv_properties, args):
"""Retrieves fields info from topic model resource
"""
if not csv_properties:
csv_properties = {}
csv_properties.update(verbose=True)
if args.user_locale is None:
args.user_locale = topic_model['object'].get('locale', None)
csv_properties.update(data_locale=args.user_locale)
csv_properties.update(missing_tokens=DEFAULT_MISSING_TOKENS)
return Fields(topic_model, **csv_properties)
def get_logistic_fields(logistic_regression, csv_properties, args):
"""Retrieves fields info from logistic regression resource
"""
if not csv_properties:
csv_properties = {}
csv_properties.update(verbose=True)
if args.user_locale is None:
args.user_locale = logistic_regression['object'].get('locale', None)
csv_properties.update(data_locale=args.user_locale)
csv_properties.update(missing_tokens=DEFAULT_MISSING_TOKENS)
return Fields(logistic_regression['object'][ \
'logistic_regression']['fields'], \
**csv_properties)
def get_model_fields(model,
csv_properties,
args,
single_model=True,
multi_label_data=None,
local_ensemble=None):
"""Retrieves fields info from model resource
"""
if not csv_properties:
csv_properties = {}
csv_properties.update(verbose=True)
if args.user_locale is None:
args.user_locale = model['object'].get('locale', None)
csv_properties.update(data_locale=args.user_locale)
if single_model and 'model_fields' in model['object']['model']:
model_fields = model['object']['model']['model_fields'].keys()
csv_properties.update(include=model_fields)
else:
csv_properties.update(include=None)
if 'missing_tokens' in model['object']['model']:
missing_tokens = model['object']['model']['missing_tokens']
else:
missing_tokens = MISSING_TOKENS
csv_properties.update(missing_tokens=missing_tokens)
# if the model belongs to a multi-label set of models, the real objective
# field is never amongst the set of fields of each individual model, so
# we must add it.
fields_dict = copy.deepcopy(model['object']['model']['fields'])
if args.multi_label:
# Adds the real objective field to fields_dict
objective_field = multi_label_data['objective_name']
objective_id = multi_label_data['objective_id']
objective_column = multi_label_data['objective_column']
fields_dict[objective_id] = {
"op_type": "categorical",
"name": objective_field,
"column_number": objective_column
}
else:
if local_ensemble is not None:
fields_dict = copy.deepcopy(local_ensemble.fields)
objective_field = model['object']['objective_fields']
if isinstance(objective_field, list):
objective_field = objective_field[0]
csv_properties.update(objective_field=objective_field)
fields = Fields(fields_dict, **csv_properties)
return fields, objective_field
def get_deepnet_fields(deepnet, csv_properties, args):
"""Retrieves fields info from deepnet resource
"""
if not csv_properties:
csv_properties = {}
csv_properties.update(verbose=True)
if args.user_locale is None:
args.user_locale = deepnet['object'].get('locale', None)
csv_properties.update(data_locale=args.user_locale)
csv_properties.update(missing_tokens=DEFAULT_MISSING_TOKENS)
csv_properties.update(objective_field=deepnet['object'].get( \
'objective_field'))
return Fields(deepnet['object'][ \
'deepnet']['fields'], **csv_properties)
def get_pca_fields(pca, csv_properties, args):
"""Retrieves fields info from PCA resource
"""
args.retrieve_api_.ok(pca)
if not csv_properties:
csv_properties = {}
csv_properties.update(verbose=True)
if args.user_locale is None:
args.user_locale = pca['object'].get('locale', None)
csv_properties.update(data_locale=args.user_locale)
csv_properties.update(missing_tokens=DEFAULT_MISSING_TOKENS)
if args.exclude_objective:
csv_properties.update({"objective_field_present": False})
csv_properties.update({"objective_field": None})
return Fields(pca['object']['pca']['fields'], \
**csv_properties)
def get_fields_changes(resource,
referrer=None,
updatable_attrs=DEFAULT_UPDATABLE):
"""Changed field attributes
"""
if referrer is None:
referrer = {}
fields_attributes = {}
resource_fields = Fields({
'resource': resource['resource'],
'object': resource
}).fields
resource_type = get_resource_type(resource)
# for sources, extract all the updatable attributes
if resource_type == 'source':
updatable_attrs = SOURCE_UPDATABLE
for field_id in resource_fields.keys():
field_opts = {}
field = resource_fields[field_id]
for attribute in updatable_attrs:
if field.get(attribute):
field_opts.update({attribute: field[attribute]})
if field_opts != {}:
fields_attributes.update({field_id: field_opts})
return fields_attributes
# for the rest of resources, check which attributes changed
if referrer:
referrer_fields = Fields({
'resource': referrer['resource'],
'object': referrer
}).fields
for field_id in resource_fields.keys():
field_opts = {}
if not field_id in referrer_fields.keys():
continue
field = resource_fields[field_id]
for attribute in updatable_attrs:
ref_values = ["", referrer_fields[field_id].get(attribute, "")]
if not field.get(attribute, "") in ref_values:
field_opts.update({attribute: field[attribute]})
if field_opts != {}:
fields_attributes.update({field_id: field_opts})
return fields_attributes
def get_input_fields(resource, referrer=None):
"""New list of input fields
"""
if referrer is None:
referrer = {}
input_fields_ids = resource.get('input_fields', [])
if referrer:
referrer_input_fields = [[]]
# compare fields by name
resource_fields = Fields(
{'resource': resource['resource'], 'object': resource})
referrer_fields = Fields(
{'resource': referrer['resource'], 'object': referrer})
input_fields = [resource_fields.field_name(field_id) for field_id in
input_fields_ids]
input_fields = sorted(input_fields)
referrer_type = get_resource_type(referrer)
if referrer_type == 'dataset':
referrer_fields = Fields(referrer_fields.preferred_fields())
referrer_fields_names = sorted( \
[field['name'] for _, field in referrer_fields.fields.items()])
else:
referrer_fields_names = sorted( \
referrer_fields.fields_by_name.keys())
# check referrer input fields to see if they are equal
referrer_input_fields.append(referrer_fields_names)
# check whether the resource has an objective field not included in
# the input fields list
resource_type = get_resource_type(resource)
if resource_type == 'model':
objective_id = resource.get('objective_field')
try:
objective_id = objective_id.get('id')
except AttributeError:
pass
referrer_objective = resource_fields.field_name(
objective_id)
referrer_input_fields.append([name for name in
referrer_fields_names
if name != referrer_objective])
if input_fields in referrer_input_fields:
return []
return referrer_fields.fields.keys()
# retrieving the first dataset in the file
try:
with open(datasets_file, u.open_mode("r")) as datasets_handler:
dataset_id = datasets_handler.readline().strip()
except IOError, exc:
sys.exit("Could not read the generated datasets file: %s" %
str(exc))
try:
stored_dataset = u.storage_file_name(args.output_dir, dataset_id)
with open(stored_dataset, u.open_mode("r")) as dataset_handler:
dataset = json.loads(dataset_handler.read())
except IOError:
dataset = api.check_resource(dataset_id,
query_string=ALL_FIELDS_QS)
# initial feature set
fields = Fields(dataset)
excluded_features = ([] if args.exclude_features is None else
args.exclude_features.split(
args.args_separator))
try:
excluded_ids = [fields.field_id(feature) for
feature in excluded_features]
objective_id = fields.field_id(objective_name)
except ValueError, exc:
sys.exit(exc)
field_ids = [field_id for field_id in fields.preferred_fields()
if field_id != objective_id and
not field_id in excluded_ids]
# headers are extended with a column per field
fields_names = [fields.field_name(field_id) for field_id in field_ids]
features_header.extend(fields_names)
"""
counter = 0
if staleness is None:
staleness = DEFAULT_STALENESS
if penalty is None:
penalty = DEFAULT_PENALTY
# retrieving the first dataset in the file
try:
with open(datasets_file) as datasets_handler:
dataset_id = datasets_handler.readline().strip()
except IOError, exc:
sys.exit("Could not read the generated datasets file: %s" % str(exc))
dataset = api.check_resource(dataset_id, api.get_dataset)
# initial feature set
fields = Fields(dataset)
objective_id = fields.field_id(objective_column)
field_ids = [field_id for field_id in fields.preferred_fields()
if field_id != objective_id]
initial_state = [False for field_id in field_ids]
open_list = [(initial_state,0)]
closed_list = []
best_score = -1
best_unchanged_count = 0
metric = args.maximize
while best_unchanged_count < staleness and open_list:
(state, score) = find_max_state(open_list)
state_fields = [fields.field_name(field_ids[i])
for (i, val) in enumerate(state) if val]
closed_list.append((state, score))
open_list.remove((state, score))
def create_fields_from_dataset(step, objective_column):
world.fields = Fields(world.dataset,
objective_field=int(objective_column),
objective_field_present=True)
def dataset_processing(source,
api,
args,
resume,
fields=None,
csv_properties=None,
multi_label_data=None,
session_file=None,
path=None,
log=None):
"""Creating or retrieving dataset from input arguments
"""
datasets = []
dataset = None
if (args.training_set or args.source or
(hasattr(args, "evaluate") and args.evaluate and args.test_set)):
# if resuming, try to extract args.dataset form log files
if resume:
message = u.dated("Dataset not found. Resuming.\n")
resume, args.dataset = c.checkpoint(c.is_dataset_created,
path,
debug=args.debug,
message=message,
log_file=session_file,
console=args.verbosity)
# If we have a source but no dataset or model has been provided, we
# create a new dataset if the no_dataset option isn't set up. Also
# if evaluate is set and test_set has been provided.
if ((source and not args.has_datasets_ and not args.has_models_
and not args.no_dataset)
or (hasattr(args, "evaluate") and args.evaluate and args.test_set
and not args.dataset)):
dataset_args = r.set_dataset_args(args,
fields,
multi_label_data=multi_label_data)
dataset = r.create_dataset(source, dataset_args, args, api, path,
session_file, log)
# If set of datasets is provided, let's check their ids.
elif args.dataset_ids:
for i in range(0, len(args.dataset_ids)):
dataset_id = args.dataset_ids[i]
if isinstance(dataset_id, dict) and "id" in dataset_id:
dataset_id = dataset_id["id"]
datasets.append(bigml.api.get_dataset_id(dataset_id))
dataset = datasets[0]
# If a dataset is provided, let's retrieve it.
elif args.dataset:
dataset = bigml.api.get_dataset_id(args.dataset)
# If we already have a dataset, we check the status and get the fields if
# we hadn't them yet.
if dataset:
dataset = r.get_dataset(dataset, api, args.verbosity, session_file)
if ('object' in dataset and 'objective_field' in dataset['object']
and 'column_number' in dataset['object']['objective_field']):
dataset_objective = dataset['object']['objective_field'][
'column_number']
csv_properties.update(objective_field=dataset_objective,
objective_field_present=True)
fields = get_fields_structure(dataset, csv_properties)
if args.public_dataset:
r.publish_dataset(dataset, args, api, session_file)
if hasattr(args, 'objective_field'):
new_objective = get_new_objective(fields, args.objective_field)
else:
new_objective = None
updated = False
# We'll update the dataset if
# the flag --dataset_attributes is used
# the --multi-label flag is used and there's an --objective-field
# the --max-categories flag is used and there's an --objective-field
# the --impor-fields flag is used
if check_dataset_update(args, dataset):
dataset_args = r.set_dataset_args(args, fields)
if args.shared_flag and r.shared_changed(args.shared, dataset):
dataset_args.update(shared=args.shared)
dataset = r.update_dataset(dataset,
dataset_args,
args,
api=api,
path=path,
session_file=session_file)
dataset = r.get_dataset(dataset, api, args.verbosity, session_file)
updated = True
if new_objective is not None:
csv_properties.update(objective_field=args.objective_field,
objective_field_present=True)
updated = True
if updated:
fields = Fields(dataset['object']['fields'], **csv_properties)
if not datasets:
datasets = [dataset]
else:
datasets[0] = dataset
return datasets, resume, csv_properties, fields
def create_fields(step, objective_column):
world.fields = Fields(world.source,
objective_field=int(objective_column),
objective_field_present=True)
def source_processing(training_set,
test_set,
training_set_header,
test_set_header,
name,
description,
api,
args,
resume,
csv_properties=None,
field_attributes=None,
types=None,
session_file=None,
path=None,
log=None):
"""Creating or retrieving a data source from input arguments
"""
source = None
fields = None
if (training_set or (args.evaluate and test_set)):
# If resuming, try to extract args.source form log files
if resume:
message = u.dated("Source not found. Resuming.\n")
resume, args.source = c.checkpoint(c.is_source_created,
path,
debug=args.debug,
message=message,
log_file=session_file,
console=args.verbosity)
# If neither a previous source, dataset or model are provided.
# we create a new one. Also if --evaluate and test data are provided
# we create a new dataset to test with.
data_set, data_set_header = r.data_to_source(training_set, test_set,
training_set_header,
test_set_header, args)
if data_set is not None:
source_args = r.set_source_args(data_set_header, name, description,
args)
source = r.create_source(data_set, source_args, args, api, path,
session_file, log)
# If a source is provided either through the command line or in resume
# steps, we use it.
elif args.source:
source = bigml.api.get_source_id(args.source)
# If we already have source, we check that is finished, extract the
# fields, and update them if needed.
if source:
source = r.get_source(source, api, args.verbosity, session_file)
if 'source_parser' in source['object']:
source_parser = source['object']['source_parser']
if 'missing_tokens' in source_parser:
csv_properties['missing_tokens'] = (
source_parser['missing_tokens'])
if 'data_locale' in source_parser:
csv_properties['data_locale'] = source_parser['locale']
fields = Fields(source['object']['fields'], **csv_properties)
if field_attributes:
source = r.update_source_fields(source, field_attributes, fields,
api, args.verbosity, session_file)
if types:
source = r.update_source_fields(source, types, fields, api,
args.verbosity, session_file)
return source, resume, csv_properties, fields
def main(args=sys.argv[1:]):
"""Parses command-line parameters and calls the actual main function.
"""
parser = argparse.ArgumentParser(
description="Dataset analysis",
epilog="BigML, Inc")
# source with activity data
parser.add_argument('--source',
action='store',
dest='source',
default=None,
help="Full path to file")
# create private links or not
parser.add_argument('--share',
action='store_true',
default=False,
help="Share created resources or not")
# weight models or not
parser.add_argument('--balance',
action='store_true',
default=False,
help="Weight model or not")
args = parser.parse_args(args)
if not args.source:
sys.exit("You need to provide a valid path to a source")
api = BigML()
name = "Sean's activity"
log("Creating source...")
source_args = {'name': name}
source = api.create_source(args.source, source_args)
if not api.ok(source):
sys.exit("Source isn't ready...")
log("Creating dataset...")
dataset = api.create_dataset(source)
if not api.ok(dataset):
sys.exit("Dataset isn't ready...")
log("Transforming dataset...")
# Extends dataset with new field for previous activity, previous duration,
# start day, and start hour. Removes first column, start, and end fields.
new_dataset_args = {
'name': name,
'new_fields': new_fields(),
'all_but': excluded_fields()}
new_dataset = api.create_dataset(dataset, new_dataset_args)
if not api.ok(new_dataset):
sys.exit("Dataset isn't ready...")
# Set objective field to activity
fields = Fields(new_dataset['object']['fields'])
objective_id = fields.field_id('activity')
new_dataset_args = {
'objective_field': {'id': objective_id}}
new_dataset = api.update_dataset(new_dataset, new_dataset_args)
# Create training and test set for evaluation
log("Splitting dataset...")
training, test = train_test_split(api, new_dataset)
log("Creating a model using the training dataset...")
model_args = {
'objective_field': objective_id,
'balance_objective': args.balance,
'name': training['object']['name']}
model = api.create_model(training, model_args)
if not api.ok(model):
sys.exit("Model isn't ready...")
# Creating an evaluation
log("Evaluating model against the test dataset...")
eval_args = {
'name': name + ' - 80% vs 20%'}
evaluation = api.create_evaluation(model, test, eval_args)
if not api.ok(evaluation):
sys.exit("Evaluation isn't ready...")
log("Creating model for the full dataset...")
model = api.create_model(new_dataset, model_args)
if not api.ok(model):
sys.exit("Model isn't ready...")
# Create private links
if args.share:
log("Sharing resources...")
dataset_private_link = share_dataset(api, new_dataset)
model_private_link = share_model(api, model)
evaluation_private_link = share_evaluation(api, evaluation)
log(dataset_private_link)
log(model_private_link)
log(evaluation_private_link)
def SymptomInsert(self, model):
session = model.key.get()
if session is None:
raise endpoints.NotFoundException('Session not found.')
if session.symptoms is None :
session.symptoms = Symptoms()
for s in session.symptoms.items :
if s.name == model.name :
s.value = model.value
break
else :
symptom = Symptom(name=model.name, value=model.value)
session.symptoms.items.append(symptom)
logging.debug('starting prediction')
p = {}
for symptom in session.symptoms.items:
p[symptom.name] = symptom.value
bigml_local_model = bigml_model.get_local_model()
prediction = bigml_local_model.predict(p, add_confidence=True, add_path=True, add_distribution=True, add_count=True, add_next=True)
prediction_all = bigml_local_model.predict(p, multiple=5)
if prediction['next'] is not None :
logging.debug('got fields %s' % bigml_local_model.fields)
fields = Fields(bigml_local_model.fields)
field_id = fields.field_id(prediction['next'])
field = bigml_local_model.fields[field_id]
if 'label' in field :
label = field['label']
else :
label = field['name']
if 'description' in field :
description = field['description']
else :
description = ''
if 'categories' in field['summary'] :
cat = []
for c in field['summary']['categories'] :
cat.append(c[0])
session.next = Question(label=label, description=description, type=field['optype'], categories=cat)
else:
session.next = Question(label=label, description=description, type=field['optype'])
else :
session.next = None
session.outcome = Outcome(name=prediction['prediction'], confidence=str(prediction['confidence']), full=prediction_all)
session.put()
return session
def reify_dataset(self, resource_id):
"""Extracts the REST API arguments from the dataset JSON structure
"""
child = self.get_resource(resource_id)
origin, parent_id = u.get_origin_info(child)
parent = self.get_resource(parent_id)
opts = {"create": {}, "update": {}, "get": {}}
# as two-steps result from a cluster or batch prediction, centroid
# or anomaly score
grandparent = parent
if origin in ['origin_batch_resource', 'cluster']:
if origin == "cluster":
opts['create'].update({"centroid": child['centroid']})
grandparents = u.get_origin_info(parent)
# batch resources have two parents, choose the dataset
if origin == "origin_batch_resource" and \
isinstance(grandparents, list):
for gp_origin, grandparent in grandparents:
if gp_origin == "dataset":
break
else:
_, grandparent = grandparents
grandparent = self.get_resource(grandparent)
# options common to all model types
call = "update" if origin == "origin_batch_resource" else "create"
u.common_dataset_opts(child, grandparent, opts, call=call)
# update options
dataset_defaults = DEFAULTS["dataset"].get("update", {})
for attribute, default_value in dataset_defaults.items():
opts["update"].update(
u.default_setting(child, attribute, *default_value))
# name, exclude automatic naming alternatives
autonames = [u'']
u.non_automatic_name(child, opts, autonames=autonames)
# objective field
resource_fields = Fields(
{'resource': child['resource'], 'object': child})
objective_id = child['objective_field']['id']
preferred_fields = resource_fields.preferred_fields()
# if there's no preferred fields, use the fields structure
if len(preferred_fields.keys()) == 0:
preferred_fields = resource_fields.fields
max_column = sorted([field['column_number']
for _, field in preferred_fields.items()
if field['optype'] != "text"],
reverse=True)[0]
objective_column = resource_fields.fields[objective_id][ \
'column_number']
if objective_column != max_column:
opts['create'].update({"objective_field": {"id": objective_id}})
if origin != "origin_batch_resource":
# resize
if (child['size'] != grandparent['size'] and
get_resource_type(parent) == 'source'):
opts['create'].update({"size": child['size']})
# generated fields
if child.get('new_fields', None):
new_fields = child['new_fields']
for new_field in new_fields:
new_field['field'] = new_field['generator']
del new_field['generator']
opts['create'].update({"new_fields": new_fields})
u.range_opts(child, grandparent, opts)
# for batch_predictions, batch_clusters, batch_anomalies generated
# datasets, attributes cannot be set at creation time, so we
# must update the resource instead
suffix = None
if origin == "origin_batch_resource":
opts["update"].update(opts["create"])
opts["create"] = {}
suffix = "['object']['output_dataset_resource']"
calls = u.build_calls(resource_id, [parent_id], opts, suffix=suffix)
self.add(resource_id, calls)
def reify_dataset(self, resource_id):
"""Extracts the REST API arguments from the dataset JSON structure
"""
child = self.get_resource(resource_id)
origin, parent_id = u.get_origin_info(child)
parent = self.get_resource(parent_id)
opts = {"create": {}, "update": {}}
# as two-steps result from a cluster or batch prediction, centroid
# or anomaly score
if origin in ["origin_batch_resource", "cluster"]:
if origin == "cluster":
opts["create"].update({"centroid": child["centroid"]})
_, grandparent = u.get_origin_info(parent)
grandparent = self.get_resource(grandparent)
else:
grandparent = parent
# options common to all model types
u.common_dataset_opts(child, grandparent, opts)
# update options
dataset_defaults = DEFAULTS["dataset"].get("update", {})
dataset_defaults.update(COMMON_DEFAULTS.get("update", {}))
for attribute, default_value in dataset_defaults.items():
opts["update"].update(u.default_setting(child, attribute, *default_value))
# name, exclude automatic naming alternatives
autonames = [u""]
suffixes = [
u"filtered",
u"sampled",
u"dataset",
u"extended",
u"- batchprediction",
u"- batchanomalyscore",
u"- batchcentroid",
u"- merged",
]
autonames.extend([u"%s %s" % (grandparent.get("name", ""), suffix) for suffix in suffixes])
autonames.append(u"%s's dataset" % ".".join(parent["name"].split(".")[0:-1]))
autonames.append(u"%s' dataset" % ".".join(parent["name"].split(".")[0:-1]))
autonames.append(u"Cluster %s - %s" % (int(child.get("centroid", "0"), base=16), parent["name"]))
autonames.append(u"Dataset from %s model - segment" % parent["name"])
u.non_automatic_name(child, opts, autonames=autonames)
# objective field
resource_fields = Fields({"resource": child["resource"], "object": child})
objective_id = child["objective_field"]["id"]
preferred_fields = resource_fields.preferred_fields()
max_column = sorted([field["column_number"] for _, field in preferred_fields.items()], reverse=True)[0]
objective_column = resource_fields.fields[objective_id]["column_number"]
if objective_column != max_column:
opts["create"].update({"objective_field": {"id": objective_id}})
# resize
if child["size"] != grandparent["size"] and get_resource_type(parent) == "source":
opts["create"].update({"size": child["size"]})
# generated fields
if child.get("new_fields", None):
new_fields = child["new_fields"]
for new_field in new_fields:
new_field["field"] = new_field["generator"]
del new_field["generator"]
opts["create"].update({"new_fields": new_fields})
u.range_opts(child, grandparent, opts)
calls = u.build_calls(resource_id, [parent_id], opts)
self.add(resource_id, calls)
def compute_output(api, args, training_set, test_set=None, output=None,
objective_field=None,
description=None,
field_attributes=None,
types=None,
dataset_fields=None,
model_fields=None,
name=None, training_set_header=True,
test_set_header=True, model_ids=None,
votes_files=None, resume=False, fields_map=None):
""" Creates one or more models using the `training_set` or uses the ids
of previously created BigML models to make predictions for the `test_set`.
"""
source = None
dataset = None
model = None
models = None
fields = None
path = u.check_dir(output)
session_file = "%s%s%s" % (path, os.sep, SESSIONS_LOG)
csv_properties = {}
# If logging is required, open the file for logging
log = None
if args.log_file:
u.check_dir(args.log_file)
log = args.log_file
# If --clear_logs the log files are cleared
if args.clear_logs:
try:
open(log, 'w', 0).close()
except IOError:
pass
if (training_set or (args.evaluate and test_set)):
if resume:
resume, args.source = u.checkpoint(u.is_source_created, path,
bigml.api, debug=args.debug)
if not resume:
message = u.dated("Source not found. Resuming.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
# If neither a previous source, dataset or model are provided.
# we create a new one. Also if --evaluate and test data are provided
# we create a new dataset to test with.
data_set = None
if (training_set and not args.source and not args.dataset and
not args.model and not args.models):
data_set = training_set
data_set_header = training_set_header
elif (args.evaluate and test_set and not args.source):
data_set = test_set
data_set_header = test_set_header
if not data_set is None:
source_args = {
"name": name,
"description": description,
"category": args.category,
"tags": args.tag,
"source_parser": {"header": data_set_header}}
message = u.dated("Creating source.\n")
u.log_message(message, log_file=session_file, console=args.verbosity)
source = api.create_source(data_set, source_args,
progress_bar=args.progress_bar)
source = api.check_resource(source, api.get_source)
message = u.dated("Source created: %s\n" % u.get_url(source, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
u.log_message("%s\n" % source['resource'], log_file=log)
fields = Fields(source['object']['fields'],
source['object']['source_parser']['missing_tokens'],
source['object']['source_parser']['locale'])
source_file = open(path + '/source', 'w', 0)
source_file.write("%s\n" % source['resource'])
source_file.write("%s\n" % source['object']['name'])
source_file.flush()
source_file.close()
# If a source is provided, we retrieve it.
elif args.source:
message = u.dated("Retrieving source. %s\n" %
u.get_url(args.source, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
source = api.get_source(args.source)
# If we already have source, we check that is finished and extract the
# fields, and update them if needed.
if source:
if source['object']['status']['code'] != bigml.api.FINISHED:
message = u.dated("Retrieving source. %s\n" %
u.get_url(source, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
source = api.check_resource(source, api.get_source)
csv_properties = {'missing_tokens':
source['object']['source_parser']['missing_tokens'],
'data_locale':
source['object']['source_parser']['locale']}
fields = Fields(source['object']['fields'], **csv_properties)
update_fields = {}
if field_attributes:
for (column, value) in field_attributes.iteritems():
update_fields.update({
fields.field_id(column): value})
message = u.dated("Updating source. %s\n" %
u.get_url(source, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
source = api.update_source(source, {"fields": update_fields})
update_fields = {}
if types:
for (column, value) in types.iteritems():
update_fields.update({
fields.field_id(column): {'optype': value}})
message = u.dated("Updating source. %s\n" %
u.get_url(source, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
source = api.update_source(source, {"fields": update_fields})
if (training_set or args.source or (args.evaluate and test_set)):
if resume:
resume, args.dataset = u.checkpoint(u.is_dataset_created, path,
bigml.api,
debug=args.debug)
if not resume:
message = u.dated("Dataset not found. Resuming.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
# If we have a source but not dataset or model has been provided, we
# create a new dataset if the no_dataset option isn't set up. Also
# if evaluate is set and test_set has been provided.
if ((source and not args.dataset and not args.model and not model_ids and
not args.no_dataset) or
(args.evaluate and args.test_set and not args.dataset)):
dataset_args = {
"name": name,
"description": description,
"category": args.category,
"tags": args.tag
}
if args.json_filter:
dataset_args.update(json_filter=args.json_filter)
elif args.lisp_filter:
dataset_args.update(lisp_filter=args.lisp_filter)
input_fields = []
if dataset_fields:
for name in dataset_fields:
input_fields.append(fields.field_id(name))
dataset_args.update(input_fields=input_fields)
message = u.dated("Creating dataset.\n")
u.log_message(message, log_file=session_file, console=args.verbosity)
dataset = api.create_dataset(source, dataset_args)
dataset = api.check_resource(dataset, api.get_dataset)
message = u.dated("Dataset created: %s\n" % u.get_url(dataset, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
u.log_message("%s\n" % dataset['resource'], log_file=log)
dataset_file = open(path + '/dataset', 'w', 0)
dataset_file.write("%s\n" % dataset['resource'])
dataset_file.flush()
dataset_file.close()
# If a dataset is provided, let's retrieve it.
elif args.dataset:
message = u.dated("Retrieving dataset. %s\n" %
u.get_url(args.dataset, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
dataset = api.get_dataset(args.dataset)
# If we already have a dataset, we check the status and get the fields if
# we hadn't them yet.
if dataset:
if dataset['object']['status']['code'] != bigml.api.FINISHED:
message = u.dated("Retrieving dataset. %s\n" %
u.get_url(dataset, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
dataset = api.check_resource(dataset, api.get_dataset)
if not csv_properties:
csv_properties = {'data_locale':
dataset['object']['locale']}
if args.public_dataset:
if not description:
raise Exception("You should provide a description to publish.")
public_dataset = {"private": False}
if args.dataset_price:
message = u.dated("Updating dataset. %s\n" %
u.get_url(dataset, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
public_dataset.update(price=args.dataset_price)
message = u.dated("Updating dataset. %s\n" %
u.get_url(dataset, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
dataset = api.update_dataset(dataset, public_dataset)
fields = Fields(dataset['object']['fields'], **csv_properties)
# If we have a dataset but not a model, we create the model if the no_model
# flag hasn't been set up.
if (dataset and not args.model and not model_ids and not args.no_model):
model_args = {
"name": name,
"description": description,
"category": args.category,
"tags": args.tag
}
if objective_field is not None:
model_args.update({"objective_field":
fields.field_id(objective_field)})
# If evaluate flag is on, we choose a deterministic sampling with 80%
# of the data to create the model
if args.evaluate:
if args.sample_rate == 1:
args.sample_rate = EVALUATE_SAMPLE_RATE
seed = SEED
model_args.update(seed=seed)
input_fields = []
if model_fields:
for name in model_fields:
input_fields.append(fields.field_id(name))
model_args.update(input_fields=input_fields)
if args.pruning and args.pruning != 'smart':
model_args.update(stat_pruning=(args.pruning == 'statistical'))
model_args.update(sample_rate=args.sample_rate,
replacement=args.replacement,
randomize=args.randomize)
model_ids = []
models = []
if resume:
resume, model_ids = u.checkpoint(u.are_models_created, path,
args.number_of_models,
bigml.api, debug=args.debug)
if not resume:
message = u.dated("Found %s models out of %s. Resuming.\n" %
(len(model_ids),
args.number_of_models))
u.log_message(message, log_file=session_file,
console=args.verbosity)
models = model_ids
args.number_of_models -= len(model_ids)
model_file = open(path + '/models', 'w', 0)
for model_id in model_ids:
model_file.write("%s\n" % model_id)
last_model = None
if args.number_of_models > 0:
message = u.dated("Creating %s.\n" %
u.plural("model", args.number_of_models))
u.log_message(message, log_file=session_file,
console=args.verbosity)
for i in range(1, args.number_of_models + 1):
if i > args.max_parallel_models:
api.check_resource(last_model, api.get_model)
model = api.create_model(dataset, model_args)
u.log_message("%s\n" % model['resource'], log_file=log)
last_model = model
model_ids.append(model['resource'])
models.append(model)
model_file.write("%s\n" % model['resource'])
model_file.flush()
if args.number_of_models < 2 and args.verbosity:
if model['object']['status']['code'] != bigml.api.FINISHED:
model = api.check_resource(model, api.get_model)
models[0] = model
message = u.dated("Model created: %s.\n" %
u.get_url(model, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
model_file.close()
# If a model is provided, we retrieve it.
elif args.model:
message = u.dated("Retrieving model. %s\n" %
u.get_url(args.model, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
model = api.get_model(args.model)
elif args.models or args.model_tag:
models = model_ids[:]
if model_ids and test_set and not args.evaluate:
model_id = ""
if len(model_ids) == 1:
model_id = model_ids[0]
message = u.dated("Retrieving %s. %s\n" %
(u.plural("model", len(model_ids)),
u.get_url(model_id, api)))
u.log_message(message, log_file=session_file, console=args.verbosity)
if len(model_ids) < args.max_batch_models:
models = []
for model in model_ids:
model = api.check_resource(model, api.get_model)
models.append(model)
model = models[0]
else:
model = api.check_resource(model_ids[0], api.get_model)
models[0] = model
# We check that the model is finished and get the fields if haven't got
# them yet.
if model and not args.evaluate and (test_set or args.black_box
or args.white_box):
if model['object']['status']['code'] != bigml.api.FINISHED:
message = u.dated("Retrieving model. %s\n" %
u.get_url(model, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
model = api.check_resource(model, api.get_model)
if args.black_box:
if not description:
raise Exception("You should provide a description to publish.")
model = api.update_model(model, {"private": False})
if args.white_box:
if not description:
raise Exception("You should provide a description to publish.")
public_model = {"private": False, "white_box": True}
if args.model_price:
message = u.dated("Updating model. %s\n" %
u.get_url(model, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
public_model.update(price=args.model_price)
if args.cpp:
message = u.dated("Updating model. %s\n" %
u.get_url(model, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
public_model.update(credits_per_prediction=args.cpp)
model = api.update_model(model, public_model)
if not csv_properties:
csv_properties = {'data_locale':
model['object']['locale']}
csv_properties.update(verbose=True)
if args.user_locale:
csv_properties.update(data_locale=args.user_locale)
fields = Fields(model['object']['model']['fields'], **csv_properties)
if model and not models:
models = [model]
if models and test_set and not args.evaluate:
objective_field = models[0]['object']['objective_fields']
if isinstance(objective_field, list):
objective_field = objective_field[0]
predict(test_set, test_set_header, models, fields, output,
objective_field, args.remote, api, log,
args.max_batch_models, args.method, resume, args.tag,
args.verbosity, session_file, args.debug)
# When combine_votes flag is used, retrieve the predictions files saved
# in the comma separated list of directories and combine them
if votes_files:
model_id = re.sub(r'.*(model_[a-f0-9]{24})__predictions\.csv$',
r'\1', votes_files[0]).replace("_", "/")
model = api.check_resource(model_id, api.get_model)
local_model = Model(model)
message = u.dated("Combining votes.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
u.combine_votes(votes_files, local_model.to_prediction,
output, args.method)
# If evaluate flag is on, create remote evaluation and save results in
# json and human-readable format.
if args.evaluate:
if resume:
resume, evaluation = u.checkpoint(u.is_evaluation_created, path,
bigml.api,
debug=args.debug)
if not resume:
message = u.dated("Evaluation not found. Resuming.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
if not resume:
evaluation_file = open(path + '/evaluation', 'w', 0)
evaluation_args = {
"name": name,
"description": description,
"tags": args.tag
}
if not fields_map is None:
update_map = {}
for (dataset_column, model_column) in fields_map.iteritems():
update_map.update({
fields.field_id(dataset_column):
fields.field_id(model_column)})
evaluation_args.update({"fields_map": update_map})
if not ((args.dataset or args.test_set)
and (args.model or args.models or args.model_tag)):
evaluation_args.update(out_of_bag=True, seed=SEED,
sample_rate=args.sample_rate)
message = u.dated("Creating evaluation.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
evaluation = api.create_evaluation(model, dataset, evaluation_args)
u.log_message("%s\n" % evaluation['resource'], log_file=log)
evaluation_file.write("%s\n" % evaluation['resource'])
evaluation_file.flush()
evaluation_file.close()
message = u.dated("Retrieving evaluation. %s\n" %
u.get_url(evaluation, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
evaluation = api.check_resource(evaluation, api.get_evaluation)
evaluation_json = open(output + '.json', 'w', 0)
evaluation_json.write(json.dumps(evaluation['object']['result']))
evaluation_json.flush()
evaluation_json.close()
evaluation_txt = open(output + '.txt', 'w', 0)
api.pprint(evaluation['object']['result'],
evaluation_txt)
evaluation_txt.flush()
evaluation_txt.close()
# Workaround to restore windows console cp850 encoding to print the tree
if sys.platform == "win32" and sys.stdout.isatty():
import locale
data_locale = locale.getlocale()
if not data_locale[0] is None:
locale.setlocale(locale.LC_ALL, (data_locale[0], "850"))
message = (u"\nGenerated files:\n\n" +
unicode(u.print_tree(path, " "), "utf-8") + u"\n")
else:
message = "\nGenerated files:\n\n" + u.print_tree(path, " ") + "\n"
u.log_message(message, log_file=session_file, console=args.verbosity)
def create_kfold_datasets_file(args, api, common_options, resume=False):
"""Create the kfold dataset resources and store their ids in a file
one per line
"""
message = ('Creating the kfold datasets............\n')
u.log_message(message, log_file=session_file, console=args.verbosity)
if args.output_dir is None:
args.output_dir = a.NOW
csv_properties = {}
fields = None
dataset = None
datasets = []
if args.dataset_file:
# dataset is retrieved from the contents of the given local JSON file
model_dataset, csv_properties, fields = u.read_local_resource(
args.dataset_file,
csv_properties=csv_properties)
if not args.datasets:
datasets = [model_dataset]
dataset = model_dataset
else:
datasets = u.read_datasets(args.datasets)
dataset_id = dataset['resource']
elif args.dataset:
dataset_id = bigml.api.get_dataset_id(args.dataset)
datasets = [dataset_id]
elif args.dataset_ids:
datasets = args.dataset_ids
dataset_id = datasets[0]
if dataset_id:
if not dataset:
dataset = api.check_resource(dataset_id,
query_string=ALL_FIELDS_QS)
try:
args.objective_field = int(args.objective_field)
except (TypeError, ValueError):
pass
# if the user provided no objective field, try to use the one in the
# dataset
if args.objective_field is None:
try:
args.objective_field = dataset['object'][
'objective_field']['column_number']
except KeyError:
pass
# check that kfold_field is unique
fields = Fields(dataset, objective_field=args.objective_field,
objective_field_present=True)
if args.random_fields:
default_candidates_limits(args, fields)
try:
objective_id = fields.field_id(fields.objective_field)
objective_name = fields.field_name(objective_id)
except ValueError, exc:
sys.exit(exc)
kfold_field_name = avoid_duplicates(DEFAULT_KFOLD_FIELD, fields)
# create jsons to generate partial datasets
selecting_file_list, resume = create_kfold_json(args, kfold_field_name,
objective_id,
resume=resume)
# generate test datasets
datasets_file, resume = create_kfold_datasets(dataset_id, args,
selecting_file_list,
common_options,
resume=resume)
return datasets_file, objective_name, resume
def reify_dataset(self, resource_id):
"""Extracts the REST API arguments from the dataset JSON structure
"""
child = self.get_resource(resource_id)
origin, parent_id = u.get_origin_info(child)
parent = self.get_resource(parent_id)
opts = {"create": {}, "update": {}, "get": {}}
# as two-steps result from a cluster or batch prediction, centroid
# or anomaly score
grandparent = parent
if origin in ['origin_batch_resource', 'cluster']:
if origin == "cluster":
opts['create'].update({"centroid": child['centroid']})
grandparents = u.get_origin_info(parent)
# batch resources have two parents, choose the dataset
if origin == "origin_batch_resource" and \
isinstance(grandparents, list):
for gp_origin, grandparent in grandparents:
if gp_origin == "dataset":
break
else:
_, grandparent = grandparents
grandparent = self.get_resource(grandparent)
# options common to all model types
call = "update" if origin == "origin_batch_resource" else "create"
u.common_dataset_opts(child, grandparent, opts, call=call)
# update options
dataset_defaults = DEFAULTS["dataset"].get("update", {})
for attribute, default_value in dataset_defaults.items():
opts["update"].update(
u.default_setting(child, attribute, *default_value))
# name, exclude automatic naming alternatives
autonames = [u'']
u.non_automatic_name(child, opts, autonames=autonames)
# objective field
resource_fields = Fields({
'resource': child['resource'],
'object': child
})
objective_id = child['objective_field']['id']
preferred_fields = resource_fields.preferred_fields()
# if there's no preferred fields, use the fields structure
if len(preferred_fields.keys()) == 0:
preferred_fields = resource_fields.fields
max_column = sorted([
field['column_number'] for _, field in preferred_fields.items()
if field['optype'] != "text"
],
reverse=True)[0]
objective_column = resource_fields.fields[objective_id][ \
'column_number']
if objective_column != max_column:
opts['create'].update({"objective_field": {"id": objective_id}})
if origin != "origin_batch_resource":
# resize
if (child['size'] != grandparent['size']
and get_resource_type(parent) == 'source'):
opts['create'].update({"size": child['size']})
# generated fields
if child.get('new_fields', None):
new_fields = child['new_fields']
for new_field in new_fields:
new_field['field'] = new_field['generator']
del new_field['generator']
opts['create'].update({"new_fields": new_fields})
u.range_opts(child, grandparent, opts)
# for batch_predictions, batch_clusters, batch_anomalies generated
# datasets, attributes cannot be set at creation time, so we
# must update the resource instead
suffix = None
if origin == "origin_batch_resource":
opts["update"].update(opts["create"])
opts["create"] = {}
suffix = "['object']['output_dataset_resource']"
calls = u.build_calls(resource_id, [parent_id], opts, suffix=suffix)
self.add(resource_id, calls)
def compute_output(api, args):
""" Creates one or more models using the `training_set` or uses the ids
of previously created BigML models to make predictions for the `test_set`.
"""
source = None
dataset = None
model = None
models = None
fields = None
other_label = OTHER
ensemble_ids = []
multi_label_data = None
multi_label_fields = []
#local_ensemble = None
test_dataset = None
datasets = None
# variables from command-line options
resume = args.resume_
model_ids = args.model_ids_
output = args.output
dataset_fields = args.dataset_fields_
check_args_coherence(args)
path = u.check_dir(output)
session_file = "%s%s%s" % (path, os.sep, SESSIONS_LOG)
csv_properties = {}
# If logging is required set the file for logging
log = None
if args.log_file:
u.check_dir(args.log_file)
log = args.log_file
# If --clear_logs the log files are cleared
clear_log_files([log])
# labels to be used in multi-label expansion
labels = (None if args.labels is None else
[label.strip() for label in
args.labels.split(args.args_separator)])
if labels is not None:
labels = sorted([label for label in labels])
# multi_label file must be preprocessed to obtain a new extended file
if args.multi_label and args.training_set is not None:
(args.training_set, multi_label_data) = ps.multi_label_expansion(
args.training_set, args.train_header, args, path,
labels=labels, session_file=session_file)
args.train_header = True
args.objective_field = multi_label_data["objective_name"]
all_labels = l.get_all_labels(multi_label_data)
if not labels:
labels = all_labels
else:
all_labels = labels
if args.objective_field:
csv_properties.update({'objective_field': args.objective_field})
if args.source_file:
# source is retrieved from the contents of the given local JSON file
source, csv_properties, fields = u.read_local_resource(
args.source_file,
csv_properties=csv_properties)
else:
# source is retrieved from the remote object
source, resume, csv_properties, fields = ps.source_processing(
api, args, resume,
csv_properties=csv_properties, multi_label_data=multi_label_data,
session_file=session_file, path=path, log=log)
if source is not None:
args.source = bigml.api.get_source_id(source)
if args.multi_label and source:
multi_label_data = l.get_multi_label_data(source)
(args.objective_field,
labels,
all_labels,
multi_label_fields) = l.multi_label_sync(args.objective_field,
labels,
multi_label_data,
fields,
multi_label_fields)
if fields and args.export_fields:
fields.summary_csv(os.path.join(path, args.export_fields))
if args.dataset_file:
# dataset is retrieved from the contents of the given local JSON file
model_dataset, csv_properties, fields = u.read_local_resource(
args.dataset_file,
csv_properties=csv_properties)
if not args.datasets:
datasets = [model_dataset]
dataset = model_dataset
else:
datasets = u.read_datasets(args.datasets)
if not datasets:
# dataset is retrieved from the remote object
datasets, resume, csv_properties, fields = pd.dataset_processing(
source, api, args, resume,
fields=fields,
csv_properties=csv_properties,
multi_label_data=multi_label_data,
session_file=session_file, path=path, log=log)
if datasets:
dataset = datasets[0]
if args.to_csv is not None:
resume = pd.export_dataset(dataset, api, args, resume,
session_file=session_file, path=path)
# Now we have a dataset, let's check if there's an objective_field
# given by the user and update it in the fields structure
args.objective_id_ = get_objective_id(args, fields)
# If test_split is used, split the dataset in a training and a test dataset
# according to the given split
if args.test_split > 0:
dataset, test_dataset, resume = pd.split_processing(
dataset, api, args, resume,
multi_label_data=multi_label_data,
session_file=session_file, path=path, log=log)
datasets[0] = dataset
# Check if the dataset has a categorical objective field and it
# has a max_categories limit for categories
if args.max_categories > 0 and len(datasets) == 1:
if pd.check_max_categories(fields.fields[args.objective_id_]):
distribution = pd.get_categories_distribution(dataset,
args.objective_id_)
if distribution and len(distribution) > args.max_categories:
categories = [element[0] for element in distribution]
other_label = pd.create_other_label(categories, other_label)
datasets, resume = pd.create_categories_datasets(
dataset, distribution, fields, args,
api, resume, session_file=session_file, path=path, log=log,
other_label=other_label)
else:
sys.exit("The provided objective field is not categorical nor "
"a full terms only text field. "
"Only these fields can be used with"
" --max-categories")
# If multi-dataset flag is on, generate a new dataset from the given
# list of datasets
if args.multi_dataset:
dataset, resume = pd.create_new_dataset(
datasets, api, args, resume, fields=fields,
session_file=session_file, path=path, log=log)
datasets = [dataset]
# Check if the dataset has a generators file associated with it, and
# generate a new dataset with the specified field structure. Also
# if the --to-dataset flag is used to clone or sample the original dataset
if args.new_fields or (args.sample_rate != 1 and args.no_model) or \
(args.lisp_filter or args.json_filter) and not has_source(args):
if fields is None:
if isinstance(dataset, basestring):
dataset = u.check_resource(dataset, api=api)
fields = Fields(dataset, csv_properties)
args.objective_id_ = get_objective_id(args, fields)
args.objective_name_ = fields.field_name(args.objective_id_)
dataset, resume = pd.create_new_dataset(
dataset, api, args, resume, fields=fields,
session_file=session_file, path=path, log=log)
datasets[0] = dataset
# rebuild fields structure for new ids and fields
csv_properties.update({'objective_field': args.objective_name_,
'objective_field_present': True})
fields = pd.get_fields_structure(dataset, csv_properties)
args.objective_id_ = get_objective_id(args, fields)
if args.multi_label and dataset and multi_label_data is None:
multi_label_data = l.get_multi_label_data(dataset)
(args.objective_field,
labels,
all_labels,
multi_label_fields) = l.multi_label_sync(args.objective_field,
labels,
multi_label_data,
fields, multi_label_fields)
if dataset:
# retrieves max_categories data, if any
args.max_categories = get_metadata(dataset, 'max_categories',
args.max_categories)
other_label = get_metadata(dataset, 'other_label',
other_label)
if fields and args.export_fields:
fields.summary_csv(os.path.join(path, args.export_fields))
if args.model_file:
# model is retrieved from the contents of the given local JSON file
model, csv_properties, fields = u.read_local_resource(
args.model_file,
csv_properties=csv_properties)
models = [model]
model_ids = [model['resource']]
ensemble_ids = []
elif args.ensemble_file:
# model is retrieved from the contents of the given local JSON file
ensemble, csv_properties, fields = u.read_local_resource(
args.ensemble_file,
csv_properties=csv_properties)
model_ids = ensemble['object']['models'][:]
ensemble_ids = [ensemble['resource']]
models = model_ids[:]
model = retrieve_resource(args.retrieve_api_,
models[0],
query_string=r.ALL_FIELDS_QS)
models[0] = model
else:
# model is retrieved from the remote object
models, model_ids, ensemble_ids, resume = pm.models_processing(
datasets, models, model_ids,
api, args, resume, fields=fields,
session_file=session_file, path=path, log=log, labels=labels,
multi_label_data=multi_label_data, other_label=other_label)
if models:
model = models[0]
single_model = len(models) == 1
# If multi-label flag is set and no training_set was provided, label
# info is extracted from the user_metadata. If models belong to an
# ensemble, the ensemble must be retrieved to get the user_metadata.
if model and args.multi_label and multi_label_data is None:
if len(ensemble_ids) > 0 and isinstance(ensemble_ids[0], dict):
resource = ensemble_ids[0]
elif belongs_to_ensemble(model):
ensemble_id = get_ensemble_id(model)
resource = r.get_ensemble(ensemble_id, api=api,
verbosity=args.verbosity,
session_file=session_file)
else:
resource = model
multi_label_data = l.get_multi_label_data(resource)
# We update the model's public state if needed
if model:
if (isinstance(model, basestring) or
bigml.api.get_status(model)['code'] != bigml.api.FINISHED):
if not args.evaluate and not a.has_train(args) and \
not a.has_test(args):
query_string = MINIMUM_MODEL
elif not args.test_header:
query_string = r.ALL_FIELDS_QS
else:
query_string = "%s;%s" % (r.ALL_FIELDS_QS, r.FIELDS_QS)
model = u.check_resource(model, api.get_model,
query_string=query_string)
models[0] = model
if (args.black_box or args.white_box or
(args.shared_flag and r.shared_changed(args.shared, model))):
model_args = {}
if args.shared_flag and r.shared_changed(args.shared, model):
model_args.update(shared=args.shared)
if args.black_box or args.white_box:
model_args.update(r.set_publish_model_args(args))
if model_args:
model = r.update_model(model, model_args, args,
api=api, path=path,
session_file=session_file)
models[0] = model
# We get the fields of the model if we haven't got
# them yet and need them
if model and not args.evaluate and (a.has_test(args) or
args.export_fields):
# if we are using boosted ensembles to predict, activate boosting
if model['object'].get('boosted_ensemble'):
args.boosting = True
# If more than one model, use the full field structure
if (not single_model and not args.multi_label and
belongs_to_ensemble(model)):
if len(ensemble_ids) > 0:
ensemble_id = ensemble_ids[0]
args.ensemble_ids_ = ensemble_ids
else:
ensemble_id = get_ensemble_id(model)
fields = pm.get_model_fields(
model, csv_properties, args, single_model=single_model,
multi_label_data=multi_label_data)
# Free memory after getting fields
# local_ensemble = None
gc.collect()
# Fills in all_labels from user_metadata
if args.multi_label and not all_labels:
(args.objective_field,
labels,
all_labels,
multi_label_fields) = l.multi_label_sync(args.objective_field, labels,
multi_label_data, fields,
multi_label_fields)
if model:
# retrieves max_categories data, if any
args.max_categories = get_metadata(model, 'max_categories',
args.max_categories)
other_label = get_metadata(model, 'other_label',
other_label)
if fields and args.export_fields:
fields.summary_csv(os.path.join(path, args.export_fields))
# If predicting
if (models and (a.has_test(args) or (test_dataset and args.remote))
and not args.evaluate):
models_per_label = 1
if test_dataset is None:
test_dataset = get_test_dataset(args)
if args.multi_label:
# When prediction starts from existing models, the
# multi_label_fields can be retrieved from the user_metadata
# in the models
if args.multi_label_fields is None and multi_label_fields:
multi_label_field_names = [field[1] for field
in multi_label_fields]
args.multi_label_fields = ",".join(multi_label_field_names)
test_set = ps.multi_label_expansion(
args.test_set, args.test_header, args, path,
labels=labels, session_file=session_file, input_flag=True)[0]
test_set_header = True
# Remote predictions: predictions are computed as batch predictions
# in bigml.com except when --no-batch flag is set on or multi-label
# or max-categories are used
if (args.remote and not args.no_batch and not args.multi_label
and not args.method == COMBINATION):
# create test source from file
test_name = "%s - test" % args.name
if args.test_source is None:
test_properties = ps.test_source_processing(
api, args, resume, session_file=session_file,
path=path, log=log)
(test_source, resume, csv_properties,
test_fields) = test_properties
else:
test_source_id = bigml.api.get_source_id(args.test_source)
test_source = api.check_resource(test_source_id)
if test_dataset is None:
# create test dataset from test source
dataset_args = r.set_basic_dataset_args(args, name=test_name)
test_dataset, resume = pd.alternative_dataset_processing(
test_source, "test", dataset_args, api, args,
resume, session_file=session_file, path=path, log=log)
else:
test_dataset_id = bigml.api.get_dataset_id(test_dataset)
test_dataset = api.check_resource(test_dataset_id)
csv_properties.update(objective_field=None,
objective_field_present=False)
test_fields = pd.get_fields_structure(test_dataset,
csv_properties)
if args.to_dataset and args.dataset_off:
model = api.check_resource(model['resource'],
query_string=r.ALL_FIELDS_QS)
model_fields = Fields(model)
objective_field_name = model_fields.field_name( \
model_fields.objective_field)
if objective_field_name in test_fields.fields_by_name.keys():
args.prediction_name = "%s (predicted)" % \
objective_field_name
batch_prediction_args = r.set_batch_prediction_args(
args, fields=fields,
dataset_fields=test_fields)
remote_predict(model, test_dataset, batch_prediction_args, args,
api, resume, prediction_file=output,
session_file=session_file, path=path, log=log)
else:
models_per_label = args.number_of_models
if (args.multi_label and len(ensemble_ids) > 0
and args.number_of_models == 1):
# use case where ensembles are read from a file
models_per_label = len(models) / len(ensemble_ids)
predict(models, fields, args, api=api, log=log,
resume=resume, session_file=session_file, labels=labels,
models_per_label=models_per_label, other_label=other_label,
multi_label_data=multi_label_data)
# When combine_votes flag is used, retrieve the predictions files saved
# in the comma separated list of directories and combine them
if args.votes_files_:
model_id = re.sub(r'.*(model_[a-f0-9]{24})__predictions\.csv$',
r'\1', args.votes_files_[0]).replace("_", "/")
try:
model = u.check_resource(model_id, api.get_model)
except ValueError, exception:
sys.exit("Failed to get model %s: %s" % (model_id, str(exception)))
local_model = Model(model)
message = u.dated("Combining votes.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
combine_votes(args.votes_files_, local_model.to_prediction,
output, method=args.method)
def i_get_the_missing_values(step):
resource = world.dataset
fields = Fields(resource['fields'])
world.step_result = fields.missing_counts()
def source_processing(api,
args,
resume,
csv_properties=None,
multi_label_data=None,
session_file=None,
path=None,
log=None):
"""Creating or retrieving a data source from input arguments
"""
source = None
fields = None
if (args.training_set or
(hasattr(args, "evaluate") and args.evaluate and args.test_set)):
# If resuming, try to extract args.source form log files
if resume:
message = u.dated("Source not found. Resuming.\n")
resume, args.source = c.checkpoint(c.is_source_created,
path,
debug=args.debug,
message=message,
log_file=session_file,
console=args.verbosity)
# If neither a previous source, dataset or model are provided.
# we create a new one. Also if --evaluate and test data are provided
# we create a new dataset to test with.
data_set, data_set_header = r.data_to_source(args)
if data_set is not None:
# Check if there's a created project for it
args.project_id = pp.project_processing(api,
args,
resume,
session_file=session_file,
path=path,
log=log)
source_args = r.set_source_args(args,
multi_label_data=multi_label_data,
data_set_header=data_set_header)
source = r.create_source(data_set, source_args, args, api, path,
session_file, log)
# If a source is provided either through the command line or in resume
# steps, we use it.
elif args.source:
source = bigml.api.get_source_id(args.source)
# If we already have source, we check that is finished, extract the
# fields, and update them if needed.
if source:
source = r.get_source(source, api, args.verbosity, session_file)
if 'source_parser' in source['object']:
source_parser = source['object']['source_parser']
if 'missing_tokens' in source_parser:
csv_properties['missing_tokens'] = (
source_parser['missing_tokens'])
if 'locale' in source_parser:
csv_properties['data_locale'] = source_parser['locale']
# No changes if user locale is the one in the source.
if (args.user_locale is not None and bigml_locale(
args.user_locale) == source_parser['locale']):
args.user_locale = None
fields = Fields(source['object']['fields'], **csv_properties)
if (args.field_attributes_ or args.types_ or args.user_locale
or args.json_args.get('source')):
# avoid updating project_id in source
project_id, args.project_id = args.project_id, None
source_args = r.set_source_args(args, fields=fields)
source = r.update_source(source, source_args, args, api,
session_file)
args.project_id = project_id
fields = Fields(source['object']['fields'], **csv_properties)
return source, resume, csv_properties, fields
if penalty is None:
penalty = DEFAULT_PENALTY
# retrieving the first dataset in the file
try:
with open(datasets_file, u.open_mode("r")) as datasets_handler:
dataset_id = datasets_handler.readline().strip()
except IOError, exc:
sys.exit("Could not read the generated datasets file: %s" % str(exc))
try:
stored_dataset = u.storage_file_name(args.output_dir, dataset_id)
with open(stored_dataset, u.open_mode("r")) as dataset_handler:
dataset = json.loads(dataset_handler.read())
except IOError:
dataset = api.check_resource(dataset_id, query_string=ALL_FIELDS_QS)
# initial feature set
fields = Fields(dataset)
excluded_features = ([] if args.exclude_features is None else
args.exclude_features.split(args.args_separator))
try:
excluded_ids = [
fields.field_id(feature) for feature in excluded_features
]
objective_id = fields.field_id(objective_name)
except ValueError, exc:
sys.exit(exc)
field_ids = [
field_id for field_id in fields.preferred_fields()
if field_id != objective_id and not field_id in excluded_ids
]
field_ids.sort()
# headers are extended with a column per field
def i_get_the_missing_values(step):
resource = world.dataset
fields = Fields(resource['fields'])
world.step_result = fields.missing_counts()
def test_source_processing(api,
args,
resume,
name=None,
csv_properties=None,
session_file=None,
path=None,
log=None):
"""Creating or retrieving a test data source from input arguments
"""
test_source = None
fields = None
if csv_properties is None:
csv_properties = {}
if args.test_set and args.remote:
# If resuming, try to extract args.source form log files
if resume:
message = u.dated("Test source not found. Resuming.\n")
resume, args.test_source = c.checkpoint(c.is_source_created,
path,
suffix="_test",
debug=args.debug,
message=message,
log_file=session_file,
console=args.verbosity)
if not resume:
source_args = r.set_source_args(args,
name=name,
data_set_header=args.test_header)
test_source = r.create_source(args.test_set,
source_args,
args,
api,
path,
session_file,
log,
source_type="test")
# If a source is provided either through the command line or in resume
# steps, we use it.
elif args.test_source:
test_source = bigml.api.get_source_id(args.test_source)
# If we already have source, we check that is finished, extract the
# fields, and update them if needed.
if test_source:
test_source = r.get_source(test_source, api, args.verbosity,
session_file)
if 'source_parser' in test_source['object']:
source_parser = test_source['object']['source_parser']
if 'missing_tokens' in source_parser:
csv_properties['missing_tokens'] = (
source_parser['missing_tokens'])
if 'locale' in source_parser:
csv_properties['data_locale'] = source_parser['locale']
if (args.user_locale is not None and bigml_locale(
args.user_locale) == source_parser['locale']):
args.user_locale = None
fields = Fields(test_source['object']['fields'], **csv_properties)
if (args.field_attributes_ or args.types_ or args.user_locale
or args.json_args.get('source')):
# avoid updating project_id in source
project_id, args.project_id = args.project_id, None
test_source_args = r.set_source_args(args, fields=fields)
test_source = r.update_source(test_source, source_args, args, api,
session_file)
args.project_id = project_id
fields = Fields(source['object']['fields'], **csv_properties)
return test_source, resume, csv_properties, fields
def create_kfold_datasets_file(args, api, command_obj, resume=False):
"""Create the kfold dataset resources and store their ids in a file
one per line
"""
message = ('Creating the kfold datasets............\n')
u.log_message(message, log_file=session_file, console=args.verbosity)
if args.output_dir is None:
args.output_dir = a.NOW
csv_properties = {}
fields = None
dataset = None
datasets = []
if args.dataset_file:
# dataset is retrieved from the contents of the given local JSON file
model_dataset, csv_properties, fields = u.read_local_resource(
args.dataset_file, csv_properties=csv_properties)
if not args.datasets:
datasets = [model_dataset]
dataset = model_dataset
else:
datasets = u.read_datasets(args.datasets)
dataset_id = dataset['resource']
elif args.dataset:
dataset_id = bigml.api.get_dataset_id(args.dataset)
datasets = [dataset_id]
elif args.dataset_ids:
datasets = args.dataset_ids
dataset_id = datasets[0]
if dataset_id:
if not dataset:
dataset = api.check_resource(dataset_id,
query_string=ALL_FIELDS_QS)
try:
args.objective_field = int(args.objective_field)
except (TypeError, ValueError):
pass
# if the user provided no objective field, try to use the one in the
# dataset
if args.objective_field is None:
try:
args.objective_field = dataset['object']['objective_field'][
'column_number']
except KeyError:
pass
# check that kfold_field is unique
fields = Fields(dataset,
objective_field=args.objective_field,
objective_field_present=True)
if args.random_fields:
default_candidates_limits(args, fields)
try:
objective_id = fields.field_id(fields.objective_field)
objective_name = fields.field_name(objective_id)
except ValueError, exc:
sys.exit(exc)
kfold_field_name = avoid_duplicates(DEFAULT_KFOLD_FIELD, fields)
# create jsons to generate partial datasets
selecting_file_list, resume = create_kfold_json(args,
kfold_field_name,
objective_id,
resume=resume)
# generate test datasets
datasets_file, resume = create_kfold_datasets(dataset_id,
args,
selecting_file_list,
command_obj,
resume=resume)
return datasets_file, objective_name, resume
def best_first_search(datasets_file, api, args, common_options,
staleness=None, penalty=None, objective_name=None,
resume=False):
"""Selecting the fields to be used in the model construction
"""
counter = 0
loop_counter = 0
features_file = os.path.normpath(os.path.join(args.output_dir,
FEATURES_LOG))
with open(features_file, 'w', 0) as features_handler:
features_writer = csv.writer(features_handler, lineterminator="\n")
features_writer.writerow([
"step", "state", "score", "metric_value", "best_score"])
features_handler.flush()
if staleness is None:
staleness = DEFAULT_STALENESS
if penalty is None:
penalty = DEFAULT_PENALTY
# retrieving the first dataset in the file
try:
with open(datasets_file) as datasets_handler:
dataset_id = datasets_handler.readline().strip()
except IOError, exc:
sys.exit("Could not read the generated datasets file: %s" %
str(exc))
dataset = api.check_resource(dataset_id, api.get_dataset)
# initial feature set
fields = Fields(dataset)
excluded_features = ([] if args.exclude_features is None else
args.exclude_features.split(
args.args_separator))
excluded_ids = [fields.field_id(feature) for
feature in excluded_features]
objective_id = fields.field_id(objective_name)
field_ids = [field_id for field_id in fields.preferred_fields()
if field_id != objective_id and
not field_id in excluded_ids]
initial_state = [False for field_id in field_ids]
open_list = [(initial_state, - float('inf'), -float('inf'))]
closed_list = []
best_state, best_score, best_metric_value = open_list[0]
best_unchanged_count = 0
metric = args.maximize
while best_unchanged_count < staleness and open_list:
loop_counter += 1
features_set = find_max_state(open_list)
state, score, metric_value = features_set
features_writer.writerow([
loop_counter, [int(in_set) for in_set in state],
score, metric_value, best_score])
features_handler.flush()
state_fields = [fields.field_name(field_ids[index])
for (index, in_set) in enumerate(state) if in_set]
closed_list.append(features_set)
open_list.remove(features_set)
if (score - EPSILON) > best_score:
best_state, best_score, best_metric_value = features_set
best_unchanged_count = 0
if state_fields:
message = 'New best state: %s\n' % (state_fields)
u.log_message(message, log_file=session_file,
console=args.verbosity)
if metric in PERCENT_EVAL_METRICS:
message = '%s = %0.2f%% (score = %s)\n' % (
metric.capitalize(), metric_value * 100, score)
else:
message = '%s = %f (score = %s)\n' % (
metric.capitalize(),metric_value, score)
u.log_message(message, log_file=session_file,
console=args.verbosity)
else:
best_unchanged_count += 1
children = expand_state(state)
for child in children:
if (child not in [state for state, _, _ in open_list] and
child not in [state for state, _, _ in closed_list]):
input_fields = [fields.field_name(field_id)
for (i, field_id)
in enumerate(field_ids) if child[i]]
# create models and evaluation with input_fields
args.model_fields = args.args_separator.join(input_fields)
counter += 1
(score,
metric_value,
metric,
resume) = kfold_evaluate(datasets_file,
args, counter, common_options,
penalty=penalty, resume=resume,
metric=metric)
open_list.append((child, score, metric_value))
best_features = [fields.field_name(field_ids[i]) for (i, score)
in enumerate(best_state) if score]
message = (u'The best feature subset is: %s \n'
% u", ".join(best_features))
u.log_message(message, log_file=session_file, console=1)
if metric in PERCENT_EVAL_METRICS:
message = (u'%s = %0.2f%%\n' % (metric.capitalize(),
(best_metric_value * 100)))
else:
message = (u'%s = %f\n' % (metric.capitalize(), best_metric_value))
u.log_message(message, log_file=session_file, console=1)
message = (u'Evaluated %d/%d feature subsets\n' %
((len(open_list) + len(closed_list)), 2 ** len(field_ids)))
u.log_message(message, log_file=session_file, console=1)
def compute_output(api, args):
""" Creates a dataset using the `training_set`.
"""
source = None
dataset = None
fields = None
other_label = OTHER
multi_label_data = None
multi_label_fields = []
datasets = None
# variables from command-line options
resume = args.resume_
output = args.output
check_args_coherence(args)
path = u.check_dir(output)
session_file = "%s%s%s" % (path, os.sep, SESSIONS_LOG)
csv_properties = {}
# If logging is required set the file for logging
log = None
if args.log_file:
u.check_dir(args.log_file)
log = args.log_file
# If --clear_logs the log files are cleared
clear_log_files([log])
# labels to be used in multi-label expansion
labels = (None if args.labels is None else [
label.strip() for label in args.labels.split(args.args_separator)
])
if labels is not None:
labels = sorted([label for label in labels])
# multi_label file must be preprocessed to obtain a new extended file
if args.multi_label and args.training_set is not None:
(args.training_set, multi_label_data) = ps.multi_label_expansion(
args.training_set,
args.train_header,
args,
path,
labels=labels,
session_file=session_file)
args.train_header = True
args.objective_field = multi_label_data["objective_name"]
all_labels = l.get_all_labels(multi_label_data)
if not labels:
labels = all_labels
else:
all_labels = labels
if args.objective_field:
csv_properties.update({'objective_field': args.objective_field})
if args.source_file:
# source is retrieved from the contents of the given local JSON file
source, csv_properties, fields = u.read_local_resource(
args.source_file, csv_properties=csv_properties)
else:
# source is retrieved from the remote object
source, resume, csv_properties, fields = ps.source_processing(
api,
args,
resume,
csv_properties=csv_properties,
multi_label_data=multi_label_data,
session_file=session_file,
path=path,
log=log)
if source is not None:
args.source = bigml.api.get_source_id(source)
if args.multi_label and source:
multi_label_data = l.get_multi_label_data(source)
(args.objective_field, labels, all_labels,
multi_label_fields) = l.multi_label_sync(args.objective_field, labels,
multi_label_data, fields,
multi_label_fields)
if fields and args.export_fields:
fields.summary_csv(os.path.join(path, args.export_fields))
if args.dataset_file:
# dataset is retrieved from the contents of the given local JSON file
model_dataset, csv_properties, fields = u.read_local_resource(
args.dataset_file, csv_properties=csv_properties)
if not args.datasets:
datasets = [model_dataset]
dataset = model_dataset
else:
datasets = u.read_datasets(args.datasets)
if not datasets:
# dataset is retrieved from the remote object
datasets, resume, csv_properties, fields = pd.dataset_processing(
source,
api,
args,
resume,
fields=fields,
csv_properties=csv_properties,
multi_label_data=multi_label_data,
session_file=session_file,
path=path,
log=log)
if datasets:
dataset = datasets[-1]
if args.to_csv is not None:
resume = pd.export_dataset(dataset,
api,
args,
resume,
session_file=session_file,
path=path)
# Now we have a dataset, let's check if there's an objective_field
# given by the user and update it in the fields structure
args.objective_id_ = get_objective_id(args, fields)
# If test_split is used, split the dataset in a training and a test dataset
# according to the given split
if args.test_split > 0:
dataset, _, resume = pd.split_processing(
dataset,
api,
args,
resume,
multi_label_data=multi_label_data,
session_file=session_file,
path=path,
log=log)
datasets[0] = dataset
# Check if the dataset has a categorical objective field and it
# has a max_categories limit for categories
if args.max_categories > 0 and len(datasets) == 1:
if pd.check_max_categories(fields.fields[args.objective_id_]):
distribution = pd.get_categories_distribution(
dataset, args.objective_id_)
if distribution and len(distribution) > args.max_categories:
categories = [element[0] for element in distribution]
other_label = pd.create_other_label(categories, other_label)
datasets, resume = pd.create_categories_datasets(
dataset,
distribution,
fields,
args,
api,
resume,
session_file=session_file,
path=path,
log=log,
other_label=other_label)
else:
sys.exit("The provided objective field is not categorical nor "
"a full terms only text field. "
"Only these fields can be used with"
" --max-categories")
# If any of the transformations is applied,
# generate a new dataset from the given list of datasets
if args.new_dataset:
dataset, resume = pd.create_new_dataset(datasets,
api,
args,
resume,
fields=fields,
session_file=session_file,
path=path,
log=log)
datasets = [dataset]
# Check if the dataset has a generators file associated with it, and
# generate a new dataset with the specified field structure. Also
# if the --to-dataset flag is used to clone or sample the original dataset
if args.new_fields or args.sample_rate != 1 or \
(args.lisp_filter or args.json_filter) and not a.has_source(args):
if fields is None:
if isinstance(dataset, basestring):
dataset = u.check_resource(dataset, api=api)
fields = Fields(dataset, csv_properties)
args.objective_id_ = get_objective_id(args, fields)
args.objective_name_ = fields.field_name(args.objective_id_)
dataset, resume = pd.create_new_dataset(dataset,
api,
args,
resume,
fields=fields,
session_file=session_file,
path=path,
log=log)
datasets[0] = dataset
# rebuild fields structure for new ids and fields
csv_properties.update({
'objective_field': args.objective_name_,
'objective_field_present': True
})
fields = pd.get_fields_structure(dataset, csv_properties)
args.objective_id_ = get_objective_id(args, fields)
if args.multi_label and dataset and multi_label_data is None:
multi_label_data = l.get_multi_label_data(dataset)
(args.objective_field, labels, all_labels,
multi_label_fields) = l.multi_label_sync(args.objective_field, labels,
multi_label_data, fields,
multi_label_fields)
if dataset:
# retrieves max_categories data, if any
args.max_categories = get_metadata(dataset, 'max_categories',
args.max_categories)
other_label = get_metadata(dataset, 'other_label', other_label)
if fields and args.export_fields:
fields.summary_csv(os.path.join(path, args.export_fields))
u.print_generated_files(path,
log_file=session_file,
verbosity=args.verbosity)
if args.reports:
clear_reports(path)
if args.upload:
upload_reports(args.reports, path)
