def transform_args(command_args, flags, api): """Transforms the formatted argument strings into structured arguments """ attribute_args(command_args) # Parses dataset generators in json format if provided try: if command_args.new_fields: json_generators = u.read_json(command_args.new_fields) command_args.dataset_json_generators = json_generators else: command_args.dataset_json_generators = {} except AttributeError: pass # Parses multi-dataset attributes in json such as field maps try: if command_args.multi_dataset_attributes: multi_dataset_json = u.read_json( command_args.multi_dataset_attributes) command_args.multi_dataset_json = multi_dataset_json else: command_args.multi_dataset_json = {} except AttributeError: pass transform_dataset_options(command_args, api) script_ids = None command_args.script_ids = [] # Parses script/id if provided. try: if command_args.scripts: script_ids = u.read_resources(command_args.scripts) if len(script_ids) == 1: command_args.script = script_ids[0] command_args.script_ids = script_ids except AttributeError: pass # Retrieve script/ids if provided. try: if command_args.script_tag: script_ids = script_ids.extend( u.list_ids(api.list_scripts, "tags__in=%s" % command_args.script_tag)) if len(script_ids) == 1: command_args.script = script_ids[0] command_args.script_ids = script_ids except AttributeError: pass # Reads a json filter if provided. try: if command_args.json_filter: json_filter = u.read_json_filter(command_args.json_filter) command_args.json_filter = json_filter except AttributeError: pass # Reads a lisp filter if provided. try: if command_args.lisp_filter: lisp_filter = u.read_lisp_filter(command_args.lisp_filter) command_args.lisp_filter = lisp_filter except AttributeError: pass # Adds default tags unless that it is requested not to do so. try: if command_args.no_tag: command_args.tag.append('BigMLer') command_args.tag.append('BigMLer_%s' % NOW) except AttributeError: pass # Checks combined votes method try: if (command_args.method and command_args.method != COMBINATION_LABEL and not command_args.method in COMBINATION_WEIGHTS.keys()): command_args.method = 0 else: combiner_methods = dict([[value, key] for key, value in COMBINER_MAP.items()]) combiner_methods[COMBINATION_LABEL] = COMBINATION command_args.method = combiner_methods.get(command_args.method, 0) except AttributeError: pass # Checks missing_strategy try: if (command_args.missing_strategy and not (command_args.missing_strategy in MISSING_STRATEGIES.keys())): command_args.missing_strategy = 0 else: command_args.missing_strategy = MISSING_STRATEGIES.get( command_args.missing_strategy, 0) except AttributeError: pass try: # Old value for --prediction-info='full data' maps to 'full' if command_args.prediction_info == 'full data': print("WARNING: 'full data' is a deprecated value. Use" " 'full' instead") command_args.prediction_info = FULL_FORMAT except AttributeError: pass # Parses class, weight pairs for objective weight try: if command_args.objective_weights: objective_weights = (u.read_objective_weights( command_args.objective_weights)) command_args.objective_weights_json = objective_weights except AttributeError: pass try: command_args.multi_label_fields_list = [] if command_args.multi_label_fields is not None: multi_label_fields = command_args.multi_label_fields.strip() command_args.multi_label_fields_list = multi_label_fields.split( command_args.args_separator) except AttributeError: pass # Sets shared_flag if --shared or --unshared has been used command_args.shared_flag = '--shared' in flags or '--unshared' in flags # Set remote on if scoring a trainind dataset in bigmler anomaly try: if command_args.score: command_args.remote = True if not "--prediction-info" in flags: command_args.prediction_info = FULL_FORMAT except AttributeError: pass command_args.has_supervised_ = ( (hasattr(command_args, 'model') and command_args.model) or (hasattr(command_args, 'models') and command_args.models) or (hasattr(command_args, 'ensemble') and command_args.ensemble) or (hasattr(command_args, 'ensembles') and command_args.ensembles) or (hasattr(command_args, 'model_tag') and command_args.model_tag) or (hasattr(command_args, 'logistic_regression') and command_args.logistic_regression) or (hasattr(command_args, 'logistic_regressions') and command_args.logistic_regressions) or (hasattr(command_args, 'logistic_regression_tag') and command_args.logistic_regression_tag) or (hasattr(command_args, 'deepnet') and command_args.deepnet) or (hasattr(command_args, 'deepnets') and command_args.deepnets) or (hasattr(command_args, 'deepnet_tag') and command_args.deepnet_tag) or (hasattr(command_args, 'ensemble_tag') and command_args.ensemble_tag)) command_args.has_models_ = ( command_args.has_supervised_ or (hasattr(command_args, 'cluster') and command_args.cluster) or (hasattr(command_args, 'clusters') and command_args.clusters) or (hasattr(command_args, 'anomaly') and command_args.anomaly) or (hasattr(command_args, 'anomalies') and command_args.anomalies) or (hasattr(command_args, 'cluster_tag') and command_args.cluster_tag) or (hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag)) command_args.has_datasets_ = ( (hasattr(command_args, 'dataset') and command_args.dataset) or (hasattr(command_args, 'datasets') and command_args.datasets) or (hasattr(command_args, 'dataset_ids') and command_args.dataset_ids) or (hasattr(command_args, 'dataset_tag') and command_args.dataset_tag)) command_args.has_test_datasets_ = ( (hasattr(command_args, 'test_dataset') and command_args.test_dataset) or (hasattr(command_args, 'test_datasets') and command_args.test_datasets) or (hasattr(command_args, 'test_dataset_tag') and command_args.test_dataset_tag)) command_args.new_dataset = ( (hasattr(command_args, 'datasets_json') and command_args.datasets_json) or (hasattr(command_args, 'multi_dataset') and command_args.multi_dataset) or (hasattr(command_args, 'juxtapose') and command_args.juxtapose) or (hasattr(command_args, 'sql_query') and command_args.sql_query) or (hasattr(command_args, 'sql_output_fields') and command_args.sql_output_fields) or (hasattr(command_args, 'json_query') and command_args.json_query))
def transform_args(command_args, flags, api, user_defaults): """Transforms the formatted argument strings into structured arguments """ # Parses attributes in json format if provided command_args.json_args = {} for resource_type in RESOURCE_TYPES: attributes_file = getattr(command_args, "%s_attributes" % resource_type, None) if attributes_file is not None: command_args.json_args[resource_type] = u.read_json( attributes_file) else: command_args.json_args[resource_type] = {} # Parses dataset generators in json format if provided if command_args.new_fields: json_generators = u.read_json(command_args.new_fields) command_args.dataset_json_generators = json_generators else: command_args.dataset_json_generators = {} # Parses multi-dataset attributes in json such as field maps if command_args.multi_dataset_attributes: multi_dataset_json = u.read_json(command_args.multi_dataset_attributes) command_args.multi_dataset_json = multi_dataset_json else: command_args.multi_dataset_json = {} dataset_ids = None command_args.dataset_ids = [] # Parses dataset/id if provided. if command_args.datasets: dataset_ids = u.read_datasets(command_args.datasets) if len(dataset_ids) == 1: command_args.dataset = dataset_ids[0] command_args.dataset_ids = dataset_ids # Reading test dataset ids is delayed till the very moment of use to ensure # that the newly generated resources files can be used there too command_args.test_dataset_ids = [] # Retrieve dataset/ids if provided. if command_args.dataset_tag: dataset_ids = dataset_ids.extend( u.list_ids(api.list_datasets, "tags__in=%s" % command_args.dataset_tag)) if len(dataset_ids) == 1: command_args.dataset = dataset_ids[0] command_args.dataset_ids = dataset_ids # Reads a json filter if provided. if command_args.json_filter: json_filter = u.read_json_filter(command_args.json_filter) command_args.json_filter = json_filter # Reads a lisp filter if provided. if command_args.lisp_filter: lisp_filter = u.read_lisp_filter(command_args.lisp_filter) command_args.lisp_filter = lisp_filter # Adds default tags unless that it is requested not to do so. if command_args.no_tag: command_args.tag.append('BigMLer') command_args.tag.append('BigMLer_%s' % NOW) # Checks combined votes method try: if (command_args.method and command_args.method != COMBINATION_LABEL and not (command_args.method in COMBINATION_WEIGHTS.keys())): command_args.method = 0 else: combiner_methods = dict([[value, key] for key, value in COMBINER_MAP.items()]) combiner_methods[COMBINATION_LABEL] = COMBINATION command_args.method = combiner_methods.get(command_args.method, 0) except AttributeError: pass # Checks missing_strategy try: if (command_args.missing_strategy and not (command_args.missing_strategy in MISSING_STRATEGIES.keys())): command_args.missing_strategy = 0 else: command_args.missing_strategy = MISSING_STRATEGIES.get( command_args.missing_strategy, 0) except AttributeError: pass # Adds replacement=True if creating ensemble and nothing is specified try: if (command_args.number_of_models > 1 and not command_args.replacement and not '--no-replacement' in flags and not 'replacement' in user_defaults and not '--no-randomize' in flags and not 'randomize' in user_defaults and not '--sample-rate' in flags and not 'sample_rate' in user_defaults): command_args.replacement = True except AttributeError: pass try: # Old value for --prediction-info='full data' maps to 'full' if command_args.prediction_info == 'full data': print("WARNING: 'full data' is a deprecated value. Use" " 'full' instead") command_args.prediction_info = FULL_FORMAT except AttributeError: pass # Parses class, weight pairs for objective weight try: if command_args.objective_weights: objective_weights = (u.read_objective_weights( command_args.objective_weights)) command_args.objective_weights_json = objective_weights except AttributeError: pass try: command_args.multi_label_fields_list = [] if command_args.multi_label_fields is not None: multi_label_fields = command_args.multi_label_fields.strip() command_args.multi_label_fields_list = multi_label_fields.split( command_args.args_separator) except AttributeError: pass # Sets shared_flag if --shared or --unshared has been used if '--shared' in flags or '--unshared' in flags: command_args.shared_flag = True else: command_args.shared_flag = False # Set remote on if scoring a trainind dataset in bigmler anomaly try: if command_args.score: command_args.remote = True if not "--prediction-info" in flags: command_args.prediction_info = FULL_FORMAT except AttributeError: pass command_args.has_models_ = ( (hasattr(command_args, 'model') and command_args.model) or (hasattr(command_args, 'models') and command_args.models) or (hasattr(command_args, 'ensemble') and command_args.ensemble) or (hasattr(command_args, 'ensembles') and command_args.ensembles) or (hasattr(command_args, 'cluster') and command_args.cluster) or (hasattr(command_args, 'clusters') and command_args.clusters) or (hasattr(command_args, 'model_tag') and command_args.model_tag) or (hasattr(command_args, 'anomaly') and command_args.anomaly) or (hasattr(command_args, 'anomalies') and command_args.anomalies) or (hasattr(command_args, 'ensemble_tag') and command_args.ensemble_tag) or (hasattr(command_args, 'cluster_tag') and command_args.cluster_tag) or (hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag)) command_args.has_datasets_ = ( (hasattr(command_args, 'dataset') and command_args.dataset) or (hasattr(command_args, 'datasets') and command_args.datasets) or (hasattr(command_args, 'dataset_tag') and command_args.dataset_tag)) command_args.has_test_datasets_ = ( (hasattr(command_args, 'test_dataset') and command_args.test_dataset) or (hasattr(command_args, 'test_datasets') and command_args.test_datasets) or (hasattr(command_args, 'test_dataset_tag') and command_args.test_dataset_tag))
def transform_args(command_args, flags, api, user_defaults): """Transforms the formatted argument strings into structured arguments """ # Parses attributes in json format if provided command_args.json_args = {} json_attribute_options = { 'source': command_args.source_attributes, 'dataset': command_args.dataset_attributes, 'model': command_args.model_attributes, 'ensemble': command_args.ensemble_attributes, 'evaluation': command_args.evaluation_attributes, 'batch_prediction': command_args.batch_prediction_attributes} for resource_type, attributes_file in json_attribute_options.items(): if attributes_file is not None: command_args.json_args[resource_type] = u.read_json( attributes_file) else: command_args.json_args[resource_type] = {} # Parses dataset generators in json format if provided if command_args.new_fields: json_generators = u.read_json(command_args.new_fields) command_args.dataset_json_generators = json_generators else: command_args.dataset_json_generators = {} # Parses multi-dataset attributes in json such as field maps if command_args.multi_dataset_attributes: multi_dataset_json = u.read_json(command_args.multi_dataset_attributes) command_args.multi_dataset_json= multi_dataset_json else: command_args.multi_dataset_json = {} dataset_ids = None command_args.dataset_ids = [] # Parses dataset/id if provided. if command_args.datasets: dataset_ids = u.read_datasets(command_args.datasets) if len(dataset_ids) == 1: command_args.dataset = dataset_ids[0] command_args.dataset_ids = dataset_ids test_dataset_ids = None command_args.test_dataset_ids = [] # Parses dataset/id if provided. if command_args.test_datasets: test_dataset_ids = u.read_datasets(command_args.test_datasets) command_args.test_dataset_ids = test_dataset_ids # Retrieve dataset/ids if provided. if command_args.dataset_tag: dataset_ids = dataset_ids.extend( u.list_ids(api.list_datasets, "tags__in=%s" % command_args.dataset_tag)) if len(dataset_ids) == 1: command_args.dataset = dataset_ids[0] command_args.dataset_ids = dataset_ids # Reads a json filter if provided. if command_args.json_filter: json_filter = u.read_json_filter(command_args.json_filter) command_args.json_filter = json_filter # Reads a lisp filter if provided. if command_args.lisp_filter: lisp_filter = u.read_lisp_filter(command_args.lisp_filter) command_args.lisp_filter = lisp_filter # Adds default tags unless that it is requested not to do so. if command_args.no_tag: command_args.tag.append('BigMLer') command_args.tag.append('BigMLer_%s' % NOW) # Checks combined votes method if (command_args.method and command_args.method != COMBINATION_LABEL and not (command_args.method in COMBINATION_WEIGHTS.keys())): command_args.method = 0 else: combiner_methods = dict([[value, key] for key, value in COMBINER_MAP.items()]) combiner_methods[COMBINATION_LABEL] = COMBINATION command_args.method = combiner_methods.get(command_args.method, 0) # Checks missing_strategy if (command_args.missing_strategy and not (command_args.missing_strategy in MISSING_STRATEGIES.keys())): command_args.missing_strategy = 0 else: command_args.missing_strategy = MISSING_STRATEGIES.get( command_args.missing_strategy, 0) # Adds replacement=True if creating ensemble and nothing is specified if (command_args.number_of_models > 1 and not command_args.replacement and not '--no-replacement' in flags and not 'replacement' in user_defaults and not '--no-randomize' in flags and not 'randomize' in user_defaults and not '--sample-rate' in flags and not 'sample_rate' in user_defaults): command_args.replacement = True # Old value for --prediction-info='full data' maps to 'full' if command_args.prediction_info == 'full data': print "WARNING: 'full data' is a deprecated value. Use 'full' instead" command_args.prediction_info = FULL_FORMAT # Parses class, weight pairs for objective weight if command_args.objective_weights: objective_weights = ( u.read_objective_weights(command_args.objective_weights)) command_args.objective_weights_json = objective_weights command_args.multi_label_fields_list = [] if command_args.multi_label_fields is not None: multi_label_fields = command_args.multi_label_fields.strip() command_args.multi_label_fields_list = multi_label_fields.split( command_args.args_separator) # Sets shared_flag if --shared or --unshared has been used if '--shared' in flags or '--unshared' in flags: command_args.shared_flag = True else: command_args.shared_flag = False
def transform_args(command_args, flags, api, user_defaults): """Transforms the formatted argument strings into structured arguments """ attribute_args(command_args) # Parses dataset generators in json format if provided if command_args.new_fields: json_generators = u.read_json(command_args.new_fields) command_args.dataset_json_generators = json_generators else: command_args.dataset_json_generators = {} # Parses multi-dataset attributes in json such as field maps if command_args.multi_dataset_attributes: multi_dataset_json = u.read_json(command_args.multi_dataset_attributes) command_args.multi_dataset_json = multi_dataset_json else: command_args.multi_dataset_json = {} transform_dataset_options(command_args, api) # Reads a json filter if provided. if command_args.json_filter: json_filter = u.read_json_filter(command_args.json_filter) command_args.json_filter = json_filter # Reads a lisp filter if provided. if command_args.lisp_filter: lisp_filter = u.read_lisp_filter(command_args.lisp_filter) command_args.lisp_filter = lisp_filter # Adds default tags unless it is requested not to do so. if command_args.no_tag: command_args.tag.append('BigMLer') command_args.tag.append('BigMLer_%s' % NOW) # Checks combined votes method try: if (command_args.method and command_args.method != COMBINATION_LABEL and not (command_args.method in COMBINATION_WEIGHTS.keys())): command_args.method = 0 else: combiner_methods = dict( [[value, key] for key, value in COMBINER_MAP.items()]) combiner_methods[COMBINATION_LABEL] = COMBINATION command_args.method = combiner_methods.get(command_args.method, 0) except AttributeError: pass # Checks missing_strategy try: if (command_args.missing_strategy and not (command_args.missing_strategy in MISSING_STRATEGIES.keys())): command_args.missing_strategy = 0 else: command_args.missing_strategy = MISSING_STRATEGIES.get( command_args.missing_strategy, 0) except AttributeError: pass # Adds replacement=True if creating ensemble and nothing is specified try: if (command_args.number_of_models > 1 and not command_args.replacement and not '--no-replacement' in flags and not 'replacement' in user_defaults and not '--no-randomize' in flags and not 'randomize' in user_defaults and not '--sample-rate' in flags and not 'sample_rate' in user_defaults): command_args.replacement = True except AttributeError: pass try: # Old value for --prediction-info='full data' maps to 'full' if command_args.prediction_info == 'full data': print ("WARNING: 'full data' is a deprecated value. Use" " 'full' instead") command_args.prediction_info = FULL_FORMAT except AttributeError: pass # Parses class, weight pairs for objective weight try: if command_args.objective_weights: objective_weights = ( u.read_objective_weights(command_args.objective_weights)) command_args.objective_weights_json = objective_weights except AttributeError: pass try: command_args.multi_label_fields_list = [] if command_args.multi_label_fields is not None: multi_label_fields = command_args.multi_label_fields.strip() command_args.multi_label_fields_list = multi_label_fields.split( command_args.args_separator) except AttributeError: pass # Sets shared_flag if --shared or --unshared has been used if '--shared' in flags or '--unshared' in flags: command_args.shared_flag = True else: command_args.shared_flag = False # Set remote on if scoring a trainind dataset in bigmler anomaly try: if command_args.score: command_args.remote = True if not "--prediction-info" in flags: command_args.prediction_info = FULL_FORMAT except AttributeError: pass command_args.has_models_ = ( (hasattr(command_args, 'model') and command_args.model) or (hasattr(command_args, 'models') and command_args.models) or (hasattr(command_args, 'ensemble') and command_args.ensemble) or (hasattr(command_args, 'ensembles') and command_args.ensembles) or (hasattr(command_args, 'cluster') and command_args.cluster) or (hasattr(command_args, 'clusters') and command_args.clusters) or (hasattr(command_args, 'model_tag') and command_args.model_tag) or (hasattr(command_args, 'anomaly') and command_args.anomaly) or (hasattr(command_args, 'anomalies') and command_args.anomalies) or (hasattr(command_args, 'ensemble_tag') and command_args.ensemble_tag) or (hasattr(command_args, 'cluster_tag') and command_args.cluster_tag) or (hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag)) command_args.has_datasets_ = ( (hasattr(command_args, 'dataset') and command_args.dataset) or (hasattr(command_args, 'datasets') and command_args.datasets) or (hasattr(command_args, 'dataset_tag') and command_args.dataset_tag)) command_args.has_test_datasets_ = ( (hasattr(command_args, 'test_dataset') and command_args.test_dataset) or (hasattr(command_args, 'test_datasets') and command_args.test_datasets) or (hasattr(command_args, 'test_dataset_tag') and command_args.test_dataset_tag))
def transform_args(command_args, flags, api, user_defaults): """Transforms the formatted argument strings into structured arguments """ # Parses attributes in json format if provided command_args.json_args = {} json_attribute_options = { 'source': command_args.source_attributes, 'dataset': command_args.dataset_attributes, 'model': command_args.model_attributes, 'ensemble': command_args.ensemble_attributes, 'evaluation': command_args.evaluation_attributes, 'batch_prediction': command_args.batch_prediction_attributes } for resource_type, attributes_file in json_attribute_options.items(): if attributes_file is not None: command_args.json_args[resource_type] = u.read_json( attributes_file) else: command_args.json_args[resource_type] = {} # Parses dataset generators in json format if provided if command_args.new_fields: json_generators = u.read_json(command_args.new_fields) command_args.dataset_json_generators = json_generators else: command_args.dataset_json_generators = {} # Parses multi-dataset attributes in json such as field maps if command_args.multi_dataset_attributes: multi_dataset_json = u.read_json(command_args.multi_dataset_attributes) command_args.multi_dataset_json = multi_dataset_json else: command_args.multi_dataset_json = {} dataset_ids = None command_args.dataset_ids = [] # Parses dataset/id if provided. if command_args.datasets: dataset_ids = u.read_datasets(command_args.datasets) if len(dataset_ids) == 1: command_args.dataset = dataset_ids[0] command_args.dataset_ids = dataset_ids # Retrieve dataset/ids if provided. if command_args.dataset_tag: dataset_ids = dataset_ids.extend( u.list_ids(api.list_datasets, "tags__in=%s" % command_args.dataset_tag)) if len(dataset_ids) == 1: command_args.dataset = dataset_ids[0] command_args.dataset_ids = dataset_ids # Reads a json filter if provided. if command_args.json_filter: json_filter = u.read_json_filter(command_args.json_filter) command_args.json_filter = json_filter # Reads a lisp filter if provided. if command_args.lisp_filter: lisp_filter = u.read_lisp_filter(command_args.lisp_filter) command_args.lisp_filter = lisp_filter # Adds default tags unless that it is requested not to do so. if command_args.no_tag: command_args.tag.append('BigMLer') command_args.tag.append('BigMLer_%s' % NOW) # Checks combined votes method if (command_args.method and command_args.method != COMBINATION_LABEL and not (command_args.method in COMBINATION_WEIGHTS.keys())): command_args.method = 0 else: combiner_methods = dict([[value, key] for key, value in COMBINER_MAP.items()]) combiner_methods[COMBINATION_LABEL] = COMBINATION command_args.method = combiner_methods.get(command_args.method, 0) # Checks missing_strategy if (command_args.missing_strategy and not (command_args.missing_strategy in MISSING_STRATEGIES.keys())): command_args.missing_strategy = 0 else: command_args.missing_strategy = MISSING_STRATEGIES.get( command_args.missing_strategy, 0) # Adds replacement=True if creating ensemble and nothing is specified if (command_args.number_of_models > 1 and not command_args.replacement and not '--no-replacement' in flags and not 'replacement' in user_defaults and not '--no-randomize' in flags and not 'randomize' in user_defaults and not '--sample-rate' in flags and not 'sample_rate' in user_defaults): command_args.replacement = True # Old value for --prediction-info='full data' maps to 'full' if command_args.prediction_info == 'full data': print "WARNING: 'full data' is a deprecated value. Use 'full' instead" command_args.prediction_info = FULL_FORMAT # Parses class, weight pairs for objective weight if command_args.objective_weights: objective_weights = (u.read_objective_weights( command_args.objective_weights)) command_args.objective_weights_json = objective_weights command_args.multi_label_fields_list = [] if command_args.multi_label_fields is not None: multi_label_fields = command_args.multi_label_fields.strip() command_args.multi_label_fields_list = multi_label_fields.split(',')
def transform_args(command_args, flags, api): """Transforms the formatted argument strings into structured arguments """ attribute_args(command_args) # Parses dataset generators in json format if provided try: if command_args.new_fields: json_generators = u.read_json(command_args.new_fields) command_args.dataset_json_generators = json_generators else: command_args.dataset_json_generators = {} except AttributeError: pass # Parses multi-dataset attributes in json such as field maps try: if command_args.multi_dataset_attributes: multi_dataset_json = u.read_json( command_args.multi_dataset_attributes) command_args.multi_dataset_json = multi_dataset_json else: command_args.multi_dataset_json = {} except AttributeError: pass transform_dataset_options(command_args, api) script_ids = None command_args.script_ids = [] # Parses script/id if provided. try: if command_args.scripts: script_ids = u.read_resources(command_args.scripts) if len(script_ids) == 1: command_args.script = script_ids[0] command_args.script_ids = script_ids except AttributeError: pass # Retrieve script/ids if provided. try: if command_args.script_tag: script_ids = script_ids.extend( u.list_ids(api.list_scripts, "tags__in=%s" % command_args.script_tag)) if len(script_ids) == 1: command_args.script = script_ids[0] command_args.script_ids = script_ids except AttributeError: pass # Reads a json filter if provided. try: if command_args.json_filter: json_filter = u.read_json_filter(command_args.json_filter) command_args.json_filter = json_filter except AttributeError: pass # Reads a lisp filter if provided. try: if command_args.lisp_filter: lisp_filter = u.read_lisp_filter(command_args.lisp_filter) command_args.lisp_filter = lisp_filter except AttributeError: pass # Adds default tags unless that it is requested not to do so. try: if command_args.no_tag: command_args.tag.append('BigMLer') command_args.tag.append('BigMLer_%s' % NOW) except AttributeError: pass # Checks combined votes method try: if (command_args.method and command_args.method != COMBINATION_LABEL and not command_args.method in COMBINATION_WEIGHTS.keys()): command_args.method = 0 else: combiner_methods = dict( [[value, key] for key, value in COMBINER_MAP.items()]) combiner_methods[COMBINATION_LABEL] = COMBINATION command_args.method = combiner_methods.get(command_args.method, 0) except AttributeError: pass # Checks missing_strategy try: if (command_args.missing_strategy and not (command_args.missing_strategy in MISSING_STRATEGIES.keys())): command_args.missing_strategy = 0 else: command_args.missing_strategy = MISSING_STRATEGIES.get( command_args.missing_strategy, 0) except AttributeError: pass try: # Old value for --prediction-info='full data' maps to 'full' if command_args.prediction_info == 'full data': print ("WARNING: 'full data' is a deprecated value. Use" " 'full' instead") command_args.prediction_info = FULL_FORMAT except AttributeError: pass # Parses class, weight pairs for objective weight try: if command_args.objective_weights: objective_weights = ( u.read_objective_weights(command_args.objective_weights)) command_args.objective_weights_json = objective_weights except AttributeError: pass try: command_args.multi_label_fields_list = [] if command_args.multi_label_fields is not None: multi_label_fields = command_args.multi_label_fields.strip() command_args.multi_label_fields_list = multi_label_fields.split( command_args.args_separator) except AttributeError: pass # Sets shared_flag if --shared or --unshared has been used command_args.shared_flag = '--shared' in flags or '--unshared' in flags # Set remote on if scoring a trainind dataset in bigmler anomaly try: if command_args.score: command_args.remote = True if not "--prediction-info" in flags: command_args.prediction_info = FULL_FORMAT except AttributeError: pass command_args.has_supervised_ = ( (hasattr(command_args, 'model') and command_args.model) or (hasattr(command_args, 'models') and command_args.models) or (hasattr(command_args, 'ensemble') and command_args.ensemble) or (hasattr(command_args, 'ensembles') and command_args.ensembles) or (hasattr(command_args, 'model_tag') and command_args.model_tag) or (hasattr(command_args, 'logistic_regression') and command_args.logistic_regression) or (hasattr(command_args, 'logistic_regressions') and command_args.logistic_regressions) or (hasattr(command_args, 'logistic_regression_tag') and command_args.logistic_regression_tag) or (hasattr(command_args, 'deepnet') and command_args.deepnet) or (hasattr(command_args, 'deepnets') and command_args.deepnets) or (hasattr(command_args, 'deepnet_tag') and command_args.deepnet_tag) or (hasattr(command_args, 'ensemble_tag') and command_args.ensemble_tag)) command_args.has_models_ = ( command_args.has_supervised_ or (hasattr(command_args, 'cluster') and command_args.cluster) or (hasattr(command_args, 'clusters') and command_args.clusters) or (hasattr(command_args, 'anomaly') and command_args.anomaly) or (hasattr(command_args, 'anomalies') and command_args.anomalies) or (hasattr(command_args, 'cluster_tag') and command_args.cluster_tag) or (hasattr(command_args, 'anomaly_tag') and command_args.anomaly_tag)) command_args.has_datasets_ = ( (hasattr(command_args, 'dataset') and command_args.dataset) or (hasattr(command_args, 'datasets') and command_args.datasets) or (hasattr(command_args, 'dataset_tag') and command_args.dataset_tag)) command_args.has_test_datasets_ = ( (hasattr(command_args, 'test_dataset') and command_args.test_dataset) or (hasattr(command_args, 'test_datasets') and command_args.test_datasets) or (hasattr(command_args, 'test_dataset_tag') and command_args.test_dataset_tag))
def transform_args(command_args, flags, api, user_defaults): """Transforms the formatted argument strings into structured arguments """ # Parses attributes in json format if provided command_args.json_args = {} for resource_type in RESOURCE_TYPES: attributes_file = getattr(command_args, "%s_attributes" % resource_type, None) if attributes_file is not None: command_args.json_args[resource_type] = u.read_json(attributes_file) else: command_args.json_args[resource_type] = {} # Parses dataset generators in json format if provided if command_args.new_fields: json_generators = u.read_json(command_args.new_fields) command_args.dataset_json_generators = json_generators else: command_args.dataset_json_generators = {} # Parses multi-dataset attributes in json such as field maps if command_args.multi_dataset_attributes: multi_dataset_json = u.read_json(command_args.multi_dataset_attributes) command_args.multi_dataset_json = multi_dataset_json else: command_args.multi_dataset_json = {} dataset_ids = None command_args.dataset_ids = [] # Parses dataset/id if provided. if command_args.datasets: dataset_ids = u.read_datasets(command_args.datasets) if len(dataset_ids) == 1: command_args.dataset = dataset_ids[0] command_args.dataset_ids = dataset_ids test_dataset_ids = None command_args.test_dataset_ids = [] # Parses dataset/id if provided. if command_args.test_datasets: test_dataset_ids = u.read_datasets(command_args.test_datasets) command_args.test_dataset_ids = test_dataset_ids # Retrieve dataset/ids if provided. if command_args.dataset_tag: dataset_ids = dataset_ids.extend(u.list_ids(api.list_datasets, "tags__in=%s" % command_args.dataset_tag)) if len(dataset_ids) == 1: command_args.dataset = dataset_ids[0] command_args.dataset_ids = dataset_ids # Reads a json filter if provided. if command_args.json_filter: json_filter = u.read_json_filter(command_args.json_filter) command_args.json_filter = json_filter # Reads a lisp filter if provided. if command_args.lisp_filter: lisp_filter = u.read_lisp_filter(command_args.lisp_filter) command_args.lisp_filter = lisp_filter # Adds default tags unless that it is requested not to do so. if command_args.no_tag: command_args.tag.append("BigMLer") command_args.tag.append("BigMLer_%s" % NOW) # Checks combined votes method try: if ( command_args.method and command_args.method != COMBINATION_LABEL and not (command_args.method in COMBINATION_WEIGHTS.keys()) ): command_args.method = 0 else: combiner_methods = dict([[value, key] for key, value in COMBINER_MAP.items()]) combiner_methods[COMBINATION_LABEL] = COMBINATION command_args.method = combiner_methods.get(command_args.method, 0) except AttributeError: pass # Checks missing_strategy try: if command_args.missing_strategy and not (command_args.missing_strategy in MISSING_STRATEGIES.keys()): command_args.missing_strategy = 0 else: command_args.missing_strategy = MISSING_STRATEGIES.get(command_args.missing_strategy, 0) except AttributeError: pass # Adds replacement=True if creating ensemble and nothing is specified try: if ( command_args.number_of_models > 1 and not command_args.replacement and not "--no-replacement" in flags and not "replacement" in user_defaults and not "--no-randomize" in flags and not "randomize" in user_defaults and not "--sample-rate" in flags and not "sample_rate" in user_defaults ): command_args.replacement = True except AttributeError: pass # Old value for --prediction-info='full data' maps to 'full' if command_args.prediction_info == "full data": print "WARNING: 'full data' is a deprecated value. Use 'full' instead" command_args.prediction_info = FULL_FORMAT # Parses class, weight pairs for objective weight try: if command_args.objective_weights: objective_weights = u.read_objective_weights(command_args.objective_weights) command_args.objective_weights_json = objective_weights except AttributeError: pass try: command_args.multi_label_fields_list = [] if command_args.multi_label_fields is not None: multi_label_fields = command_args.multi_label_fields.strip() command_args.multi_label_fields_list = multi_label_fields.split(command_args.args_separator) except AttributeError: pass # Sets shared_flag if --shared or --unshared has been used if "--shared" in flags or "--unshared" in flags: command_args.shared_flag = True else: command_args.shared_flag = False command_args.has_models_ = ( (hasattr(command_args, "model") and command_args.model) or (hasattr(command_args, "models") and command_args.models) or (hasattr(command_args, "ensemble") and command_args.ensemble) or (hasattr(command_args, "ensembles") and command_args.ensembles) or (hasattr(command_args, "cluster") and command_args.cluster) or (hasattr(command_args, "clusters") and command_args.clusters) or (hasattr(command_args, "model_tag") and command_args.model_tag) or (hasattr(command_args, "anomaly") and command_args.anomaly) or (hasattr(command_args, "anomalies") and command_args.anomalies) or (hasattr(command_args, "ensemble_tag") and command_args.ensemble_tag) or (hasattr(command_args, "cluster_tag") and command_args.cluster_tag) or (hasattr(command_args, "anomaly_tag") and command_args.anomaly_tag) ) command_args.has_datasets_ = ( (hasattr(command_args, "dataset") and command_args.dataset) or (hasattr(command_args, "datasets") and command_args.datasets) or (hasattr(command_args, "dataset_tag") and command_args.dataset_tag) )