def create_datasets(all_data): train_data, train_metadata = all_data['train'] valid_data, valid_metadata = all_data['valid'] finaltrain_data, finaltrain_metadata = all_data['finaltrain'] test_data, test_metadata = all_data['test'] lbl = np.array([int(data[1]) for data in test_data]) spatial_dimensions = 1 def reduce_dimensionality(mlproblem_data, mlproblem_metadata): mlproblem_metadata['input_size'] = 3 # we need to change the input size from 6 to 3. return [mlproblem_data[0][:3] , mlproblem_data[1]] if spatial_dimensions ==1: import mlpython.mlproblems.classification as mlpb trainset = mlpb.ClassificationProblem(train_data, train_metadata) validset = trainset.apply_on(valid_data,valid_metadata) finaltrainset = trainset.apply_on(finaltrain_data,finaltrain_metadata) testset = trainset.apply_on(test_data,test_metadata) elif spatial_dimensions ==0: import mlpython.mlproblems.generic as mlpg trainset = mlpg.PreprocessedProblem(data = train_data , metadata = train_metadata , preprocess = reduce_dimensionality) validset = trainset.apply_on(valid_data, valid_metadata) testset = trainset.apply_on(test_data, test_metadata) finaltrainset = trainset.apply_on(finaltrain_data, finaltrain_metadata) import mlpython.mlproblems.classification as mlpb trainset = mlpb.ClassificationProblem(trainset, trainset.metadata) validset = trainset.apply_on(validset,validset.metadata) finaltrainset = trainset.apply_on(finaltrainset,finaltrainset.metadata) testset = trainset.apply_on(testset,testset.metadata) return {'trainset':trainset,'validset':validset ,'finaltrainset':finaltrainset, 'testset':testset ,'ground_truth':lbl}
def load_data(dataset_directory, dataset_name): print "Loading datasets ..." import os repo = os.environ.get('MLPYTHON_DATASET_REPO') if repo is None: raise ValueError( 'environment variable MLPYTHON_DATASET_REPO is not defined') dataset_dir = os.path.join( os.environ.get('MLPYTHON_DATASET_REPO') + '/' + dataset_directory, dataset_name) input_size = 6 spatial_dimensions = 1 all_data = data_utils.load_data(dir_path=dataset_dir, input_size=input_size, train_filename=None, test_filename=None, background_filename=None, load_to_memory=False) train_data, train_metadata = all_data['train'] valid_data, valid_metadata = all_data['valid'] finaltrain_data, finaltrain_metadata = all_data['finaltrain'] test_data, test_metadata = all_data['test'] lbl = np.array([int(data[1]) for data in test_data]) def reduce_dimensionality(mlproblem_data, mlproblem_metadata): mlproblem_metadata[ 'input_size'] = 3 # we need to change the input size from 6 to 3. return [mlproblem_data[0][:3], mlproblem_data[1]] if spatial_dimensions == 1: import mlpython.mlproblems.classification as mlpb trainset = mlpb.ClassificationProblem(train_data, train_metadata) validset = trainset.apply_on(valid_data, valid_metadata) finaltrainset = trainset.apply_on(finaltrain_data, finaltrain_metadata) testset = trainset.apply_on(test_data, test_metadata) elif spatial_dimensions == 0: import mlpython.mlproblems.generic as mlpg trainset = mlpg.PreprocessedProblem(data=train_data, metadata=train_metadata, preprocess=reduce_dimensionality) validset = trainset.apply_on(valid_data, valid_metadata) testset = trainset.apply_on(test_data, test_metadata) finaltrainset = trainset.apply_on(finaltrain_data, finaltrain_metadata) import mlpython.mlproblems.classification as mlpb trainset = mlpb.ClassificationProblem(trainset, trainset.metadata) validset = trainset.apply_on(validset, validset.metadata) finaltrainset = trainset.apply_on(finaltrainset, finaltrainset.metadata) testset = trainset.apply_on(testset, testset.metadata) return { 'finaltrainset': finaltrainset, 'testset': testset, 'ground_truth': lbl }
def get_classification_problem(name, dataset_dir=None, load_to_memory=True, **kw): """ Creates train/valid/test classification MLProblems from dataset ``name``. ``name`` must be one of the supported dataset (see variable ``classification_names`` of this module). Option ``load_to_memory`` determines whether the dataset should be loaded into memory or always read from its files. If environment variable MLPYTHON_DATASET_REPO has been set to a valid directory path, this function will look into its appropriate subdirectory to find the dataset. Alternatively the subdirectory path can be given by the user through option ``dataset_dir``. """ if name not in classification_names: raise ValueError('dataset ' + name + ' unknown for classification learning') exec 'import mlpython.datasets.' + name + ' as mldataset' if dataset_dir is None: # Try to find dataset in MLPYTHON_DATASET_REPO import os repo = os.environ.get('MLPYTHON_DATASET_REPO') if repo is None: raise ValueError( 'environment variable MLPYTHON_DATASET_REPO is not defined') dataset_dir = os.environ.get('MLPYTHON_DATASET_REPO') + '/' + name all_data = mldataset.load(dataset_dir, load_to_memory=load_to_memory, **kw) train_data, train_metadata = all_data['train'] valid_data, valid_metadata = all_data['valid'] test_data, test_metadata = all_data['test'] import mlpython.mlproblems.classification as mlpb trainset = mlpb.ClassificationProblem(train_data, train_metadata) validset = trainset.apply_on(valid_data, valid_metadata) testset = trainset.apply_on(test_data, test_metadata) return trainset, validset, testset
train_data, train_metadata = all_data['train'] valid_data, valid_metadata = all_data['valid'] finaltrain_data, finaltrain_metadata = all_data['finaltrain'] test_data, test_metadata = all_data['test'] def reduce_dimensionality(mlproblem_data, mlproblem_metadata): mlproblem_metadata[ 'input_size'] = 3 # we need to change the input size from 6 to 3. return [mlproblem_data[0][:3], mlproblem_data[1]] if spatial_dimensions == 1: import mlpython.mlproblems.classification as mlpb trainset = mlpb.ClassificationProblem(train_data, train_metadata) validset = trainset.apply_on(valid_data, valid_metadata) finaltrainset = trainset.apply_on(finaltrain_data, finaltrain_metadata) testset = trainset.apply_on(test_data, test_metadata) elif spatial_dimensions == 0: import mlpython.mlproblems.generic as mlpg trainset = mlpg.PreprocessedProblem(data=train_data, metadata=train_metadata, preprocess=reduce_dimensionality) validset = trainset.apply_on(valid_data, valid_metadata) testset = trainset.apply_on(test_data, test_metadata) finaltrainset = trainset.apply_on(finaltrain_data, finaltrain_metadata) import mlpython.mlproblems.classification as mlpb trainset = mlpb.ClassificationProblem(trainset, trainset.metadata) validset = trainset.apply_on(validset, validset.metadata)
pb3 = mlpbgen.MergedProblem([pb2, pb1]) print 'pb3:' for example in pb3: print example print 'metadata:', pb3.metadata print '' pb4 = mlpbgen.PreprocessedProblem(pb3, preprocess=features) print 'pb4:' for example in pb4: print example print 'metadata:', pb4.metadata print '' pb5 = mlpbclass.ClassificationProblem(pb4) print 'pb5:' for example in pb5: print example print 'metadata:', pb5.metadata print '' pb6 = mlpbclass.ClassSubsetProblem(pb5, subset=set(['A', 'C'])) print 'pb6 (final):' for example in pb6: print example print 'metadata:', pb6.metadata print '' pb7 = mlpbgen.SubsetFieldsProblem(pb6, fields=[0, 0, 1]) print 'pb7 (final):'
def test_mlproblem_combinations(): """ Test a combination of many different MLProblems. """ raw_data = zip(range(6), ['A', 'A', 'B', 'C', 'A', 'B']) metadata = {'length': 6, 'targets': ['A', 'B', 'C'], 'input_size': 1} def features(example, metadata): metadata['input_size'] = 2 return ((example[0], example[0]), example[1]) pb1 = mlpbgen.MLProblem(raw_data, metadata) print 'pb1', pb1.metadata pb2 = mlpbgen.SubsetProblem(pb1, subset=set([1, 3, 5])) print 'pb2', pb2.metadata pb3 = mlpbgen.MergedProblem([pb2, pb1]) print 'pb3', pb3.metadata pb4 = mlpbgen.PreprocessedProblem(pb3, preprocess=features) print 'pb4', pb4.metadata pb5 = mlpbclass.ClassificationProblem(pb4) print 'pb5', pb5.metadata pb6 = mlpbclass.ClassSubsetProblem(pb5, subset=set(['A', 'C'])) print 'pb6', pb6.metadata pb7 = mlpbgen.SubsetFieldsProblem(pb6, fields=[0, 0, 1]) print 'pb7', pb7.metadata final_data = [[(1, 1), (1, 1), 0], [(3, 3), (3, 3), 1], [(0, 0), (0, 0), 0], [(1, 1), (1, 1), 0], [(3, 3), (3, 3), 1], [(4, 4), (4, 4), 0]] final_metadata = { 'input_size': 2, 'targets': set(['A', 'C']), 'class_to_id': { 'A': 0, 'C': 1 } } for ex1, ex2 in zip(pb7, final_data): assert cmp(ex1, ex2) == 0 print pb7.metadata, final_metadata assert cmp(pb7.metadata, final_metadata) == 0 raw_data2 = zip(range(6, 10), ['C', 'B', 'A', 'C']) metadata2 = {'length': 4, 'targets': ['A', 'B', 'C'], 'input_size': 1} pbtest = pb7.apply_on(raw_data2, metadata2) final_test_data = [[(6, 6), (6, 6), 1], [(8, 8), (8, 8), 0], [(9, 9), (9, 9), 1]] final_test_metadata = { 'input_size': 2, 'targets': set(['A', 'C']), 'class_to_id': { 'A': 0, 'C': 1 } } for ex1, ex2 in zip(pbtest, final_test_data): assert cmp(ex1, ex2) == 0 assert cmp(pbtest.metadata, final_test_metadata) == 0