def analysis(job, sliceno, slices): #Divide the subject list so that each sliceno processes their own subjects. sublist = array_split(options.subjects, slices)[sliceno] for epo in options.epoch: for sub in sublist: fname = job.input_filename( f'./{sub}/6-ClassificationData/{sub}_{epo}_data_amp.mat') cdata = loadmat(fname, chars_as_strings=True, simplify_cells=True)[f'{sub}_{epo}_data_amp'] directory = f'./data/Visual/{sub}/7-ClassifierTraining/' os.makedirs(os.path.dirname(directory), exist_ok=True) #Partition of the data cfg = {'classifiernumber': 40, 'fold': 10} datapart = dp.mvpa_datapartition(cfg, cdata) #Partition of the data print( 'Data was partitioned and feature reduction for each partition is complete...' ) #Commented out because datapartition is by far the largest dataobject and takes ridicolous space, of course no problem if one has more than like 50 Gb #job.save({f'{sub}_{epo}_datapart_amp': datapart}, directory + f'{sub}_{epo}_datapart_amp.pickle') #Train the cross validated classifier cfg = { 'training_algorithm': 1, 'fold': 10, 'classifiernumber': 40, 'category_model': ['Face', 'Landmark', 'Object'] } crossval = tcc.mvpa_traincrossvalclassifier(cfg, datapart) print('Cross Validated model successfully performed') job.save({f'{sub}_{epo}_crossvalclass_amp': crossval}, directory + f'{sub}_{epo}_crossvalclass_amp.pickle') #Performance cfg = { 'performance': 1, 'category_model': ['Face', 'Landmark', 'Object'], 'classifiernumber': 40 } crossval_perf = cp.mvpa_classifierperf(cfg, crossval) print('Performance calculated') job.save( {f'{sub}_{epo}_crossvalclass_performance_amp': crossval_perf}, directory + f'{sub}_{epo}_crossvalclass_performance_amp.pickle') #Serial code after analysis to aggregate results, or do something which must be serial. #def synthesis(analysis_res): # pass
def iClass(epo): directory = f'./data/Visual/{sub}/7-ClassifierTraining/' #directory for saving of the data #Create the new directory in case it doesn't already exist os.makedirs(os.path.dirname(directory), exist_ok=True) #Python handles loading .mat files almost better than matlab itself cdata = loadmat( f'./Visual/{sub}/6-ClassificationData/{sub}_{epo}_data', chars_as_strings=True, simplify_cells=True )[f'{sub}_{epo}_data'] #Skip first key since it's unecessary. #Partition of the data cfg = {'classifiernumber': 20, 'fold': 10} datapart = mvpa_datapartition(cfg, cdata, random_state=random_state) print( 'Data was partitioned and feature reduction for each partition is complete...' ) #While it is possible to save the files as .mat files with scipy.io I didn't get it to work without giving severe penalaties on performance with open(directory + f'{sub}_{epo}_datapart.pkl', 'wb') as file: dump({f'{sub}_{epo}_datapart': datapart}, file) #Train the cross validated classifier cfg = { 'training_algorithm': 1, 'fold': 10, 'classifiernumber': 20, 'category_model': ['Face', 'Landmark', 'Object'] } crossvalclass = mvpa_traincrossvalclassifier(cfg, datapart) del datapart #Deleting now redundant variable to free memory print('Cross Validated model successfully performed') #Save with open(directory + f'{sub}_{epo}_crossvalclass.pkl', 'wb') as file: dump({f'{sub}_{epo}_crossvalclass': crossvalclass}, file) #Performance cfg = { 'performance': 1, 'category_model': ['Face', 'Landmark', 'Object'], 'classifiernumber': 20 } crossvalclass_performance = mvpa_classifierperf(cfg, crossvalclass) print('Performance calculated') #Save with open(directory + f'{sub}_{epo}_crossvalclass_performance.pkl', 'wb') as file: dump( { f'{sub}_{epo}_crossvalclass_performance': crossvalclass_performance }, file)
def Do11DatapartAmp(sub, epo): """Datapartition Amp part, sub -> a subject string or similiar to process epo -> the epoch for the subject """ #Connection Object to connect to object store conn = connection.Connection( region_name='RegionOne', auth=dict(auth_url='https://xerces.ericsson.net:5000', username='******', password='******', project_id='b5c7a743b2554fd6972b841d3e853ac9', user_domain_name='xerces', project_name='bcix'), compute_api_version='3', identity_interface='public') #The container's name in the objectstore ContainerName = 'DroneContainer' #Creating bytesIO object to read data raw or specifically as a binary data to load in scipy.io cFile = BytesIO() #Retrieving data from object store conn.get_object( ContainerName, f'Visual/{sub}/6-ClassificationData/{sub}_{epo}_data_amp.mat', outfile=cFile) cdata = sio.loadmat(cFile, chars_as_strings=True, simplify_cells=True)[f'{sub}_{epo}_data_amp'] #Cleanup Crew, uncessary but nice cFile.close() conn.close() #partition of the data cfg = {'classifiernumber': 40, 'fold': 10} return mvpa_datapartition(cfg, cdata)
from mvpa.mvpa_applycrossvalclassifierOptiOneLookupDict import mvpa_applycrossvalclassifier from SingleThread.Do11TrainClassifier import Do11TrainClassifierNonAmp from scipy.io import loadmat from pickle import load import numpy as np import pytest #Variables datapart = None crossdata = None cfg = {'classifiernumber': 2, 'fold': 5} testdata = loadmat('./testData/TestData_study_data.mat', chars_as_strings=True, simplify_cells=True)['testData_study_data'] datapart = mvpa_datapartition(cfg, testdata, random_state=0) cfg = { 'training_algorithm': 1, 'fold': 5, 'classifiernumber': 2, 'category_model': ['Face', 'Landmark', 'Object'] } crossdata = mvpa_traincrossvalclassifier(cfg, datapart) cfg = { 'performance': 1, 'category_model': ['Face', 'Landmark', 'Object'], 'classifiernumber': 2 } crossPerf = mvpa_classifierperf(cfg, crossdata)
def iClass(epo): directory = f'./data/Visual/{sub}/7-ClassifierTraining/' #Create dir, if not exist os.makedirs(os.path.dirname(directory), exist_ok=True) cdata = loadmat( f'./Visual/{sub}/6-ClassificationData/{sub}_{epo}_data_amp', chars_as_strings=True, simplify_cells=True)[f'{sub}_{epo}_data_amp'] #partition of the data cfg = {'classifiernumber': 40, 'fold': 10} datapart_amp = mvpa_datapartition(cfg, cdata, random_state=random_state) print( 'Data was partitioned and feature reduction for each partition is complete' ) #Again saving data. with open(directory + f'{sub}_{epo}_datapart_amp.pkl', 'wb') as file: dump({f'{sub}_{epo}_datapart_amp': datapart_amp}, file) #Train the cross validated classifier cfg = { 'training_algorithm': 1, 'fold': 10, 'classifiernumber': 40, 'category_model': ['Face', 'Landmark', 'Object'] } crossvalclass_amp = mvpa_traincrossvalclassifier(cfg, datapart_amp) del datapart_amp #Free up memory print('Cross Validated model successfully performed!') with open(directory + f'{sub}_{epo}_crossvalclass_amp.pkl', 'wb') as file: dump({f'{sub}_{epo}_crossvalclass_amp': crossvalclass_amp}, file) #Performance cfg = { 'performance': 1, 'category_model': ['Face', 'Landmark', 'Object'], 'classifiernumber': 40 } crossvalclass_performance_amp = mvpa_classifierperf( cfg, crossvalclass_amp) print('Performance calculated!') #save with open( directory + f'{sub}_{epo}_crossvalclass_performance_amp.pkl', 'wb') as file: dump( { f'{sub}_{epo}_crossvalclass_performance_amp': crossvalclass_performance_amp }, file)