def analysis(job, sliceno, slices):
#Divide the subject list so that each sliceno processes their own subjects.
sublist = array_split(options.subjects, slices)[sliceno]
for epo in options.epoch:
for sub in sublist:
fname = job.input_filename(
f'./{sub}/6-ClassificationData/{sub}_{epo}_data_amp.mat')
cdata = loadmat(fname, chars_as_strings=True,
simplify_cells=True)[f'{sub}_{epo}_data_amp']
directory = f'./data/Visual/{sub}/7-ClassifierTraining/'
os.makedirs(os.path.dirname(directory), exist_ok=True)
#Partition of the data
cfg = {'classifiernumber': 40, 'fold': 10}
datapart = dp.mvpa_datapartition(cfg, cdata)
#Partition of the data
print(
'Data was partitioned and feature reduction for each partition is complete...'
)
#Commented out because datapartition is by far the largest dataobject and takes ridicolous space, of course no problem if one has more than like 50 Gb
#job.save({f'{sub}_{epo}_datapart_amp': datapart}, directory + f'{sub}_{epo}_datapart_amp.pickle')
#Train the cross validated classifier
cfg = {
'training_algorithm': 1,
'fold': 10,
'classifiernumber': 40,
'category_model': ['Face', 'Landmark', 'Object']
}
crossval = tcc.mvpa_traincrossvalclassifier(cfg, datapart)
print('Cross Validated model successfully performed')
job.save({f'{sub}_{epo}_crossvalclass_amp': crossval},
directory + f'{sub}_{epo}_crossvalclass_amp.pickle')
#Performance
cfg = {
'performance': 1,
'category_model': ['Face', 'Landmark', 'Object'],
'classifiernumber': 40
}
crossval_perf = cp.mvpa_classifierperf(cfg, crossval)
print('Performance calculated')
job.save(
{f'{sub}_{epo}_crossvalclass_performance_amp': crossval_perf},
directory +
f'{sub}_{epo}_crossvalclass_performance_amp.pickle')
#Serial code after analysis to aggregate results, or do something which must be serial.
#def synthesis(analysis_res):
# pass
def iClass(epo):
directory = f'./data/Visual/{sub}/7-ClassifierTraining/' #directory for saving of the data
#Create the new directory in case it doesn't already exist
os.makedirs(os.path.dirname(directory), exist_ok=True)
#Python handles loading .mat files almost better than matlab itself
cdata = loadmat(
f'./Visual/{sub}/6-ClassificationData/{sub}_{epo}_data',
chars_as_strings=True,
simplify_cells=True
)[f'{sub}_{epo}_data'] #Skip first key since it's unecessary.
#Partition of the data
cfg = {'classifiernumber': 20, 'fold': 10}
datapart = mvpa_datapartition(cfg,
cdata,
random_state=random_state)
print(
'Data was partitioned and feature reduction for each partition is complete...'
)
#While it is possible to save the files as .mat files with scipy.io I didn't get it to work without giving severe penalaties on performance
with open(directory + f'{sub}_{epo}_datapart.pkl', 'wb') as file:
dump({f'{sub}_{epo}_datapart': datapart}, file)
#Train the cross validated classifier
cfg = {
'training_algorithm': 1,
'fold': 10,
'classifiernumber': 20,
'category_model': ['Face', 'Landmark', 'Object']
}
crossvalclass = mvpa_traincrossvalclassifier(cfg, datapart)
del datapart #Deleting now redundant variable to free memory
print('Cross Validated model successfully performed')
#Save
with open(directory + f'{sub}_{epo}_crossvalclass.pkl',
'wb') as file:
dump({f'{sub}_{epo}_crossvalclass': crossvalclass}, file)
#Performance
cfg = {
'performance': 1,
'category_model': ['Face', 'Landmark', 'Object'],
'classifiernumber': 20
}
crossvalclass_performance = mvpa_classifierperf(cfg, crossvalclass)
print('Performance calculated')
#Save
with open(directory + f'{sub}_{epo}_crossvalclass_performance.pkl',
'wb') as file:
dump(
{
f'{sub}_{epo}_crossvalclass_performance':
crossvalclass_performance
}, file)
def Do11DatapartAmp(sub, epo):
"""Datapartition Amp part,
sub -> a subject string or similiar to process
epo -> the epoch for the subject
"""
#Connection Object to connect to object store
conn = connection.Connection(
region_name='RegionOne',
auth=dict(auth_url='https://xerces.ericsson.net:5000',
username='******',
password='******',
project_id='b5c7a743b2554fd6972b841d3e853ac9',
user_domain_name='xerces',
project_name='bcix'),
compute_api_version='3',
identity_interface='public')
#The container's name in the objectstore
ContainerName = 'DroneContainer'
#Creating bytesIO object to read data raw or specifically as a binary data to load in scipy.io
cFile = BytesIO()
#Retrieving data from object store
conn.get_object(
ContainerName,
f'Visual/{sub}/6-ClassificationData/{sub}_{epo}_data_amp.mat',
outfile=cFile)
cdata = sio.loadmat(cFile, chars_as_strings=True,
simplify_cells=True)[f'{sub}_{epo}_data_amp']
#Cleanup Crew, uncessary but nice
cFile.close()
conn.close()
#partition of the data
cfg = {'classifiernumber': 40, 'fold': 10}
return mvpa_datapartition(cfg, cdata)
from mvpa.mvpa_applycrossvalclassifierOptiOneLookupDict import mvpa_applycrossvalclassifier
from SingleThread.Do11TrainClassifier import Do11TrainClassifierNonAmp
from scipy.io import loadmat
from pickle import load
import numpy as np
import pytest
#Variables
datapart = None
crossdata = None
cfg = {'classifiernumber': 2, 'fold': 5}
testdata = loadmat('./testData/TestData_study_data.mat',
chars_as_strings=True,
simplify_cells=True)['testData_study_data']
datapart = mvpa_datapartition(cfg, testdata, random_state=0)
cfg = {
'training_algorithm': 1,
'fold': 5,
'classifiernumber': 2,
'category_model': ['Face', 'Landmark', 'Object']
}
crossdata = mvpa_traincrossvalclassifier(cfg, datapart)
cfg = {
'performance': 1,
'category_model': ['Face', 'Landmark', 'Object'],
'classifiernumber': 2
}
crossPerf = mvpa_classifierperf(cfg, crossdata)
def iClass(epo):
directory = f'./data/Visual/{sub}/7-ClassifierTraining/'
#Create dir, if not exist
os.makedirs(os.path.dirname(directory), exist_ok=True)
cdata = loadmat(
f'./Visual/{sub}/6-ClassificationData/{sub}_{epo}_data_amp',
chars_as_strings=True,
simplify_cells=True)[f'{sub}_{epo}_data_amp']
#partition of the data
cfg = {'classifiernumber': 40, 'fold': 10}
datapart_amp = mvpa_datapartition(cfg,
cdata,
random_state=random_state)
print(
'Data was partitioned and feature reduction for each partition is complete'
)
#Again saving data.
with open(directory + f'{sub}_{epo}_datapart_amp.pkl',
'wb') as file:
dump({f'{sub}_{epo}_datapart_amp': datapart_amp}, file)
#Train the cross validated classifier
cfg = {
'training_algorithm': 1,
'fold': 10,
'classifiernumber': 40,
'category_model': ['Face', 'Landmark', 'Object']
}
crossvalclass_amp = mvpa_traincrossvalclassifier(cfg, datapart_amp)
del datapart_amp #Free up memory
print('Cross Validated model successfully performed!')
with open(directory + f'{sub}_{epo}_crossvalclass_amp.pkl',
'wb') as file:
dump({f'{sub}_{epo}_crossvalclass_amp': crossvalclass_amp},
file)
#Performance
cfg = {
'performance': 1,
'category_model': ['Face', 'Landmark', 'Object'],
'classifiernumber': 40
}
crossvalclass_performance_amp = mvpa_classifierperf(
cfg, crossvalclass_amp)
print('Performance calculated!')
#save
with open(
directory +
f'{sub}_{epo}_crossvalclass_performance_amp.pkl',
'wb') as file:
dump(
{
f'{sub}_{epo}_crossvalclass_performance_amp':
crossvalclass_performance_amp
}, file)