def handle_save_xml(self,force_save = False):
if not self.annotation_type == 'channel_corrector':
if self.annotation_type == 'corrector':
filename = self.save_dir_artifact + self.coder + '_' + windower.make_name(self.b) + '.xml'
elif self.save_dir_artifact != '':
filename = path.data + self.save_dir_artifact + self.filename.split('/')[-1]
else: filename = self.filename
save_ok = False
for be in self.bad_epochs:
if be.ok: save_ok = True
if save_ok and (time.time() - self.last_save > 60 or force_save == True):
print('saving:',filename)
print('nbad epochs:',len(self.bad_epochs))
self.last_save = time.time()
xml = xml_handler.xml_handler(bad_epochs=self.bad_epochs,filename=filename)
xml.bad_epochs2xml(multiplier = 10)
xml.write()
save_ok = False
if self.annotation_type == 'channel_corrector':
filename = self.save_dir_ch + self.coder + '_' + windower.make_name(self.b) + '.xml'
else: filename = self.filename_channels
for bc in self.bad_channels:
if bc.ok: save_ok = True
if bc.correct != 'correct':
bc.color == 'grey'
bc.correct = 'incorrect'
if save_ok and (time.time() - self.last_save > 60 or force_save == True):
print('saving:',filename)
print('nbad channels:',len(self.bad_channels))
self.last_save = time.time()
xml = xml_handler.xml_handler(bad_channels =self.bad_channels,filename = filename)
xml.bad_channels2xml(multiplier = 10)
xml.write()
def load_from_xml(self,filename = ''):
if os.path.isfile(filename): self.filename = filename
elif os.path.isfile(path.artifacts + filename): self.filename = path.artifacts + filename
elif os.path.isfile(self.filename): pass
elif os.path.isfile(self.filename_channels): pass
elif self.show_cnn_pred:
name = windower.make_name(self.b)
fn = glob.glob(path.artifact_cnn_xml + '*' + name + '*')
if len(fn) > 0: self.filename = fn[0]
else:
print('Auto generating filename based on block information.')
self.filename = path.artifacts + self.coder + '_pp' + str(self.pp_id) + '_exp-' + self.exp_type + '_bid-' + str(self.bid) + '.xml'
self.filename_channels = path.bad_channels+ self.coder + '_pp' + str(self.pp_id) + '_exp-' + self.exp_type + '_bid-' + str(self.bid) + '_channels.xml'
if os.path.isfile(self.filename):
print('loading bad epochs with filename:', self.filename)
xml = xml_handler.xml_handler(filename = self.filename)
self.bad_epochs = xml.xml2bad_epochs(multiplier = 0.1,remove_clean = True)
for be in self.bad_epochs:
if be.start == None or be.end == None:
print(be)
else:
self.boundaries.append(be.start)
self.boundaries.append(be.end)
if self.filename != '' and self.filename_channels == '':
self.filename_channels = self.filename.strip('.xml') + '_channels.xml'
if not os.path.isfile(self.filename_channels):
name= self.filename_channels.split('/')[-1]
self.filename_channels = path.bad_channels + name
print(self.filename_channels)
if os.path.isfile(self.filename_channels):
print('loading xml channels')
xml = xml_handler.xml_handler(filename = self.filename_channels)
temp= xml.xml2bad_channels(multiplier = 0.1)
self.bad_channels= [bc for bc in temp if bc.channel not in self.remove_ch]
if path.artifact_ch_cnn_xml in self.filename_channels:
print('loading from cnn directory, setting bc to incorrect.')
for bc in self.bad_channels:
bc.correct = 'incorrect'
for bc in self.bad_channels:
# if bc.channel in self.remove_ch: continue
if bc.annotation == 'all':self.complete_bad_channel.append(bc.channel)
if bc.start == None or bc.end == None:
print(bc)
else:
self.channel_boundaries.append(bc.start)
self.channel_boundaries.append(bc.end)
else: print('filename xml channels not found.',self.filename_channels)
def read_bad_channels(self):
'''read the bad channel from file, should be the corrected automatic annotations.
'''
self.auto_f_bc = get_xml_filename(self.name,
bad_type='bad_channel',
correction_type='auto')
self.man_f_bc = get_xml_filename(self.name,
bad_type='bad_channel',
correction_type='manual')
self.auto_bc = xml_handler.xml_handler(filename=self.auto_f_bc,
artifact_type='bad_channel')
self.man_bc = xml_handler.xml_handler(filename=self.man_f_bc,
artifact_type='bad_channel')
self.auto_bc.xml2bad_channels()
self.man_bc.xml2bad_channels()
def read_bad_epochs(self):
'''read the bad epochs from file, should be the corrected automatic annotations.
'''
self.f_be = get_xml_filename(self.name,
bad_type='bad_epoch',
corrected=self.corrected)
self.be = xml_handler.xml_handler(filename=self.f_be,
artifact_type='bad_epoch')
self.be.xml2bad_epochs()
def read_bad_epochs(self):
'''read the bad epochs from file, should be the corrected automatic annotations.
'''
self.auto_f_be = get_xml_filename(self.name,
bad_type='bad_epoch',
correction_type='auto')
self.man_f_be = get_xml_filename(self.name,
bad_type='bad_epoch',
correction_type='manual')
self.auto_be = xml_handler.xml_handler(filename=self.auto_f_be,
artifact_type='bad_epoch')
self.man_be = xml_handler.xml_handler(filename=self.man_f_be,
artifact_type='bad_epoch')
self.auto_be.xml2bad_epochs()
self.man_be.xml2bad_epochs()
self.auto_be.bad_epochs = [
be for be in self.auto_be.bad_epochs if be.annotation != 'clean'
]
self.man_be.bad_epochs = [
be for be in self.man_be.bad_epochs if be.annotation != 'clean'
]
def load_xml(f, filetype='artifacts'):
'''Read the xml file.
filetype channel or artifacts (epochs)
'''
xml = xml_handler.xml_handler(filename=f)
xml.load_xml()
if filetype == 'artifacts': xml.xml2bad_epochs()
elif filetype == 'channels': xml.xml2bad_channels()
else:
raise ValueError('filetype not recognized: artifacts or channels',
filetype)
return xml
def __init__(self, initializer, logger, writer):
"""
Arguments:
initializer {app1_initializer} -- initializer obsahujuci argumenty zo spustenia
logger {progress_logger} -- logger do ,ktoreho sa bude zapisovat priebeh
writer {file_writer} -- writer do ktoreho sa zapisu ohlasy
"""
self.initializer = initializer
self.logger = logger
self.writer = writer
self.connector = crepc_connector()
self.handler = xml_handler()
self.counter_zapisanych = 0
def load_annotations(self):
'''load the labels corresponding to this block in the bad_epoch filed and order the bad_epochs based on start time.'''
self.annotations = xml_handler.xml_handler(filename=self.fn_annotation)
self.bad_epochs = self.annotations.xml2bad_epochs()
self.bad_epochs.sort()
if len(self.bad_epochs) > 0:
be_block_st = self.bad_epochs[0].block_st_sample
if self.st_sample != be_block_st:
raise ValueError('Start sample block datastats', be_block_st,
'does not equal start sample bad_epoch',
self.st_sample)
else:
print('No bad epochs in this block.')
def __init__(self,
name,
ch_threshold=.4,
minimal_clean_duration=2000,
fo=None,
force_make=False,
corrected=True):
'''Create bads object to hold annotation information and save this to xml.
name name of block
ch_threshold percentage of block bad for a channel to be completely removed
minimal... the minimal time between artifacts (otherwise artifacts are combined)
fo fid2ort object, speeds up loading of blocks
force_make whether to overwrite existing bad xml
'''
if name == '-': return
if corrected:
self.f = path.bads_annotations + 'bads_' + name + '.xml'
else:
self.f = path.uncorrected_bads_annotations + 'bads_' + name + '.xml'
self.epoch_id = 1
self.name = name
self.fo = fo
self.force_make = force_make
self.corrected = corrected
if not force_make and os.path.isfile(self.f):
x = xml_handler.xml_handler(filename=self.f, artifact_type='bads')
bads = x.xml2bads()
self.__dict__.update(bads.__dict__)
else:
self.b = utils.name2block(name, fo)
self.minimal_clean_duration = minimal_clean_duration
self.read_all()
self.annotations()
self.block_duration = utils.load_100hz_numpy_block(
name).shape[-1] * 10
self.select_remove_ch(ch_threshold)
self.make_bads()
self.make_info()
self.save()
def compute_artifact_clean_duration():
exptype2int = {'o': 1, 'k': 2, 'ifadv': 3}
bad_pp = load_bad_pp()
fn_xml = glob.glob(path.artifacts_clean + '*')
fout = open('artifacts_duration.txt', 'w')
for f in fn_xml:
print(f)
xml_artifacts = xml_handler.xml_handler(filename=f)
xml_artifacts.load_xml()
xml_artifacts.xml2bad_epochs()
artifacts = xml_artifacts.bad_epochs
be = artifacts[0]
pp_id = be.pp_id
exp_type = exptype2int[be.exp_type]
bid = be.bid
clean = sum(make_duration_list(artifacts, 'clean'))
artifact = sum(make_duration_list(artifacts, 'artifact'))
other = sum(make_duration_list(artifacts, 'other'))
print(f, clean, artifact, other)
fout.write(','.join(
map(str, [pp_id, exp_type, bid, clean, artifact, other])) + '\n')
fout.close()
def save(self):
'''Write the object to xml.
'''
self.x = xml_handler.xml_handler(bads=self, artifact_type='bads')
self.x.make_bads2xml()
self.x.save(self.f)
def save(self):
'''Write the object to xml.
'''
x = xml_handler.xml_handler(bads=self, artifact_type='bads')
x.bads2xml()
x.save(path.bads_annotations + 'bads_' + self.name + '.xml')
def run(fo=None):
'''Create new xml files with two classes of bad_epochs: clean and artifact.
If block is part of bad_pp_artifact training clean is changed with other
all epochs are non overlapping and clean epochs are minimally 500ms in length.'''
exptype2int = {'o': 1, 'k': 2, 'ifadv': 3}
bad_pp = load_bad_pp()
fn = get_xml_files()
all_clean, all_artifact = 0, 0
pp = {}
for i, f in enumerate(fn):
print(i, f)
bad_epochs = get_bad_epochs(f)
be = copy.deepcopy(bad_epochs[0])
b = bad_epoch2block(be, fo)
#check whether block is part of bad_pp -> non annotated data is not clean
if [b.pp_id, exptype2int[b.exp_type], b.bid] in bad_pp:
default = 'other'
else:
default = 'clean'
artifacts = select_artifacts(f)
if len(artifacts) == 0:
be.start.x = 0
be.end.x = b.duration_sample
be.set_info()
be.epoch_id = '0.' + be.epoch_id
be.color = 'white'
be.annotation = default
artifacts = [be]
else:
artifacts = combine_overlaps(artifacts)
stiches = stitch_artifacts(artifacts)
stiches = stitch_stiches(stiches)
artifacts = combine_artifacts(artifacts, stiches)
check_artifacts(artifacts, fo, default)
for a in artifacts:
a.block_et_sample = b.et_sample
if default == 'other': clean = 0
else: clean = sum(make_duration_list(artifacts, default))
artifact = sum(make_duration_list(artifacts, 'artifact'))
a = artifacts[0]
if str(a.pp_id) + 'clean' not in pp.keys():
pp[str(a.pp_id) + 'clean'] = [clean]
else:
pp[str(a.pp_id) + 'clean'].append(clean)
if str(a.pp_id) + 'artifact' not in pp.keys():
pp[str(a.pp_id) + 'artifact'] = [artifact]
else:
pp[str(a.pp_id) + 'artifact'].append(artifact)
if artifact == 0: artifact = 1
if str(a.pp_id) + 'ratio' not in pp.keys():
pp[str(a.pp_id) + 'ratio'] = [clean / artifact]
else:
pp[str(a.pp_id) + 'ratio'].append(clean / artifact)
print(i, f, clean, artifact, clean / artifact)
all_clean += clean
all_artifact += artifact
print('_' * 9)
x = xml_handler.xml_handler(artifacts)
x.bad_epochs2xml()
x.save(path.artifacts_clean + f.split('/')[-1])
print(all_clean, all_artifact, all_clean / all_artifact)
return pp
def get_bad_epochs(filename):
'''Load the xml file and create the bad_epochs from the contents.'''
x = xml_handler.xml_handler(filename=filename)
x.load_xml()
bad_epochs = x.xml2bad_epochs()
return bad_epochs
def __init__(self, logger):
self.handler = xml_handler()
self.connector = crepc_connector()
self.logger = logger
self.known_inst = set()