def task_validate_rawdata_file(pk, **kwargs):
"""validates rawdata file - that is an archive holding voltage traces from
extracellular recordings to be a data to be analysed with spike sorting methods
:type pk: int
:param pk: pk of `Asset` entity
:returns: bool -- True if file validates, False else. Processing
log, including errors, will be written to the `Datafile` entity.
"""
# init and checks
valid = False
logger = Logger.get_logger(StringIO())
try:
obj = Asset.objects.get(id=pk)
assert obj.kind == "rd_file"
df = obj.content_object
except:
logger.log("ERROR")
return valid
try:
logger.log("looking at raw data file with pk: %s" % pk)
rd, sr = read_hdf5_arc(obj.data.path)
logger.log("found rd_file: %s" % obj.name)
len_rd_sec = rd.shape[0] / sr
logger.log("found data in %d channels, for %d sec" % (rd.shape[1], len_rd_sec))
# TODO: more checks?
logger.log("rd_file passed all checks")
valid = True
except Exception, ex:
logger.log("ERROR: rawdata file check: %s" % str(ex))
def test_logger_file(self):
fname = os.path.join(os.path.dirname(__file__), 'test_logger.txt')
if os.path.exists(fname):
os.remove(fname)
canvas = open(fname, 'a+')
logger = Logger.get_logger(canvas)
logger.log(*self.str_test)
self.assertEqual(logger.get_content().split(), self.str_test)
os.remove(fname)
def test_logger_file(self):
fname = os.path.join(os.path.dirname(__file__), 'test_logger.txt')
if os.path.exists(fname):
os.remove(fname)
canvas = open(fname, 'a+')
logger = Logger.get_logger(canvas)
logger.log(*self.str_test)
self.assertEqual(logger.get_content().split(), self.str_test)
os.remove(fname)
def task_run_modules(ana_pk, **kwargs):
"""run all enabled modules for an analysis
:type ana_pk: Analysis
:param ana_pk: Analysis entity
:keyword: any, will be passed to modules as parameters
:returns: True on success, False on error
"""
ana = None
success = None
logger = Logger.get_logger(StringIO(""))
try:
# get analysis
ana = Analysis.objects.get(pk=ana_pk.pk)
logger.log_delimiter_line()
logger.log("processing %s" % ana)
ana.status = Analysis.STATUS.running
ana.save()
# get module list
mod_list = ana.datafile.dataset.module_set.all()
assert mod_list, "module list is empty!"
# get file handles
logger.log("reading input files")
rd_file = ana.datafile.rd_file
gt_file = ana.datafile.gt_file
st_file = ana.st_file
rd, sampling_rate = read_hdf5_arc(rd_file.data.path)
if sampling_rate is not None:
kwargs.update(sampling_rate=sampling_rate)
ev_sts = read_gdf_sts(st_file.data.path)
gt_sts = read_gdf_sts(gt_file.data.path)
logger.log("done reading input files")
# apply modules
_tick_all = datetime.now()
for mod in mod_list:
logger.log_delimiter_line()
logger.log("starting {}".format(mod))
module_cls = mod.get_module_cls()
_tick_ = datetime.now()
module = module_cls(rd, gt_sts, ev_sts, logger, **kwargs)
module.apply()
module.save(mod, ana)
_tock_ = datetime.now()
logger.log("finished in {}".format(str(_tock_ - _tick_)))
del module, module_cls, _tick_, _tock_
_tock_all = datetime.now()
logger.log_delimiter_line()
logger.log("finished all module in {}".format(str(_tock_all - _tick_all)))
except Exception, ex:
logger.log_delimiter_line()
logger.log("ERROR ({}) :: {}".format(ex.__class__.__name__, ex))
success = False
def task_validate_spiketrain_file(pk, **kwargs):
"""validate spike train file - that is a text file in gdf format (space separated, 2col, [key,time])
:type pk: int
:param pk: pk of `Datafile` entity
:returns: bool -- True if file validates, False else. Processing
log, including errors, will be written to the `Datafile` entity.
"""
# init and checks
valid = False
logger = Logger.get_logger(StringIO())
try:
obj = Asset.objects.get(id=pk)
assert obj.kind == "st_file"
df = obj.content_object
except:
logger.log("ERROR")
return valid
try:
logger.log("looking at spike train file with pk: %s" % obj.id)
sts = read_gdf_sts(obj.data.path)
logger.log("found st_file: %s" % obj.name)
for st in sts:
if not isinstance(sts[st], sp.ndarray):
raise TypeError("spike train %s not ndarray" % st)
if not sts[st].ndim == 1:
raise ValueError("spike trains have to be ndim==1")
# TODO: more checks?
logger.log("st_file passed all checks")
valid = True
except Exception, ex:
logger.log('ERROR: spike train file check: %s' % str(ex))
def task_validate_spiketrain_file(pk, **kwargs):
"""validate spike train file - that is a text file in gdf format (space separated, 2col, [key,time])
:type pk: int
:param pk: pk of `Datafile` entity
:returns: bool -- True if file validates, False else. Processing
log, including errors, will be written to the `Datafile` entity.
"""
# init and checks
valid = False
logger = Logger.get_logger(StringIO())
try:
obj = Asset.objects.get(id=pk)
assert obj.kind == "st_file"
df = obj.content_object
except:
logger.log("ERROR")
return valid
try:
logger.log("looking at spike train file with pk: %s" % obj.id)
sts = read_gdf_sts(obj.data.path)
logger.log("found st_file: %s" % obj.name)
for st in sts:
if not isinstance(sts[st], sp.ndarray):
raise TypeError("spike train %s not ndarray" % st)
if not sts[st].ndim == 1:
raise ValueError("spike trains have to be ndim==1")
# TODO: more checks?
logger.log("st_file passed all checks")
valid = True
except Exception, ex:
logger.log('ERROR: spike train file check: %s' % str(ex))
def task_validate_rawdata_file(pk, **kwargs):
"""validates rawdata file - that is an archive holding voltage traces from
extracellular recordings to be a data to be analysed with spike sorting methods
:type pk: int
:param pk: pk of `Asset` entity
:returns: bool -- True if file validates, False else. Processing
log, including errors, will be written to the `Datafile` entity.
"""
# init and checks
valid = False
logger = Logger.get_logger(StringIO())
try:
obj = Asset.objects.get(id=pk)
assert obj.kind == "rd_file"
df = obj.content_object
except:
logger.log("ERROR")
return valid
try:
logger.log("looking at raw data file with pk: %s" % pk)
rd, sr = read_hdf5_arc(obj.data.path)
logger.log("found rd_file: %s" % obj.name)
len_rd_sec = rd.shape[0] / sr
logger.log("found data in %d channels, for %d sec" %
(rd.shape[1], len_rd_sec))
# TODO: more checks?
logger.log("rd_file passed all checks")
valid = True
except Exception, ex:
logger.log("ERROR: rawdata file check: %s" % str(ex))
def test_delimiter_line(self):
L = Logger(StringIO())
L.log_delimiter_line()
self.assertEqual(L.get_content().strip(), '*' * 20)
def test_logger_strio(self):
canvas = StringIO()
logger = Logger.get_logger(canvas)
logger.log(*self.str_test)
self.assertEqual(logger.get_content().split(), self.str_test)
def test_is_file_like(self):
self.assertTrue(Logger.is_file_like(StringIO()))
self.assertFalse(Logger.is_file_like(self))
def test_delimiter_line(self):
L = Logger(StringIO())
L.log_delimiter_line()
self.assertEqual(L.get_content().strip(), '*' * 20)
def test_logger_strio(self):
canvas = StringIO()
logger = Logger.get_logger(canvas)
logger.log(*self.str_test)
self.assertEqual(logger.get_content().split(), self.str_test)
def test_is_file_like(self):
self.assertTrue(Logger.is_file_like(StringIO()))
self.assertFalse(Logger.is_file_like(self))
