def memoize(fn, paths):
cwd = os.getcwd()
def change_to_target_dir():
for dir in paths[:-1]:
try:
os.mkdir(dir)
except OSError as e:
pass
os.chdir(dir)
change_to_target_dir()
filename = paths[-1]
if os.path.exists(filename):
data = hdf5.read(filename)
os.chdir(cwd)
return data
os.chdir(cwd)
data = fn()
change_to_target_dir()
tmp = '%s.pid.%d.tmp' % (filename, os.getpid())
hdf5.write(tmp, data)
os.rename(tmp, filename)
os.chdir(cwd)
return jsdict(data)
def process_data_sub_job(settings, filename_in_fmt, filename_out_fmt, id,
num_jobs):
pid = os.getpid()
reader = mat_reader(target, settings.data_dir)
num_processed = 0
for i in range(id + 1, sys.maxsize, num_jobs):
out_index = i - 1
filename_in = filename_in_fmt % i
filename_out = filename_out_fmt % out_index if filename_out_fmt is not None else None
filename_out_temp = '%s.pid.%d.tmp' % (
filename_out, pid) if filename_out is not None else None
if filename_out is not None and os.path.exists(filename_out):
num_processed += 1
continue
if not reader.exists(filename_in):
if i == id + 1:
print('Could not find file', reader.filename(filename_in))
return 0
break
print('Runner %d processing %s' % (id, reader.filename(filename_in)))
segment = reader.read(filename_in)
data = process_data(segment)
hdf5.write(filename_out_temp, data)
os.rename(filename_out_temp, filename_out)
num_processed += 1
return num_processed
def process_data_sub_job(settings, filename_in_fmt, filename_out_fmt, id, num_jobs):
pid = os.getpid()
reader = mat_reader(target, settings.data_dir)
num_processed = 0
for i in xrange(id + 1, sys.maxint, num_jobs):
out_index = i - 1
filename_in = filename_in_fmt % i
filename_out = filename_out_fmt % out_index if filename_out_fmt is not None else None
filename_out_temp = '%s.pid.%d.tmp' % (filename_out, pid) if filename_out is not None else None
if filename_out is not None and os.path.exists(filename_out):
num_processed += 1
continue
if not reader.exists(filename_in):
if i == id + 1:
print 'Could not find file', reader.filename(filename_in)
return 0
break
print 'Runner %d processing %s' % (id, reader.filename(filename_in))
segment = reader.read(filename_in)
data = process_data(segment)
hdf5.write(filename_out_temp, data)
os.rename(filename_out_temp, filename_out)
num_processed += 1
return num_processed
except OSError, e:
pass
os.chdir(dir)
change_to_target_dir()
filename = paths[-1]
if os.path.exists(filename):
data = hdf5.read(filename)
os.chdir(cwd)
return data
os.chdir(cwd)
data = fn()
change_to_target_dir()
tmp = '%s.pid.%d.tmp' % (filename, os.getpid())
hdf5.write(tmp, data)
os.rename(tmp, filename)
os.chdir(cwd)
return jsdict(data)
# Fast process-if-not-yet-processed method for training data
def check_training_data_loaded(settings, target, pipeline, quiet=False):
if not load_pipeline_data(
settings, target, 'preictal', pipeline, check_only=True,
quiet=quiet):
load_pipeline_data(settings,
target,
'preictal',
pipeline,
except OSError, e:
pass
os.chdir(dir)
change_to_target_dir()
filename = paths[-1]
if os.path.exists(filename):
data = hdf5.read(filename)
os.chdir(cwd)
return data
os.chdir(cwd)
data = fn()
change_to_target_dir()
tmp = '%s.pid.%d.tmp' % (filename, os.getpid())
hdf5.write(tmp, data)
os.rename(tmp, filename)
os.chdir(cwd)
return jsdict(data)
# Fast process-if-not-yet-processed method for training data
def check_training_data_loaded(settings, target, pipeline, quiet=False):
if not load_pipeline_data(settings, target, 'preictal', pipeline, check_only=True, quiet=quiet):
load_pipeline_data(settings, target, 'preictal', pipeline, check_only=False, quiet=quiet)
if not load_pipeline_data(settings, target, 'interictal', pipeline, check_only=True, quiet=quiet):
load_pipeline_data(settings, target, 'interictal', pipeline, check_only=False, quiet=quiet)
# Fast process-if-not-yet-processed method for test data
