def scale_range(in_file, value=1.0, in_mask=None, out_file=None):
import numpy as np
import nibabel as nb
import os.path as op
if out_file is None:
fname, fext = op.splitext(op.basename(in_file))
if fext == '.gz':
fname, _ = op.splitext(fname)
out_file = op.abspath('./%s_range.nii.gz' % fname)
im = nb.load(in_file)
idata = im.get_data()
mask = np.ones_like(idata)
if in_mask is not None:
mask = nb.load(in_mask).get_data()
mask[mask > 0.0] = 1
mask[mask < 0.0] = 0
rp0 = -1.0 * value
rp1 = value
ip0 = np.percentile(idata[mask > 0.0], 1.5)
ip1 = np.percentile(idata[mask > 0.0], 98.5)
factor = (rp1 - rp0) / (ip1 - ip0)
idata *= factor
idata -= np.median(idata[mask > 0.0])
nb.Nifti1Image(
idata, im.get_affine(), im.get_header()).to_filename(out_file)
return out_file
def _get_matfile_data():
# compare to Arielle's in the .mat file
mat_file = "sub01_session_1_raw_ROI_timeseries.mat"
mat_file = osp.join(osp.dirname(osp.realpath(__file__)), mat_file)
# to_check.keys() == ['all_rois', 'time_series',
# '__globals__', 'Nvox', '__header__', '__version__']
to_check = sio.loadmat(mat_file)
nvox = to_check['Nvox'][0]
nb_runs = to_check['time_series'].shape[2] # has shape (time, rois, nb_runs)
assert nb_runs == 4
# make a dict for nvox
check_nvox = {}
for idx, roi in enumerate(to_check['all_rois']):
k, _ = osp.splitext(osp.basename(roi[0][0]))
check_nvox[k] = nvox[idx]
# make a dict for signals
arielle_runs = []
for run in range(nb_runs):
check_signals = {}
for idx, roi in enumerate(to_check['all_rois']):
k = osp.splitext(osp.basename(roi[0][0]))[0]
check_signals[k] = to_check['time_series'][:,idx,run]
arielle_runs.append(check_signals)
return check_nvox, arielle_runs
def main_loop(argv=None):
if argv is None:
argv = sys.argv
args = docopt.docopt(get_updated_docstring(), argv=argv[1:],
version=".".join(map(str, __version__)))
if not args["--silent"]:
logcfg.set_loglevel(log, "INFO")
for h in log.handlers:
logcfg.set_loglevel(h, "INFO")
elif args["--verbose"] > 0:
logcfg.make_verbose()
log.debug(pf(args))
ext = args["--extension"]
recursive = args["--recursive"]
files_and_folders = []
files_and_folders.extend(args["<file_or_folder>"])
for faf in files_and_folders:
if osp.isfile(faf):
parse_file(faf, args)
elif osp.isdir(faf):
for entry in os.listdir(faf):
path = osp.join(faf, entry)
valid_file = osp.isfile(path)\
and osp.splitext(path)[-1] == ext\
and osp.basename(osp.splitext(path)[0]) != "cfg"
valid_folder = recursive and osp.isdir(path)
if valid_file or valid_folder:
files_and_folders.append(path)
def filter_fmap(in_file, in_mask=None, out_file=None):
from pyacwereg.filters import wavelets_denoise, laplacian_filter
import numpy as np
import nibabel as nb
import os.path as op
from math import pi
from scipy.ndimage import median_filter
if out_file is None:
fname, fext = op.splitext(op.basename(in_file))
if fext == '.gz':
fname, _ = op.splitext(fname)
out_file = op.abspath('./%s_filtered.nii.gz' % fname)
filtered = wavelets_denoise(laplacian_filter(in_file, in_mask))
im = nb.load(filtered)
result = im.get_data()
result = median_filter(result, 10)
result -= np.median(result)
result *= (pi / np.percentile(result, 99.97))
nb.Nifti1Image(result, im.get_affine(),
im.get_header()).to_filename(out_file)
return out_file
def hasChildren(self, index):
QApplication.setOverrideCursor(Qt.WaitCursor)
if index.isValid():
item = self.itemFromIndex(index)
path = item.path
if item.hasChildren():
children = True
elif item.isClass or item.isFunction:
children = False
elif isdir(path):
paths = [join(path, p) for p in listdir(path)]
paths = [p for p in paths if isPackage(p) or isSource(p)]
paths.sort()
for key, subiter in groupby(paths, lambda x:splitext(x)[0]):
pth = sorted(subiter)[0]
item.appendRow(SysPathItem(pth, split(pth)[1]))
children = bool(paths)
elif splitext(path)[1] in pyexts:
contents = readModule(self.dottedName(item), split(path)[0])
for name, obj in sorted(contents.items()):
item.appendRow(SysPathItem(path, name, obj))
children = bool(contents)
else:
children = False
else:
children = True
QApplication.restoreOverrideCursor()
return children
def generate(self):
# "make" wants Unix paths
self.resources.win_to_unix()
to_be_compiled = []
for r_type in ['s_sources', 'c_sources', 'cpp_sources']:
r = getattr(self.resources, r_type)
if r:
for source in r:
base, ext = splitext(source)
to_be_compiled.append(base + '.o')
libraries = []
for lib in self.resources.libraries:
l, _ = splitext(basename(lib))
libraries.append(l[3:])
ctx = {
'name': self.program_name,
'to_be_compiled': to_be_compiled,
'object_files': self.resources.objects,
'include_paths': self.resources.inc_dirs,
'library_paths': self.resources.lib_dirs,
'linker_script': self.resources.linker_script,
'libraries': libraries,
'symbols': self.toolchain.get_symbols()
}
self.gen_file('codesourcery_%s.tmpl' % self.target.lower(), ctx, 'Makefile')
def run_bootstrap(task_fname, action_fname, bootstrap_fname, burn_in = 40, tree_sizes = None, animate=False, no_cmat=False):
"""
generates a bootstrapping tree
taskfile has the training examples to use
bootstrap_fname will be used as the file to create all of the bootstrapping trees
tree_sizes controls the number of trees we want to build
results for tree size i will be in bootstrap_fname_i.h5
"""
if not tree_sizes:
tree_sizes = DEFAULT_TREE_SIZES[:]
taskf = h5py.File(task_fname, 'r')
assert len(taskf) >= burn_in + max(tree_sizes)
taskf.close()
task_ctr = 0
setup_bootstrap_file(action_fname, bootstrap_fname)
bootstrap_orig = osp.splitext(bootstrap_fname)[0] + '_orig.h5'
shutil.copyfile(bootstrap_fname, bootstrap_orig)
results = []
for i in range(burn_in):
print 'doing burn in {}/{}'.format(i, burn_in)
res = run_example((task_fname, str(task_ctr), bootstrap_orig, bootstrap_fname, animate, no_cmat))
results.append(res)
task_ctr += 1
for i in range(max(tree_sizes)):
print 'doing bootstrapping {}/{}'.format(i, max(tree_sizes))
if i in tree_sizes:
bootstrap_i_fname = osp.splitext(bootstrap_fname)[0] + '_{}.h5'.format(i)
shutil.copyfile(bootstrap_fname, bootstrap_i_fname)
res = run_example((task_fname, str(task_ctr), bootstrap_fname, bootstrap_fname, animate, no_cmat))
results.append(res)
task_ctr += 1
print 'success rate', sum(results)/float(len(results))
return sum(results)/float(len(results))
def enhance(in_file, clip_limit=0.010, in_mask=None, out_file=None):
import numpy as np
import nibabel as nb
import os.path as op
from skimage import exposure, img_as_int
if out_file is None:
fname, fext = op.splitext(op.basename(in_file))
if fext == '.gz':
fname, _ = op.splitext(fname)
out_file = op.abspath('./%s_enh.nii.gz' % fname)
im = nb.load(in_file)
imdata = im.get_data()
imshape = im.get_shape()
if in_mask is not None:
msk = nb.load(in_mask).get_data()
msk[msk > 0] = 1
msk[msk < 1] = 0
imdata = imdata * msk
immin = imdata.min()
imdata = (imdata - immin).astype(np.uint16)
adapted = exposure.equalize_adapthist(imdata.reshape(imshape[0], -1),
clip_limit=clip_limit)
nb.Nifti1Image(adapted.reshape(imshape), im.get_affine(),
im.get_header()).to_filename(out_file)
return out_file
def read_data(self):
print("\nRead Data")
# List input directory containing the .csv files
for filename in listdir(self.input_dir):
fn = path.splitext(filename)[0]
if path.splitext(filename)[1] == ".csv":
file_path = path.join(self.input_dir, filename)
print("File path: " + file_path)
# Pandas.read_csv method returns DataFrame object
try:
if "campus" in fn or 'cfy' == fn[0:3]:
df = read_csv(file_path,
delimiter=",",
header=0,
low_memory=False)
df = df[df['Category'].isin(values)]
elif ('district' in fn or 'dfy' == fn[0:3] or
'state' in fn or 'sfy' == fn[0:3]):
df = read_csv(file_path,
delimiter=",",
header=0,
low_memory=False)
df = df[df['Category'].isin(values)]
if 'state' in fn or 'sfy' == fn[0:3]:
print('\t State file modification')
df.insert(0, 'DISTRICT', "1")
else:
print("\t Skipping file: %s" % file_path)
continue
self.write_data(df, filename)
except:
print("Error while reading %s" % filename)
def siemens2rads(in_file, out_file=None):
"""
Converts input phase difference map to rads
"""
import numpy as np
import nibabel as nb
import os.path as op
import math
if out_file is None:
fname, fext = op.splitext(op.basename(in_file))
if fext == '.gz':
fname, _ = op.splitext(fname)
out_file = op.abspath('./%s_rads.nii.gz' % fname)
in_file = np.atleast_1d(in_file).tolist()
im = nb.load(in_file[0])
data = im.get_data().astype(np.float32)
hdr = im.get_header().copy()
if len(in_file) == 2:
data = nb.load(in_file[1]).get_data().astype(np.float32) - data
elif (data.ndim == 4) and (data.shape[-1] == 2):
data = np.squeeze(data[..., 1] - data[..., 0])
hdr.set_data_shape(data.shape[:3])
imin = data.min()
imax = data.max()
data = (2.0 * math.pi * (data - imin)/(imax-imin)) - math.pi
hdr.set_data_dtype(np.float32)
hdr.set_xyzt_units('mm')
hdr['datatype'] = 16
nb.Nifti1Image(data, im.get_affine(), hdr).to_filename(out_file)
return out_file
def demean_image(in_file, in_mask=None, out_file=None):
"""
Demean image data inside mask
"""
import numpy as np
import nibabel as nb
import os.path as op
import math
if out_file is None:
fname, fext = op.splitext(op.basename(in_file))
if fext == '.gz':
fname, _ = op.splitext(fname)
out_file = op.abspath('./%s_demean.nii.gz' % fname)
im = nb.load(in_file)
data = im.get_data().astype(np.float32)
msk = np.ones_like(data)
if in_mask is not None:
msk = nb.load(in_mask).get_data().astype(np.float32)
msk[msk > 0] = 1.0
msk[msk < 1] = 0.0
mean = np.median(data[msk == 1].reshape(-1))
data[msk == 1] = data[msk == 1] - mean
nb.Nifti1Image(data, im.get_affine(),
im.get_header()).to_filename(out_file)
return out_file
def recompose_dwi(in_dwi, in_bval, in_corrected, out_file=None):
"""
Recompose back the dMRI data accordingly the b-values table after EC
correction
"""
import numpy as np
import nibabel as nb
import os.path as op
if out_file is None:
fname, ext = op.splitext(op.basename(in_dwi))
if ext == ".gz":
fname, ext2 = op.splitext(fname)
ext = ext2 + ext
out_file = op.abspath("%s_eccorrect%s" % (fname, ext))
im = nb.load(in_dwi)
dwidata = im.get_data()
bvals = np.loadtxt(in_bval)
dwis = np.where(bvals != 0)[0].tolist()
if len(dwis) != len(in_corrected):
raise RuntimeError(('Length of DWIs in b-values table and after'
'correction should match'))
for bindex, dwi in zip(dwis, in_corrected):
dwidata[..., bindex] = nb.load(dwi).get_data()
nb.Nifti1Image(dwidata, im.get_affine(),
im.get_header()).to_filename(out_file)
return out_file
def b0_average(in_dwi, in_bval, out_file=None):
"""
A function that averages the *b0* volumes from a DWI dataset.
.. warning:: *b0* should be already registered (head motion artifact should
be corrected).
"""
import numpy as np
import nibabel as nb
import os.path as op
if out_file is None:
fname, ext = op.splitext(op.basename(in_dwi))
if ext == ".gz":
fname, ext2 = op.splitext(fname)
ext = ext2 + ext
out_file = op.abspath("%s_avg_b0%s" % (fname, ext))
imgs = np.array(nb.four_to_three(nb.load(in_dwi)))
bval = np.loadtxt(in_bval)
b0s = [im.get_data().astype(np.float32)
for im in imgs[np.where(bval == 0)]]
b0 = np.average(np.array(b0s), axis=0)
hdr = imgs[0].get_header().copy()
hdr.set_data_shape(b0.shape)
hdr.set_xyzt_units('mm')
hdr.set_data_dtype(np.float32)
nb.Nifti1Image(b0, imgs[0].get_affine(), hdr).to_filename(out_file)
return out_file
def _load_rbo(self):
"""Load APC2015rbo dataset"""
dataset_dir = osp.join(this_dir, 'dataset/APC2015rbo/berlin_samples')
img_glob = osp.join(dataset_dir, '*_bin_[A-L].jpg')
desc = 'rbo'
for img_file in tqdm.tqdm(glob.glob(img_glob), ncols=80, desc=desc):
basename = osp.splitext(osp.basename(img_file))[0]
# apply mask, crop and save
bin_mask_file = re.sub('.jpg$', '.pbm', img_file)
bin_mask = imread(bin_mask_file, mode='L')
where = np.argwhere(bin_mask)
roi = where.min(0), where.max(0) + 1
id_ = osp.join('rbo', basename)
dataset_index = len(self.ids) - 1
self.datasets['rbo'].append(dataset_index)
mask_glob = re.sub('.jpg$', '_*.pbm', img_file)
mask_files = [None] * self.n_class
for mask_file in glob.glob(mask_glob):
mask_basename = osp.splitext(osp.basename(mask_file))[0]
label_name = re.sub(basename + '_', '', mask_basename)
if label_name == 'shelf':
continue
mask_files[self.target_names.index(label_name)] = mask_file
self.ids.append(id_)
self.rois.append(roi)
self.img_files.append(img_file)
self.mask_files.append(mask_files)
def extract_bval(in_dwi, in_bval, b=0, out_file=None):
"""
Writes an image containing only the volumes with b-value specified at
input
"""
import numpy as np
import nibabel as nb
import os.path as op
if out_file is None:
fname, ext = op.splitext(op.basename(in_dwi))
if ext == ".gz":
fname, ext2 = op.splitext(fname)
ext = ext2 + ext
out_file = op.abspath("%s_tsoi%s" % (fname, ext))
im = nb.load(in_dwi)
dwidata = im.get_data()
bvals = np.loadtxt(in_bval)
if b == 'diff':
selection = np.where(bvals != 0)
elif b == 'nodiff':
selection = np.where(bvals == 0)
else:
selection = np.where(bvals == b)
extdata = np.squeeze(dwidata.take(selection, axis=3))
hdr = im.get_header().copy()
hdr.set_data_shape(extdata.shape)
nb.Nifti1Image(extdata, im.get_affine(),
hdr).to_filename(out_file)
return out_file
def _parse_version_file(version_file):
"""Get version info from the given file. It can be any of:
- package.json with "version" field
- VERSION.txt or VERSION file with just the version string
- python file with __version_info__
- .js file with `var VERSION = "1.2.3";`
"""
f = codecs.open(version_file, 'r', 'utf-8')
content = f.read()
f.close()
if basename(version_file) == "package.json":
version_file_type = "package.json"
obj = json.loads(content)
version_info = _version_info_from_version(obj["version"])
elif splitext(version_file)[1] == ".py":
version_file_type = "python"
m = re.search(r'^__version_info__ = (.*?)$', content, re.M)
version_info = eval(m.group(1))
elif splitext(version_file)[1] == ".js":
version_file_type = "javascript"
m = re.search(r'^var VERSION = "(.*?)";$', content, re.M)
version_info = _version_info_from_version(m.group(1))
else:
# Presume a text file with just the version.
version_file_type = "version"
version_info = _version_info_from_version(content.strip())
return version_file_type, version_info
def bg_mask(in_file, in_mask, out_file=None):
"""
Rough mask of background from brain masks
"""
import nibabel as nb
import numpy as np
from scipy.ndimage import binary_dilation
import scipy.ndimage as nd
import os.path as op
if out_file is None:
fname, ext = op.splitext(op.basename(in_file))
if ext == ".gz":
fname, ext2 = op.splitext(fname)
ext = ext2 + ext
out_file = op.abspath("%s_bgmask%s" % (fname, ext))
im = nb.load(in_file)
hdr = im.get_header().copy()
hdr.set_data_dtype(np.uint8)
hdr.set_xyzt_units('mm')
imdata = im.get_data()
msk = nb.load(in_mask).get_data()
msk = 1 - binary_dilation(msk,
structure=np.ones((20, 20, 20)))
nb.Nifti1Image(msk.astype(np.uint8),
im.get_affine(), hdr).to_filename(out_file)
return out_file
def processes_file(self, filename, outfile=None, errorfile=None, directory=''):
"""Take a filename of a file containing plantuml text and processes
it into a .png image.
:param str filename: Text file containing plantuml markup
:param str outfile: Filename to write the output image to. If not
supplied, then it will be the input filename with the
file extension replaced with '.png'.
:param str errorfile: Filename to write server html error page
to. If this is not supplined, then it will be the
input ``filename`` with the extension replaced with
'_error.html'.
:returns: ``True`` if the image write succedded, ``False`` if there was
an error written to ``errorfile``.
"""
if outfile is None:
outfile = path.splitext(filename)[0] + '.png'
if errorfile is None:
errorfile = path.splitext(filename)[0] + '_error.html'
if directory and not path.exists(directory):
makedirs(directory)
data = open(filename, 'U').read()
try:
content = self.processes(data)
except PlantUMLHTTPError, e:
err = open(path.join(directory, errorfile), 'w')
err.write(e.content)
err.close()
return False
def setUp(self):
# create a list of files to cleanup
self._paths_to_clean_up = []
self._dirs_to_clean_up = []
# load query seqs
self.seqs = Alignment(MinimalFastaParser(QUERY_SEQS.split()))
# generate temp filename
tmp_dir='/tmp'
self.outfile = get_tmp_filename(tmp_dir)
# create and write out reference sequence file
self.outfasta=splitext(self.outfile)[0]+'.fasta'
fastaout=open(self.outfasta,'w')
fastaout.write(REF_SEQS)
fastaout.close()
self._paths_to_clean_up.append(self.outfasta)
# create and write out starting tree file
self.outtree=splitext(self.outfile)[0]+'.tree'
treeout=open(self.outtree,'w')
treeout.write(REF_TREE)
treeout.close()
self._paths_to_clean_up.append(self.outtree)
def read_mesh(fname):
"""Read mesh data from file.
Parameters
----------
fname : str
File name to read. Format will be inferred from the filename.
Currently only '.obj' and '.obj.gz' are supported.
Returns
-------
vertices : array
Vertices.
faces : array | None
Triangle face definitions.
normals : array
Normals for the mesh.
texcoords : array | None
Texture coordinates.
"""
# Check format
fmt = op.splitext(fname)[1].lower()
if fmt == '.gz':
fmt = op.splitext(op.splitext(fname)[0])[1].lower()
if fmt in ('.obj'):
return WavefrontReader.read(fname)
elif not format:
raise ValueError('read_mesh needs could not determine format.')
else:
raise ValueError('read_mesh does not understand format %s.' % fmt)
def run(self):
dest = self.parser.getArgs()[-1]
if path.exists(dest) and not self.opts.force:
self.error("CSV-file", dest, "exists already. Use --force to overwrite")
sources = self.parser.getArgs()[0:-1]
data = SpreadsheetData(csvName=sources[0], title=path.splitext(path.basename(sources[0]))[0])
if self.opts.time == None:
self.opts.time = data.names()[0]
for s in sources[1:]:
addition = path.splitext(path.basename(s))[0]
sData = SpreadsheetData(csvName=s)
if self.opts.addTimes:
data.addTimes(
time=self.opts.time, times=sData.data[self.opts.time], interpolate=self.opts.interpolateNewTime
)
for n in sData.names():
if n != self.opts.time:
d = data.resample(
sData,
n,
time=self.opts.time,
extendData=self.opts.extendData,
noInterpolation=not self.opts.newDataInterpolate,
)
data.append(addition + " " + n, d, allowDuplicates=True)
data.writeCSV(dest, delimiter=self.opts.delimiter)
def time_avg(in_file, index=[0], out_file=None):
"""
Average the input time-series, selecting the indices given in index
.. warning:: time steps should be already registered (corrected for
head motion artifacts).
"""
import numpy as np
import nibabel as nb
import os.path as op
if out_file is None:
fname, ext = op.splitext(op.basename(in_file))
if ext == ".gz":
fname, ext2 = op.splitext(fname)
ext = ext2 + ext
out_file = op.abspath("%s_baseline%s" % (fname, ext))
index = np.atleast_1d(index).tolist()
imgs = np.array(nb.four_to_three(nb.load(in_file)))[index]
if len(index) == 1:
data = imgs[0].get_data().astype(np.float32)
else:
data = np.average(np.array([im.get_data().astype(np.float32)
for im in imgs]), axis=0)
hdr = imgs[0].get_header().copy()
hdr.set_data_shape(data.shape)
hdr.set_xyzt_units('mm')
hdr.set_data_dtype(np.float32)
nb.Nifti1Image(data, imgs[0].get_affine(), hdr).to_filename(out_file)
return out_file
def get_real_fn(self, path):
filename = basename(path)
fn, ext = splitext(filename)
fn2, ext2 = splitext(fn)
if ext2:
return fn
return filename
def main():
inPath = path.abspath(argv[1])
inDir = path.dirname(inPath)
inName = path.splitext(path.basename(inPath))[0]
source = '%s/%s' % (inDir, inName)
tar = tarfile.open(inPath, 'r')
tar.extractall(path=inDir)
tar.close()
if path.exists('%s/normalize.lock' % source):
print 'error[44] already normalized!'
rmtree(source)
return -1
if path.exists('%s/dummy.lock' % source):
print 'error[65] already dummy!'
rmtree(source)
return -1
outPath = path.abspath(argv[2])
outDir = path.dirname(outPath)
outName = path.splitext(path.basename(outPath))[0]
output = '%s/%s' % (outDir, outName)
try:
mkdir(output)
except WindowsError, err:
print err
def pack_dir(in_dir, out_file, file_list, labels_dict=None, seed=None):
if seed is not None:
random.seed(seed)
files = get_files_paths(in_dir, ext='.png')
random.shuffle(files)
with open(file_list, 'w') as fo:
for f in files:
fo.write(splitext(basename(f))[0]+'\n')
with h5py.File(out_file, "w") as f5:
X_dataset = f5.create_dataset("X",
(len(files), constants.nChannels*np.square(constants.imSize)),
dtype=np.uint8)
if labels_dict is not None:
y_dataset = f5.create_dataset("y", (len(files),), dtype=int)
for i,f in enumerate(files):
im=Image.open(f)
# N x D (N, n_channels*HSize*WSize)
d = np.asarray(im.getdata()).T.reshape(1,-1)
X_dataset[i] = np.uint8(d)
if labels_dict is not None:
y_dataset[i] = labels_dict[splitext(splitext(basename(f))[0])[0]]
# print y_dataset[i]
if (i+1) % 50 == 0:
print('%d/%d' % (i+1, len(files)))
def run(self):
'''
Finds all the tests files in tests/, and run doctest on them.
'''
pymods = []
pyxmods = []
for root, dirs, files in os.walk('arboris'):
for file in files:
package = '.'.join(root.split(sep))
if file.endswith('.py') and file != '__init__.py':
pymods.append('.'.join([package, splitext(file)[0]]))
elif file.endswith('.so'):
pyxmods.append('.'.join([package, splitext(file)[0]]))
for mod in pymods:
exec('import {0} as module'.format(mod))
doctest.testmod(module)
for mod in pyxmods:
exec('import {0} as mod'.format(mod))
fix_module_doctests(mod)
doctest.testmod(mod)
for rst in glob(pjoin('tests', '*.rst')):
doctest.testfile(rst)
def load_images(img_mat_fname, corpus_df, normalize=True, force_rebuild=False):
_memo_fname = path.join(CACHE_DIR, 'IMAGES_' + path.splitext(path.basename(img_mat_fname))[0]) + '.pkl'
if not path.exists(_memo_fname) or force_rebuild:
used_pics = set(corpus_df['picture'])
m = loadmat(img_mat_fname)
fnames = []
X = np.empty((m['Img'].shape[0] * 20, 4096), dtype=np.float32)
idx_x = 0
for idx_m in xrange(m['Img'].shape[0]):
c = m['Img'][idx_m][0]
fname = path.splitext(path.basename(c['fname'][0][0][0]))[0]
if not fname in used_pics:
continue
codes = c['codes'][0][0][:, :-1]
for img_segment in codes:
fnames.append(fname)
X[idx_x] = img_segment
idx_x += 1
X = X[:idx_x, :]
assert(len(fnames) == X.shape[0])
with open(_memo_fname, 'wb') as fid:
pickle.dump((fnames, X), fid, -1)
else:
with open(_memo_fname, 'rb') as fid:
fnames, X = pickle.load(fid)
if normalize:
X = StandardScaler().fit_transform(X)
return fnames, X
def createDataTxt(imagePath, annotationPath, imagesInDir, split=False):
JPG = '.jpg'
TRAINING = 'training/'
VALIDATION = 'validation/'
if split:
annotatedImages = os.listdir(annotationPath)
# np.random.shuffle(annotatedImages)
splitSize = ceil(len(annotatedImages) * 0.85)
annotatedImagesTrain = annotatedImages[:splitSize]
annotatedImagesValidation = annotatedImages[splitSize:]
else:
annotatedImagesTrain = os.listdir(join(annotationPath, TRAINING))
annotatedImagesValidation = os.listdir(join(annotationPath, VALIDATION))
with open(imagesInDir + 'train.txt', 'w') as file:
for ann in tqdm(annotatedImagesTrain, desc='Writing train.txt for input dataset'):
if isfile(join(imagePath, TRAINING, splitext(ann)[0]) + JPG):
file.write(' '.join(
[join(imagePath, TRAINING, splitext(ann)[0]) + JPG,
join(annotationPath, TRAINING, ann)]) + '\n')
with open(imagesInDir + 'val.txt', 'w') as file:
for annv in tqdm(annotatedImagesValidation, desc='Writing valid.txt for input dataset'):
if isfile(join(imagePath, VALIDATION, splitext(annv)[0]) + JPG):
file.write(' '.join(
[join(imagePath, VALIDATION, splitext(annv)[0]) + JPG,
join(annotationPath, VALIDATION, annv)]) + '\n')
return
def mem_stats(self, map):
"""! Creates parser object
@param map Path to linker map file to parse and decode
@return Memory summary structure with memory usage statistics
None if map file can't be opened and processed
"""
toolchain = self.__class__.__name__
# Create memap object
memap = MemapParser()
# Parse and decode a map file
if memap.parse(abspath(map), toolchain) is False:
self.info("Unknown toolchain for memory statistics %s" % toolchain)
return None
# Write output to stdout in text (pretty table) format
memap.generate_output('table')
# Write output to file in JSON format
map_out = splitext(map)[0] + "_map.json"
memap.generate_output('json', map_out)
# Write output to file in CSV format for the CI
map_csv = splitext(map)[0] + "_map.csv"
memap.generate_output('csv-ci', map_csv)
# Here we return memory statistics structure (constructed after
# call to generate_output) which contains raw data in bytes
# about sections + summary
return memap.mem_summary
def b0_average(in_dwi, in_bval, max_b=10.0, out_file=None):
"""
A function that averages the *b0* volumes from a DWI dataset.
As current dMRI data are being acquired with all b-values > 0.0,
the *lowb* volumes are selected by specifying the parameter max_b.
.. warning:: *b0* should be already registered (head motion artifact should
be corrected).
"""
import numpy as np
import nibabel as nb
import os.path as op
if out_file is None:
fname, ext = op.splitext(op.basename(in_dwi))
if ext == ".gz":
fname, ext2 = op.splitext(fname)
ext = ext2 + ext
out_file = op.abspath("%s_avg_b0%s" % (fname, ext))
imgs = np.array(nb.four_to_three(nb.load(in_dwi)))
bval = np.loadtxt(in_bval)
index = np.argwhere(bval <= max_b).flatten().tolist()
b0s = [im.get_data().astype(np.float32)
for im in imgs[index]]
b0 = np.average(np.array(b0s), axis=0)
hdr = imgs[0].get_header().copy()
hdr.set_data_shape(b0.shape)
hdr.set_xyzt_units('mm')
hdr.set_data_dtype(np.float32)
nb.Nifti1Image(b0, imgs[0].get_affine(), hdr).to_filename(out_file)
return out_file
def main():
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('kicad_pcb', help='the `.kicad_pcb` file to DIFF')
parser.add_argument(
'-c',
'--commit',
default='HEAD',
help='git commit-id to compare current version against. Default: HEAD')
parser.add_argument(
'-l',
'--layers',
default=0,
type=int,
help='Number of inner layers (InX.Cu) to plot. Default: 0')
parser.add_argument('-r',
'--resolution',
default=400,
type=int,
help='Plotting resolution in [dpi]. Default: 400')
args = parser.parse_args()
layers = ['F.Cu', 'B.Cu', 'F.SilkS', 'B.SilkS']
layers += ['In{}.Cu'.format(i + 1) for i in range(args.layers)]
print('layers: ' + ' '.join(layers))
# Check for local (un-commited) changes
do_stash = call(['git', 'diff-index', '--quiet', 'HEAD', '--']) > 0
if not do_stash and args.commit == 'HEAD':
print('No local changes, nothing to compare. Try -c <commit-id>')
return -1
# output directory name is derived from `git describe`
try:
git1_name = desc()
except Exception:
# this will happen if user isn't in a git repo
print("No git description, can't continue")
exit(1)
# Do a .pdf plot of the current version
dir1 = 'plot_' + git1_name
print('> ' + dir1)
bounds1 = plot_layers(args.kicad_pcb, dir1, layers)
if bounds1 is None:
exit(1)
# Stash local changes if needed
if do_stash:
co(['git', 'stash'])
# checkout specified git version (default: HEAD) ...
if args.commit != 'HEAD':
co(['git', 'checkout', args.commit])
# ... and do a .pdf plot of it
dir2 = 'plot_' + desc()
print('> ' + dir2)
bounds2 = plot_layers(args.kicad_pcb, dir2, layers)
if bounds2 is None:
exit(1)
# Switch back to current version
if args.commit != 'HEAD':
co(['git', 'checkout', '-'])
# Restore local changes
if do_stash:
co(['git', 'stash', 'pop'])
# Generate plots into `diffs` directory
try:
mkdir('diffs')
except OSError:
print('diffs directory already exists')
# Create a .png diff for each layer
for ll in layers:
pdf_name = splitext(args.kicad_pcb)[0]
pdf_name += '-' + ll.replace('.', '_') + '.pdf'
out_file = 'diffs/' + ll + '.png'
print('> ' + out_file)
i1 = load_pdf(join(dir1, pdf_name), r=args.resolution, **bounds1)
i2 = load_pdf(join(dir2, pdf_name), r=args.resolution, **bounds1)
i_out = img_diff(i1, i2)
i_out.save(out_file)
print('Removing temporary directories')
rmtree(dir1)
rmtree(dir2)
def csort(
pairs_path,
chromosomes_path,
index,
chrom1,
chrom2,
pos1,
pos2,
flip_only,
nproc,
zero_based,
sep,
comment_char,
sort_options,
out,
**kwargs
):
"""
Sort and index a contact list.
Order the mates of each pair record so that all contacts are upper
triangular with respect to the chromosome ordering given by the chromosomes
file, sort contacts by genomic location, and index the resulting file.
PAIRS_PATH : Contacts (i.e. read pairs) text file, optionally compressed.
CHROMOSOMES_PATH : File listing desired chromosomes in the desired order.
May be tab-delimited, e.g. a UCSC-style chromsizes file. Contacts mapping to
other chromosomes will be discarded.
**Notes**
\b
- csort can also be used to sort and index a text representation of
a contact *matrix* in bedGraph-like format. In this case, substitute
`pos1` and `pos2` with `start1` and `start2`, respectively.
- Requires Unix tools: sort, bgzip + tabix or pairix.
If indexing with Tabix, the output file will have the following properties:
\b
- Upper triangular: the read pairs on each row are assigned to side 1 or 2
in such a way that (chrom1, pos1) is always "less than" (chrom2, pos2)
- Rows are lexicographically sorted by chrom1, pos1, chrom2, pos2;
i.e. "positionally sorted"
- Compressed with bgzip [*]
- Indexed using Tabix [*] on chrom1 and pos1.
If indexing with Pairix, the output file will have the following properties:
\b
- Upper triangular: the read pairs on each row are assigned to side 1 or 2
in such a way that (chrom1, pos1) is always "less than" (chrom2, pos2)
- Rows are lexicographically sorted by chrom1, chrom2, pos1, pos2; i.e.
"block sorted"
- Compressed with bgzip [*]
- Indexed using Pairix [+] on chrom1, chrom2 and pos1.
\b
[*] Tabix manpage: <http://www.htslib.org/doc/tabix.html>.
[+] Pairix on Github: <https://github.com/4dn-dcic/pairix>
"""
if os.name == "nt":
raise click.Abort(
'"cooler csort" does not work on Windows. To ingest unsorted pairs '
'data, see the "cooler cload pairs" command.'
)
from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE, SIG_DFL)
# Check for required Unix tools
for tool in ["sort", "bgzip"] + [index]:
if not cmd_exists(tool):
print("Command {} not found".format(tool), file=sys.stderr)
sys.exit(1)
# If output path is not given, produce output path by stripping any .txt,
# .gz or .txt.gz extension from the input path and appending .sorted[.txt].gz
infile = pairs_path
if out is None:
if infile == "-" and not flip_only:
logger.error("Output name required when input is stdin")
raise click.Abort
prefix = infile
ext = ".gz"
if prefix.endswith(".gz"):
prefix = op.splitext(prefix)[0]
if prefix.endswith(".txt"):
prefix = op.splitext(prefix)[0]
ext = ".txt.gz"
if index == "pairix":
sort_style = ".blksrt"
else:
sort_style = ".possrt"
outfile = prefix + sort_style + ext
else:
outfile = out
# Parse extra sort options and determine if sort supports --parallel option
if sort_options is not None:
sort_options = shlex.split(sort_options)
elif _has_parallel_sort():
sort_options = ["--parallel={}".format(nproc), "--buffer-size=50%"]
else:
sort_options = []
# 1-based column numbers
fields = {"C1": chrom1, "P1": pos1, "C2": chrom2, "P2": pos2}
# build commands
read_cmd = make_read_command(infile)
flip_cmd = make_flip_command(chromosomes_path, sep, comment_char, fields)
if flip_only:
# run pipeline
logger.info("Reordering pair mates...")
pipeline = []
logger.debug(" ".join(read_cmd))
pipeline.append(
subprocess.Popen(
read_cmd,
stdin=sys.stdin if infile == "-" else None,
stdout=subprocess.PIPE,
)
)
logger.debug(" ".join(flip_cmd))
pipeline.append(
subprocess.Popen(flip_cmd, stdin=pipeline[-1].stdout, stdout=sys.stdout)
)
for p in pipeline[::-1]:
p.communicate()
if p.returncode != 0:
sys.exit(1)
else:
sort_cmd = make_sort_command(index, fields, sort_options)
write_cmd = ["bgzip", "-c"]
index_cmd = make_index_command(index, fields, zero_based, outfile)
# run pipeline
logger.info("Input: '{}'".format(infile))
logger.info("Output: '{}'".format(outfile))
assert infile != outfile
with open(outfile, "wb") as fout:
pipeline = []
logger.debug(" ".join(read_cmd))
pipeline.append(
subprocess.Popen(
read_cmd,
stdin=sys.stdin if infile == "-" else None,
stdout=subprocess.PIPE,
)
)
logger.info("Reordering pair mates and sorting pair records...")
logger.debug(" ".join(flip_cmd))
pipeline.append(
subprocess.Popen(
flip_cmd, stdin=pipeline[-1].stdout, stdout=subprocess.PIPE
)
)
if index == "pairix":
logger.info("Sort order: block (chrom1, chrom2, pos1, pos2)")
else:
logger.info("Sort order: positional (chrom1, pos1, chrom2, pos2)")
logger.info(" ".join(sort_cmd))
pipeline.append(
subprocess.Popen(
sort_cmd, stdin=pipeline[-1].stdout, stdout=subprocess.PIPE
)
)
logger.debug(" ".join(write_cmd))
pipeline.append(
subprocess.Popen(write_cmd, stdin=pipeline[-1].stdout, stdout=fout)
)
for p in pipeline[::-1]:
p.communicate()
if p.returncode != 0:
logger.error(" ".join(p.args))
sys.exit(1)
# Create index file
logger.info("Indexing...")
logger.info("Indexer: {}".format(index))
logger.info(" ".join(index_cmd))
p = subprocess.Popen(index_cmd)
p.communicate()
if p.returncode != 0:
sys.exit(1)
return np.array([
x * y * (1 - y**2) * (1 - z**2) + 2 * x * y * (1 - z**2),
y**2 * (1 - x**2) * (1 - z**2) + (1 - y**2) * (2 - x**2 - z**2),
y * z * (1 - x**2) * (1 - y**2) + 2 * y * z * (1 - x**2),
])
@LinearForm
def fv(v, w):
from skfem.helpers import dot
return dot(f(*w.x), v)
A = asm(dudv, basis)
f = asm(fv, basis)
D = basis.find_dofs()
x = solve(*condense(A, f, D=D))
y_basis = InteriorBasis(m, ElementVectorH1(ElementTetP1()))
y = project(x, basis, y_basis)
if __name__ == '__main__':
from os.path import splitext
from sys import argv
name = splitext(argv[0])[0]
m.save('{}_solution.vtk'.format(name), {'field': y[y_basis.nodal_dofs].T})
# Always prefer setuptools over distutils
from setuptools import setup, find_packages
from glob import glob
from os.path import basename
from os.path import splitext
setup(
name='babyzwip',
version='0.1',
description='Z-Wave In Python',
url='https://github.com/magicus/babyzwip',
author='Magnus Ihse Bursie',
author_email='*****@*****.**',
license='GPL3',
packages=find_packages(where="src"),
package_dir={"": "src"},
py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')],
install_requires=['pyserial'],
setup_requires=['pytest-runner'],
tests_require=['pytest'],
)
def __init__(self,
input_dim=(3, 32, 32),
num_filters=None,
filter_size=5,
hidden_dims=None,
num_classes=10,
weight_scale=1e-3,
reg=0.1,
dtype=np.float32,
use_batchnorm=False,
dropout=0,
seed=None,
loadData=None,
predict_fn=None,
augment_fn=None):
"""
predic_fn, augment_fn: for data augmentation
"""
self.params = {}
self.reg = reg
self.dtype = dtype
self.num_filters = len(num_filters)
self.filter_size = filter_size
self.bn_batchnorm = use_batchnorm
self.use_dropout = (dropout > 0)
############# the total number of layers including conv layer and affine layer#######
self.num_layers = self.num_filters + len(hidden_dims) + 1
#####################################################################################
# print "how many layers ? ",self.num_layers
self.bn_params = []
self.dropout_param = {}
self.predict_fn = predict_fn
self.augment_fn = augment_fn
if augment_fn is not None:
input_dim = (3, 28, 28)
self.input_dim = input_dim
if loadData is not None:
print "Load Data is ", loadData
for f in glob.glob("%s/convNet_params_*.npy" % loadData):
name_lst = op.splitext(op.basename(f))[0].split("_")
if len(name_lst) == 3:
param = name_lst[2]
if param == "dropout":
self.dropout_param = self.load_param(f)
elif param == "bn":
self.bn_params = self.load_param(f)
else:
self.params[param] = self.load_param(
f) # W_i,b_i,beta_i, gamma_i
print self.params[param].shape,
print "load parameter %s successfully" % param
return
C, H, W = input_dim
assert filter_size % 2 == 1, 'Filter size must be odd: got %d' % filter_size
all_filters = np.array([C])
all_filters = np.concatenate((all_filters, np.array(num_filters)),
axis=0)
for i in range(self.num_filters):
t = i + 1
self.params['W%d' % t] = weight_scale * np.random.randn(
all_filters[t], all_filters[t - 1], filter_size, filter_size)
self.params['b%d' % t] = np.zeros(all_filters[t])
if self.bn_batchnorm is True:
self.params['gamma%d' % t] = np.random.randn(all_filters[t])
self.params['beta%d' % t] = np.random.randn(all_filters[t])
all_hidden_layers = np.array(
[H * W * all_filters[-1] / np.power(4, self.num_filters)])
a = np.array(hidden_dims)
all_hidden_layers = np.concatenate((all_hidden_layers, a), axis=0)
b = np.array([num_classes])
all_hidden_layers = np.concatenate((all_hidden_layers, b))
length = len(all_hidden_layers) - 1
for i in range(length):
t = i + self.num_filters + 1
self.params['W%d' % t] = weight_scale * np.random.randn(
all_hidden_layers[i], all_hidden_layers[i + 1])
self.params['b%d' % t] = np.zeros(all_hidden_layers[i + 1])
if self.bn_batchnorm is True and i < length - 1:
self.params['gamma%d' % t] = np.random.randn(
all_hidden_layers[i + 1])
self.params['beta%d' % t] = np.random.randn(
all_hidden_layers[i + 1])
if self.bn_batchnorm:
self.bn_params = [{
'mode': 'train'
} for i in xrange(self.num_layers)]
if self.use_dropout is True:
self.dropout_param = {'mode': 'train', 'p': dropout}
if seed is not None:
self.dropout_param['seed'] = seed
for k, v in self.params.iteritems():
self.params[k] = v.astype(dtype)
def __run(name, **kwargs):
"""
Run an experiment.
:param name: experiment name
:param path: expanded path of the experiment (dynamically filled in through 'command' decorator with 'expand')
:param kwargs: simulation keyword arguments (see the documentation for more information)
"""
set_logging(kwargs.get('loglevel'))
path = kwargs['path']
check_structure(path, remove=True)
with settings(hide(*HIDDEN_ALL), warn_only=True):
for sim in ["without", "with"]:
sim_path = join(path, "{}-malicious".format(sim))
data, results = join(sim_path, 'data'), join(sim_path, 'results')
# the Makefile is at experiment's root ('path')
logger.debug(
" > Running simulation {} the malicious mote...".format(sim))
task = kwargs.get('task', "run")
with lcd(sim_path):
output = local("make run TASK={}".format(task), capture=True)
remove_files(sim_path, '.{}'.format(task))
error, interrupt, error_buffer = False, False, []
for line in output.split('\n'):
if line.strip().startswith("FATAL") or line.strip().startswith(
"ERROR"):
error, interrupt = True, True
elif line.strip().startswith("INFO"):
error = False
if len(error_buffer) > 0:
logger.error('Cooja error:\n' +
'\n'.join(error_buffer))
error_buffer = []
if error:
error_buffer.append(line)
if interrupt:
logger.warn(
"Cooja failed to execute ; 'run' interrupted (no parsing done)"
)
raise Exception("Cooja failed to execute")
# once the execution is over, gather the screenshots into a single GIF and keep the first and
# the last screenshots ; move these to the results folder
logger.debug(" > Gathering screenshots in an animated GIF...")
with lcd(data):
local(
'convert -delay 10 -loop 0 network*.png wsn-{}-malicious.gif'
.format(sim),
capture=True)
network_images = {
int(fn.split('.')[0].split('_')[-1]): fn
for fn in listdir(data) if fn.startswith('network_')
}
move_files(data, results, 'wsn-{}-malicious.gif'.format(sim))
if len(network_images) > 0:
net_start_old = network_images[min(network_images.keys())]
net_start, ext = splitext(net_start_old)
net_start_new = 'wsn-{}-malicious_start{}'.format(sim, ext)
net_end_old = network_images[max(network_images.keys())]
net_end, ext = splitext(net_end_old)
net_end_new = 'wsn-{}-malicious_end{}'.format(sim, ext)
move_files(data, results, (net_start_old, net_start_new),
(net_end_old, net_end_new))
remove_files(data, *network_images.values())
# then start the parsing functions to derive more results
logger.debug(" > Parsing simulation results...")
parsing_chain(sim_path)
move_files(sim_path, results, 'COOJA.log')
# finally, generate the PDF report
generate_report(path, REPORT_THEME)
return "Both Cooja executions succeeded"
def is_python_source(filename):
"""
rtype: bool
return: True if the filename is a python source file
"""
return splitext(filename)[1][1:] in PY_SOURCE_EXTS
def get_logfile(out_filename):
"Get logfile name"
return "%s.log" % splitext(out_filename)[0]
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument('input_dir', help='input annotated directory')
parser.add_argument('output_dir', help='output dataset directory')
parser.add_argument('--labels', help='labels file', required=True)
args = parser.parse_args()
if osp.exists(args.output_dir):
print('Output directory already exists:', args.output_dir)
sys.exit(1)
os.makedirs(args.output_dir)
os.makedirs(osp.join(args.output_dir, 'JPEGImages'))
print('Creating dataset:', args.output_dir)
now = datetime.datetime.now()
data = dict(
info=dict(
description=None,
url=None,
version=None,
year=now.year,
contributor=None,
date_created=now.strftime('%Y-%m-%d %H:%M:%S.%f'),
),
licenses=[dict(
url=None,
id=0,
name=None,
)],
images=[
# license, url, file_name, height, width, date_captured, id
],
type='instances',
annotations=[
# segmentation, area, iscrowd, image_id, bbox, category_id, id
],
categories=[
# supercategory, id, name
],
)
class_name_to_id = {}
for i, line in enumerate(open(args.labels).readlines()):
class_id = i - 1 # starts with -1
class_name = line.strip()
if class_id == -1:
assert class_name == '__ignore__'
continue
class_name_to_id[class_name] = class_id
data['categories'].append(dict(
supercategory=None,
id=class_id,
name=class_name,
))
out_ann_file = osp.join(args.output_dir, 'annotations.json')
label_files = glob.glob(osp.join(args.input_dir, '*.json'))
for image_id, label_file in enumerate(label_files):
print('Generating dataset from:', label_file)
with open(label_file) as f:
label_data = json.load(f)
base = osp.splitext(osp.basename(label_file))[0]
out_img_file = osp.join(
args.output_dir, 'JPEGImages', base + '.jpg'
)
img_file = osp.join(
osp.dirname(label_file), label_data['imagePath']
)
img = np.asarray(PIL.Image.open(img_file).convert('RGB'))
PIL.Image.fromarray(img).save(out_img_file)
data['images'].append(dict(
license=0,
url=None,
file_name=osp.relpath(out_img_file, osp.dirname(out_ann_file)),
height=img.shape[0],
width=img.shape[1],
date_captured=None,
id=image_id,
))
masks = {} # for area
segmentations = collections.defaultdict(list) # for segmentation
for shape in label_data['shapes']:
points = shape['points']
label = shape['label']
group_id = shape.get('group_id')
shape_type = shape.get('shape_type')
mask = labelme.utils.shape_to_mask(
img.shape[:2], points, shape_type
)
if group_id is None:
group_id = uuid.uuid1()
instance = (label, group_id)
if instance in masks:
masks[instance] = masks[instance] | mask
else:
masks[instance] = mask
points = np.asarray(points).flatten().tolist()
segmentations[instance].append(points)
segmentations = dict(segmentations)
for instance, mask in masks.items():
cls_name, group_id = instance
if cls_name not in class_name_to_id:
continue
cls_id = class_name_to_id[cls_name]
mask = np.asfortranarray(mask.astype(np.uint8))
mask = pycocotools.mask.encode(mask)
area = float(pycocotools.mask.area(mask))
bbox = pycocotools.mask.toBbox(mask).flatten().tolist()
data['annotations'].append(dict(
id=len(data['annotations']),
image_id=image_id,
category_id=cls_id,
segmentation=segmentations[instance],
area=area,
bbox=bbox,
iscrowd=0,
))
with open(out_ann_file, 'w') as f:
json.dump(data, f)
def parts(self):
return self.parser.sections()
# Read and load all the component libraries
listdir(lenslib)
dirs=[]
#Get the directories. Each directory is a library.
for i in listdir(lenslib):
di=join(lenslib,i)
if isdir(di):
dirs.append(di)
for di in dirs:
fnames=listdir(di)
# Retain only the .cmp files
libfiles=[]
for f in fnames:
n,ext=splitext(f)
if ext==".cmp":
libfiles.append(f)
if len(libfiles)>0:
libname=basename(di)
filename=[ join(di,fn) for fn in libfiles]
print "Loading component library",libname," from files ",filename
Library(filename,libname=libname)
if (debug_path != ''):
debug_subdir_path = join(debug_path, subfolder, dirname)
if (not isdir(debug_subdir_path)):
os.makedirs(debug_subdir_path)
mask_np = scipy.misc.imread(join(dirpath, 'mask.png')) > 127
no_mask_np = np.logical_not(mask_np)
dates = []
occupancies = []
orbit_ids = []
orbit_directions = []
ious = []
last_occupancy_np = None
for filename in sorted(os.listdir(im_path)):
filename_dt = datetime.strptime('_'.join(filename.split('_')[:2]),
"%Y%m%d_%H%M%S")
fileid = splitext(filename)[0].split('_')
orbit_direction, orbit_id = common.get_direction_orbit(fileid)
if (orbit_id is None):
orbit_id = '?'
if (orbit_direction is None):
orbit_direction = '?'
filepath = join(im_path, filename)
im_np = tifffile.imread(filepath)
threshold_np = im_np[:, :, 0] > args.threshold
threshold_np[no_mask_np] = 0
if (debug_subdir_path != ''):
tifffile.imsave(join(debug_subdir_path, filename),
threshold_np.astype(float))
def get_task_lock_file(out_filename):
"Get task-lock filename."
return "%s.X" % splitext(out_filename)[0]
def _file2dict(filename, use_predefined_variables=True):
filename = osp.abspath(osp.expanduser(filename))
check_file_exist(filename)
fileExtname = osp.splitext(filename)[1]
if fileExtname not in ['.py', '.json', '.yaml', '.yml']:
raise IOError('Only py/yml/yaml/json type are supported now!')
with tempfile.TemporaryDirectory() as temp_config_dir:
temp_config_file = tempfile.NamedTemporaryFile(dir=temp_config_dir,
suffix=fileExtname)
if platform.system() == 'Windows':
temp_config_file.close()
temp_config_name = osp.basename(temp_config_file.name)
# Substitute predefined variables
if use_predefined_variables:
Config._substitute_predefined_vars(filename,
temp_config_file.name)
else:
shutil.copyfile(filename, temp_config_file.name)
if filename.endswith('.py'):
temp_module_name = osp.splitext(temp_config_name)[0]
sys.path.insert(0, temp_config_dir)
Config._validate_py_syntax(filename)
mod = import_module(temp_module_name)
sys.path.pop(0)
cfg_dict = {
name: value
for name, value in mod.__dict__.items()
if not name.startswith('__')
}
# delete imported module
del sys.modules[temp_module_name]
elif filename.endswith(('.yml', '.yaml', '.json')):
import mmcv
cfg_dict = mmcv.load(temp_config_file.name)
# close temp file
temp_config_file.close()
cfg_text = filename + '\n'
with open(filename, 'r') as f:
cfg_text += f.read()
if BASE_KEY in cfg_dict:
cfg_dir = osp.dirname(filename)
base_filename = cfg_dict.pop(BASE_KEY)
base_filename = base_filename if isinstance(
base_filename, list) else [base_filename]
cfg_dict_list = list()
cfg_text_list = list()
for f in base_filename:
_cfg_dict, _cfg_text = Config._file2dict(osp.join(cfg_dir, f))
cfg_dict_list.append(_cfg_dict)
cfg_text_list.append(_cfg_text)
base_cfg_dict = dict()
for c in cfg_dict_list:
if len(base_cfg_dict.keys() & c.keys()) > 0:
raise KeyError('Duplicate key is not allowed among bases')
base_cfg_dict.update(c)
base_cfg_dict = Config._merge_a_into_b(cfg_dict, base_cfg_dict)
cfg_dict = base_cfg_dict
# merge cfg_text
cfg_text_list.append(cfg_text)
cfg_text = '\n'.join(cfg_text_list)
return cfg_dict, cfg_text
def create_resource(self, package_id, **kwargs):
"""Creates a single resource on filestore. You must supply either
`url`, `filepath`, or `fileobj`.
Args:
package_id (str): The filestore package id.
**kwargs: Keyword arguments that are passed to resource_create.
Kwargs:
url (str): New file url (for file link).
filepath (str): New file path (for file upload).
fileobj (obj): New file like object (for file upload).
post (bool): Post data using requests instead of ckanapi.
name (str): The resource name (defaults to the filename).
description (str): The resource description.
hash (str): The resource hash.
Returns:
obj: requests.Response object if `post` option is specified,
ckan resource object otherwise.
Raises:
TypeError: If neither `url`, `filepath`, nor `fileobj` are supplied.
Examples:
>>> ckan = CKAN(quiet=True)
>>> ckan.create_resource('pid')
Traceback (most recent call last):
TypeError: You must specify either a `url`, `filepath`, or `fileobj`
>>> ckan.create_resource('pid', url='http://example.com/file')
Package `pid` was not found.
"""
if not any(map(kwargs.get, ['url', 'filepath', 'fileobj'])):
raise TypeError(
'You must specify either a `url`, `filepath`, or `fileobj`')
path = filter(None, map(kwargs.get, ['url', 'filepath', 'fileobj']))[0]
try:
if 'docs.google.com' in path:
def_name = path.split('gid=')[1].split('&')[0]
else:
def_name = p.basename(path)
except AttributeError:
def_name = None
file_format = 'csv'
else:
# copy/pasted from utils... fix later
if 'format=' in path:
file_format = path.split('format=')[1]
else:
file_format = p.splitext(path)[1].lstrip('.')
kwargs.setdefault('name', def_name)
# Will get `ckan.logic.ValidationError` if url isn't set
kwargs.setdefault('url', 'http://example.com')
kwargs['format'] = file_format
resource = {'package_id': package_id}
message = 'Creating new resource in package %s...' % package_id
return self._update_filestore(resource, message, **kwargs)
def convert_timestamp(folder: str, ext: str = ".log") -> None:
for file in listdir(folder):
if path.splitext(file)[1] != ext:
continue
date_time = datetime.strptime(file, "%Y%m%d-%H%M%S{0}".format(ext))
rename(path.join(folder, file), path.join(folder, str(date_time.timestamp()) + ext))
'rider' : 25,
'car' : 26,
'truck' : 27,
'train' : 31,
'motorcycle': 32,
'bicycle' : 33,
'bus' : 28
}
"""
instance_classes = [24, 25, 26, 27, 28, 31, 32, 33]
anns = json.load(open('/home/wangyu/env/mmdetection_train/mmdetection/data/foggy_cityscapes/annotations/instancesonly_filtered_gtFine_val.json'))
image = anns['images']
result = pickle.load(open('gnwm_result.pkl', 'rb'))
convert_result = []
for idx, img in enumerate(result):
img_name = image[idx]
name = osp.splitext(osp.split(img_name['file_name'])[-1])[0]
for cat_idx, cat in enumerate(img) :
for obj in cat:
obj = [i.astype(float) for i in obj]
det = {}
det['image_id'] = name
det['category_id'] = instance_classes[cat_idx]
det['score'] = obj[-1]
det['bbox'] = obj[0:4]
convert_result.append(det)
with open('convert_gnwm_result.json', 'w') as f:
json.dump(convert_result, f)
def data_path(url, path=None, force_update=False, update_path=None):
"""Get path to local copy of MEGSIM dataset URL
This is a low-level function useful for getting a local copy of a
remote MEGSIM dataet.
Parameters
----------
url : str
The dataset to use.
path : None | str
Location of where to look for the MEGSIM data storing location.
If None, the environment variable or config parameter
MNE_DATASETS_MEGSIM_PATH is used. If it doesn't exist, the
"mne-python/examples" directory is used. If the MEGSIM dataset
is not found under the given path (e.g., as
"mne-python/examples/MEGSIM"), the data
will be automatically downloaded to the specified folder.
force_update : bool
Force update of the dataset even if a local copy exists.
update_path : bool | None
If True, set the MNE_DATASETS_MEGSIM_PATH in mne-python
config to the given path. If None, the user is prompted.
Returns
-------
path : list of str
Local paths to the given data files. If URL was a .fif file, this
will be a list of length 1. If it was a .zip file, it may potentially
be many files.
Notes
-----
For example, one could do:
>>> from mne.datasets import megsim
>>> url = 'http://cobre.mrn.org/megsim/simdata/neuromag/visual/M87174545_vis_sim1A_4mm_30na_neuro_rn.fif'
>>> megsim.data_path(url, os.getenv('HOME') + '/datasets') # doctest:+SKIP
And this would download the given MEGSIM data file to the 'datasets'
folder, and prompt the user to save the 'datasets' path to the mne-python
config, if it isn't there already.
The MEGSIM dataset is documented in the following publication:
Aine CJ, Sanfratello L, Ranken D, Best E, MacArthur JA, Wallace T,
Gilliam K, Donahue CH, Montano R, Bryant JE, Scott A, Stephen JM
(2012) MEG-SIM: A Web Portal for Testing MEG Analysis Methods using
Realistic Simulated and Empirical Data. Neuroinform 10:141-158
"""
if path is None:
# use an intelligent guess if it's not defined
def_path = op.realpath(
op.join(op.dirname(__file__), '..', '..', '..', 'examples'))
key = 'MNE_DATASETS_MEGSIM_PATH'
if get_config(key) is None:
key = 'MNE_DATA'
path = get_config(key, def_path)
# use the same for all datasets
if not op.exists(path) or not os.access(path, os.W_OK):
try:
os.mkdir(path)
except OSError:
try:
logger.info("Checking for megsim data in '~/mne_data'...")
path = op.join(op.expanduser("~"), "mne_data")
if not op.exists(path):
logger.info("Trying to create "
"'~/mne_data' in home directory")
os.mkdir(path)
except OSError:
raise OSError(
"User does not have write permissions "
"at '%s', try giving the path as an argument "
"to data_path() where user has write "
"permissions, for ex:data_path"
"('/home/xyz/me2/')" % (path))
if not isinstance(path, string_types):
raise ValueError('path must be a string or None')
destination = _url_to_local_path(url, op.join(path, 'MEGSIM'))
destinations = [destination]
split = op.splitext(destination)
is_zip = True if split[1].lower() == '.zip' else False
# Fetch the file
do_unzip = False
if not op.isfile(destination) or force_update:
if op.isfile(destination):
os.remove(destination)
if not op.isdir(op.dirname(destination)):
os.makedirs(op.dirname(destination))
_fetch_file(url, destination, print_destination=False)
do_unzip = True
if is_zip:
z = zipfile.ZipFile(destination)
decomp_dir, name = op.split(destination)
files = z.namelist()
# decompress if necessary (if download was re-done)
if do_unzip:
stdout.write('Decompressing %g files from\n'
'"%s" ...' % (len(files), name))
z.extractall(decomp_dir)
stdout.write(' [done]\n')
z.close()
destinations = [op.join(decomp_dir, f) for f in files]
# Offer to update the path
path = op.abspath(path)
if update_path is None:
if get_config(key, '') != path:
update_path = True
msg = ('Do you want to set the path:\n %s\nas the default '
'MEGSIM dataset path in the mne-python config ([y]/n)? ' %
path)
answer = raw_input(msg)
if answer.lower() == 'n':
update_path = False
else:
update_path = False
if update_path is True:
set_config(key, path)
return destinations
def removeFileExtention(str_name):
return path.splitext(str_name)[0]
def plot_spikes(
in_file,
in_fft,
spikes_list,
cols=3,
labelfmt="t={0:.3f}s (z={1:d})",
out_file=None,
):
from mpl_toolkits.axes_grid1 import make_axes_locatable
nii = nb.as_closest_canonical(nb.load(in_file))
fft = nb.load(in_fft).get_fdata()
data = nii.get_fdata()
zooms = nii.header.get_zooms()[:2]
tstep = nii.header.get_zooms()[-1]
ntpoints = data.shape[-1]
if len(spikes_list) > cols * 7:
cols += 1
nspikes = len(spikes_list)
rows = 1
if nspikes > cols:
rows = math.ceil(nspikes / cols)
fig = plt.figure(figsize=(7 * cols, 5 * rows))
for i, (t, z) in enumerate(spikes_list):
prev = None
pvft = None
if t > 0:
prev = data[..., z, t - 1]
pvft = fft[..., z, t - 1]
post = None
psft = None
if t < (ntpoints - 1):
post = data[..., z, t + 1]
psft = fft[..., z, t + 1]
ax1 = fig.add_subplot(rows, cols, i + 1)
divider = make_axes_locatable(ax1)
ax2 = divider.new_vertical(size="100%", pad=0.1)
fig.add_axes(ax2)
plot_slice_tern(
data[..., z, t],
prev=prev,
post=post,
spacing=zooms,
ax=ax2,
label=labelfmt.format(t * tstep, z),
)
plot_slice_tern(
fft[..., z, t],
prev=pvft,
post=psft,
vmin=-5,
vmax=5,
cmap=get_parula(),
ax=ax1,
)
plt.tight_layout()
if out_file is None:
fname, ext = op.splitext(op.basename(in_file))
if ext == ".gz":
fname, _ = op.splitext(fname)
out_file = op.abspath("%s.svg" % fname)
fig.savefig(out_file, format="svg", dpi=300, bbox_inches="tight")
return out_file
def generate_filename(self, filename):
# name like: "[uid].[ext]" -> example: "bea536a0-089c-a45b.pdf"
name = self.generate_uid(
text_to_append=splitext(basename(filename))[1])
# final return result like: "profiles/bea536a0/089c/a45b.pdf":
return os.path.join(self.upload_directory, name.replace('-', '/'))
def get_options():
from optparse import OptionParser
from datetime import datetime
parser = OptionParser(usage="Usage: %prog [options] <station pickle file>", \
description="Program to create a KML file for plotting based on the contents of a station pickle file")
parser.add_option("--keys", action="store", type=str, dest="keys", default="", \
help="Specify a comma separated list of keys to return. These can be fragments " \
"of a key to include all keys matching any fragment.")
parser.add_option("-V","-v","--verb-level", action="store", type="int", dest="verb", default=0, \
help="Specify verbosity. Default 0: no output. 1: network list. 2: include station list.")
parser.add_option("-o","--outfile", action="store", type="string", dest="outfile", default="", \
help="Output kml file")
parser.add_option("-s","--icon-scale", action="store", type="string", dest="scale", default="1.8", \
help="Icon Size Scale (default 1.8)")
parser.add_option("-r","--no-random-colours", action="store_true", dest="randoff", default=False, \
help="Turn off random colours (default random on)")
parser.add_option("-T","--Document-title", action="store", type="string", dest="doctitle", default="", \
help="KML Document Title")
parser.add_option("-a","--ascii", action="store_false", dest="use_binary", default=True, \
help="Specify to write ascii Pickle files instead of binary. Ascii are larger file size, " \
"but more likely to be system independent.")
# Parse Arguments
(opts, args) = parser.parse_args()
# Parse Input Filename
if len(args) == 0:
parser.error("Must provide at least 1 input database!")
for iin in range(0, len(args)):
if args[iin].find(".pkl") > 0:
args[iin] = osp.splitext(args[iin])[0]
elif args[iin][-1] == ".":
args[iin] = args[iin][0:-1]
# create output filename
if len(opts.outfile) == 0:
outfile = args[0].replace('.pkl', '') + ".kml"
else:
outfile = opts.outfile
if outfile.find('.kml') < 0:
outfile = outfile + ".kml"
opts.outfile = outfile
# set document title
if len(opts.doctitle) == 0:
opts.doctitle = "; ".join(args) + ";; Created: " + str(datetime.now())
# Check input
errfl = []
for iin in range(0, len(args)):
if not osp.exists(args[iin] + ".pkl"):
errfl.append(args[iin] + ".pkl")
#
if len(errfl) > 0:
parser.error("Input database(s) " + ", ".join(errfl) + " do not exist")
# Construct keys
if len(opts.keys) > 0:
opts.keys = opts.keys.split(',')
# Return extension
for iin in range(0, len(args)):
args[iin] = args[iin] + ".pkl"
# return options
return opts, args
def plot_mosaic(
img,
out_file=None,
ncols=8,
title=None,
overlay_mask=None,
bbox_mask_file=None,
only_plot_noise=False,
annotate=True,
vmin=None,
vmax=None,
cmap="Greys_r",
plot_sagittal=True,
fig=None,
zmax=128,
):
if isinstance(img, (str, bytes)):
nii = nb.as_closest_canonical(nb.load(img))
img_data = nii.get_fdata()
zooms = nii.header.get_zooms()
else:
img_data = img
zooms = [1.0, 1.0, 1.0]
out_file = "mosaic.svg"
# Remove extra dimensions
img_data = np.squeeze(img_data)
if img_data.shape[2] > zmax and bbox_mask_file is None:
lowthres = np.percentile(img_data, 5)
mask_file = np.ones_like(img_data)
mask_file[img_data <= lowthres] = 0
img_data = _bbox(img_data, mask_file)
if bbox_mask_file is not None:
bbox_data = nb.as_closest_canonical(
nb.load(bbox_mask_file)).get_fdata()
img_data = _bbox(img_data, bbox_data)
z_vals = np.array(list(range(0, img_data.shape[2])))
# Reduce the number of slices shown
if len(z_vals) > zmax:
rem = 15
# Crop inferior and posterior
if not bbox_mask_file:
# img_data = img_data[..., rem:-rem]
z_vals = z_vals[rem:-rem]
else:
# img_data = img_data[..., 2 * rem:]
start_index = 2 * rem
z_vals = z_vals[start_index:]
while len(z_vals) > zmax:
# Discard one every two slices
# img_data = img_data[..., ::2]
z_vals = z_vals[::2]
n_images = len(z_vals)
nrows = math.ceil(n_images / ncols)
if plot_sagittal:
nrows += 1
if overlay_mask:
overlay_data = nb.as_closest_canonical(
nb.load(overlay_mask)).get_fdata()
# create figures
if fig is None:
fig = plt.figure(figsize=(22, nrows * 3))
est_vmin, est_vmax = _get_limits(img_data, only_plot_noise=only_plot_noise)
if not vmin:
vmin = est_vmin
if not vmax:
vmax = est_vmax
naxis = 1
for z_val in z_vals:
ax = fig.add_subplot(nrows, ncols, naxis)
if overlay_mask:
ax.set_rasterized(True)
plot_slice(
img_data[:, :, z_val],
vmin=vmin,
vmax=vmax,
cmap=cmap,
ax=ax,
spacing=zooms[:2],
label="%d" % z_val,
annotate=annotate,
)
if overlay_mask:
from matplotlib import cm
msk_cmap = cm.Reds # @UndefinedVariable
msk_cmap._init()
alphas = np.linspace(0, 0.75, msk_cmap.N + 3)
msk_cmap._lut[:, -1] = alphas
plot_slice(
overlay_data[:, :, z_val],
vmin=0,
vmax=1,
cmap=msk_cmap,
ax=ax,
spacing=zooms[:2],
)
naxis += 1
if plot_sagittal:
naxis = ncols * (nrows - 1) + 1
step = int(img_data.shape[0] / (ncols + 1))
start = step
stop = img_data.shape[0] - step
if step == 0:
step = 1
for x_val in list(range(start, stop, step))[:ncols]:
ax = fig.add_subplot(nrows, ncols, naxis)
plot_slice(
img_data[x_val, ...],
vmin=vmin,
vmax=vmax,
cmap=cmap,
ax=ax,
label="%d" % x_val,
spacing=[zooms[0], zooms[2]],
)
naxis += 1
fig.subplots_adjust(left=0.05,
right=0.95,
bottom=0.05,
top=0.95,
wspace=0.05,
hspace=0.05)
if title:
fig.suptitle(title, fontsize="10")
fig.subplots_adjust(wspace=0.002, hspace=0.002)
if out_file is None:
fname, ext = op.splitext(op.basename(img))
if ext == ".gz":
fname, _ = op.splitext(fname)
out_file = op.abspath(fname + "_mosaic.svg")
fig.savefig(out_file, format="svg", dpi=300, bbox_inches="tight")
return out_file
def is_supported_files(filename):
supported = ".png .jpg".split()
name, ext = splitext(filename)
return ext in supported
def _save_thumbnails(self, thumbnail_objects):
raw_path, raw_filename = self._get_image_path_and_name()
path = join(raw_path, settings.THUMBNAILS_NAME)
filename, _ = splitext(raw_filename)
save_thumbnail_to_path = self._init_save_thumbnail(filename, path)
return filter(lambda thumbnail_object: save_thumbnail_to_path(thumbnail_object), thumbnail_objects)
meta['fel_shutter'], # uint8
meta['laser_shutter'], # uint8
0, # uint8
meta['fel_intensity'], # float64
meta['delay_motor'], # float64
0, # float64
0, # float64
hits.get('nhits', 0))) # uint32
for hit in hits.get('hits', ()):
write(pack2(hit['t'], hit['x'], hit['y']))
print("Done!")
while True: # run conversion in infinite loop
hits = {
splitext(basename(fn))[0]: getmtime(fn)
for fn in iglob(hit_filename("*"))
}
bins = {
splitext(basename(fn))[0]: getctime(fn)
for fn in iglob(bin_filename("*"))
}
jobs = sorted(fn for fn, t in hits.items()
if fn not in bins or bins[fn] < t)
print("Jobs: {}".format(', '.join(jobs)))
if len(jobs) == 0:
print("Nothing to do!")
else:
for fn in jobs:
print('Converting file {}...'.format(fn))
try:
domainName = getenv("domainEnv")
username = getenv("uname")
password = getenv("pword")
wlUrl = getenv("url")
appLocation = getenv("appLoc")
filenames = []
lstWOExt = []
for files in listdir(appLocation):
if files.endswith('.war'):
filenames.append(files)
print(filenames)
for filext in filenames:
lstWOExt.append(splitext(filext)[0])
print(lstWOExt)
if not listdir(appLocation):
print('War file directory is empty. Nothing to deploy...')
else:
print('There are war files in this directory')
for file in lstWOExt:
deploymentFile = '' + appLocation + '\\' + file + '.war'
try:
print('Trying to connect with ' + domainName + '')
connect(username, password, wlUrl)
print('Successfully connected with ' + domainName + '')
except:
print('Couldn\'t successfully connect with ' + domainName + '...')
def from_reads(reads_folders_list_path,
output_path,
kmer_size,
abundance_min,
filter_singleton,
phenotype_name,
phenotype_metadata_path,
gzip,
temp_dir,
nb_cores,
verbose,
progress,
warning_callback=None,
error_callback=None):
supported_extensions = ['.fastq', '.fastq.gz']
compression = "gzip" if gzip > 0 else None
compression_opts = gzip if gzip > 0 else None
# Execution callback functions
if warning_callback is None:
warning_callback = lambda w: logging.warning(w)
if error_callback is None:
def normal_raise(exception):
raise exception
error_callback = normal_raise
# Make sure that the tmp data is unique to the current process
temp_dir = join(temp_dir, str(getpid()))
if not exists(temp_dir):
mkdir(temp_dir)
if (phenotype_name is None and phenotype_metadata_path is not None) or (
phenotype_name is not None and phenotype_metadata_path is None):
error_callback(
ValueError(
"If a phenotype is specified, it must have a name and a metadata file."
))
# Find the read folder for each genome and verify that it exists
reads_folder_by_genome_id = dict(
l.split() for l in open(reads_folders_list_path, "r"))
for g_id, read_dir in reads_folder_by_genome_id.iteritems():
if not exists(read_dir):
error_callback(
IOError(
"The read directory for genome %s cannot be found: %s" %
(str(g_id), read_dir)))
logging.debug("The k-mer matrix contains %d genomes." %
len(reads_folder_by_genome_id))
if len(set(reads_folder_by_genome_id.keys())) < len(
reads_folder_by_genome_id.keys()):
error_callback(
Exception(
"The genomic data contains genomes with the same identifier."))
h5py_file = _create_hdf5_file_no_chunk_caching(output_path)
h5py_file.attrs["created"] = time()
h5py_file.attrs["uuid"] = str(uuid1())
h5py_file.attrs["genome_source_type"] = "reads"
h5py_file.attrs["genomic_data"] = reads_folders_list_path
h5py_file.attrs[
"phenotype_name"] = phenotype_name if phenotype_name is not None else "NA"
h5py_file.attrs[
"phenotype_metadata_source"] = phenotype_metadata_path if phenotype_metadata_path is not None else "NA"
h5py_file.attrs["filter"] = filter_singleton
h5py_file.attrs["compression"] = "gzip (level %d)" % gzip
# Extract/write the metadata
if phenotype_name is not None:
genome_ids, labels = _parse_metadata(phenotype_metadata_path,
reads_folder_by_genome_id.keys(),
warning_callback, error_callback)
# Sort the genomes by label for optimal better performance
logging.debug(
"Sorting genomes by metadata label for optimal performance.")
sorter = np.argsort(labels)
genome_ids = genome_ids[sorter]
labels = labels[sorter]
logging.debug("Creating the phenotype metadata dataset.")
phenotype = h5py_file.create_dataset("phenotype",
data=labels,
dtype=PHENOTYPE_LABEL_DTYPE)
phenotype.attrs["name"] = phenotype_name
del phenotype, labels
# Write genome ids
logging.debug("Creating the genome identifier dataset.")
h5py_file.create_dataset("genome_identifiers",
data=genome_ids,
compression=compression,
compression_opts=compression_opts)
h5py_file.close()
logging.debug("Initializing DSK.")
# Preparing input file for multidsk
files_sorted = []
list_reads_dsk_output = []
for id in genome_ids:
files = [
join(reads_folder_by_genome_id[id], file)
for file in listdir(reads_folder_by_genome_id[id])
if file.endswith(tuple(supported_extensions))
] # Supported extensions
files_sorted.append(",".join(files) + "\n")
list_reads_dsk_output.append(
join(temp_dir,
basename(splitext(files[-1])[0]) + ".h5"))
open(join(temp_dir, "list_reads_files"), "w").writelines(files_sorted)
# Calling multidsk
reads_count_kmers(file_path=join(temp_dir, "list_reads_files"),
out_dir=temp_dir,
kmer_size=kmer_size,
abundance_min=abundance_min,
out_compress=gzip,
nb_cores=nb_cores,
verbose=int(verbose),
progress=progress)
logging.debug("K-mers counting completed.")
# Preparing input file for dsk2kover
file_dsk_output = open(join(temp_dir, "list_h5"), "w")
for line in list_reads_dsk_output:
file_dsk_output.write(line + "\n")
file_dsk_output.close()
# Calling dsk2kover
logging.debug("Initializing DSK2Kover.")
reads_pack_kmers(file_path=join(temp_dir, "list_h5"),
out_path=output_path,
filter_singleton=filter_singleton,
kmer_length=kmer_size,
compression=gzip,
chunk_size=BLOCK_SIZE,
nb_genomes=len(genome_ids),
progress=progress)
logging.debug("Removing temporary files.")
rmtree(temp_dir)
# progress_callback("dsk2kover", 1)
logging.debug("Dataset creation completed.")
def iter_files_distros(path=None, repeated_distro='first'):
if path is None:
path = sys.path
# Distributions found earlier in path will shadow those with the same name
# found later. If these distributions used different module names, it may
# actually be possible to import both, but in most cases this shadowing
# will be correct.
distro_names_seen = set()
for folder in path:
if folder.rstrip('/\\').endswith('.egg'):
# Gah, eggs
egg_name = osp.basename(folder)
distro = Distribution.from_name_version(egg_name.split(".egg")[0])
if (repeated_distro == 'first') \
and (distro.name in distro_names_seen):
continue
distro_names_seen.add(distro.name)
if osp.isdir(folder):
ep_path = osp.join(folder, 'EGG-INFO', 'entry_points.txt')
if osp.isfile(ep_path):
cp = CaseSensitiveConfigParser(delimiters=('=', ))
cp.read([ep_path])
yield cp, distro
elif zipfile.is_zipfile(folder):
z = zipfile.ZipFile(folder)
try:
info = z.getinfo('EGG-INFO/entry_points.txt')
except KeyError:
continue
cp = CaseSensitiveConfigParser(delimiters=('=', ))
with z.open(info) as f:
fu = io.TextIOWrapper(f)
cp.read_file(fu,
source=osp.join(folder, 'EGG-INFO',
'entry_points.txt'))
yield cp, distro
# zip imports, not egg
elif zipfile.is_zipfile(folder):
with zipfile.ZipFile(folder) as zf:
for info in zf.infolist():
m = file_in_zip_pattern.match(info.filename)
if not m:
continue
distro_name_version = m.group('dist_version')
distro = Distribution.from_name_version(
distro_name_version)
if (repeated_distro == 'first') \
and (distro.name in distro_names_seen):
continue
distro_names_seen.add(distro.name)
cp = CaseSensitiveConfigParser(delimiters=('=', ))
with zf.open(info) as f:
fu = io.TextIOWrapper(f)
cp.read_file(fu,
source=osp.join(folder, info.filename))
yield cp, distro
# Regular file imports (not egg, not zip file)
for path in itertools.chain(
glob.iglob(
osp.join(glob.escape(folder), '*.dist-info',
'entry_points.txt')),
glob.iglob(
osp.join(glob.escape(folder), '*.egg-info',
'entry_points.txt'))):
distro_name_version = osp.splitext(osp.basename(
osp.dirname(path)))[0]
distro = Distribution.from_name_version(distro_name_version)
if (repeated_distro == 'first') \
and (distro.name in distro_names_seen):
continue
distro_names_seen.add(distro.name)
cp = CaseSensitiveConfigParser(delimiters=('=', ))
cp.read([path])
yield cp, distro
def read_montage(kind, ch_names=None, path=None, unit='m', transform=False):
"""Read a generic (built-in) montage from a file
This function can be used to read electrode positions from a user specified
file using the `kind` and `path` parameters. Alternatively, use only the
`kind` parameter to load one of the built-in montages:
=================== =====================================================
Kind description
=================== =====================================================
standard_1005 Electrodes are named and positioned according to the
international 10-05 system.
standard_1020 Electrodes are named and positioned according to the
international 10-20 system.
standard_alphabetic Electrodes are named with LETTER-NUMBER combinations
(A1, B2, F4, etc.)
standard_postfixed Electrodes are named according to the international
10-20 system using postfixes for intermediate
positions.
standard_prefixed Electrodes are named according to the international
10-20 system using prefixes for intermediate
positions.
standard_primed Electrodes are named according to the international
10-20 system using prime marks (' and '') for
intermediate positions.
biosemi16 BioSemi cap with 16 electrodes
biosemi32 BioSemi cap with 32 electrodes
biosemi64 BioSemi cap with 64 electrodes
biosemi128 BioSemi cap with 128 electrodes
biosemi160 BioSemi cap with 160 electrodes
biosemi256 BioSemi cap with 256 electrodes
easycap-M10 Brainproducts EasyCap with electrodes named
according to the 10-05 system
easycap-M1 Brainproduct EasyCap with numbered electrodes
EGI_256 Geodesic Sensor Net with 256 channels
GSN-HydroCel-32 HydroCel Geodesic Sensor Net with 32 electrodes
GSN-HydroCel-64_1.0 HydroCel Geodesic Sensor Net with 64 electrodes
GSN-HydroCel-65_1.0 HydroCel Geodesic Sensor Net with 64 electrodes + Cz
GSN-HydroCel-128 HydroCel Geodesic Sensor Net with 128 electrodes
GSN-HydroCel-129 HydroCel Geodesic Sensor Net with 128 electrodes + Cz
GSN-HydroCel-256 HydroCel Geodesic Sensor Net with 256 electrodes
GSN-HydroCel-257 HydroCel Geodesic Sensor Net with 256 electrodes + Cz
=================== =====================================================
Parameters
----------
kind : str
The name of the montage file (e.g. kind='easycap-M10' for
'easycap-M10.txt'). Files with extensions '.elc', '.txt', '.csd',
'.elp', '.hpts', '.sfp' or '.loc' ('.locs' and '.eloc') are supported.
ch_names : list of str | None
If not all electrodes defined in the montage are present in the EEG
data, use this parameter to select subset of electrode positions to
load. If None (default), all defined electrode positions are returned.
path : str | None
The path of the folder containing the montage file. Defaults to the
mne/channels/data/montages folder in your mne-python installation.
unit : 'm' | 'cm' | 'mm'
Unit of the input file. If not 'm' (default), coordinates will be
rescaled to 'm'.
transform : bool
If True, points will be transformed to Neuromag space.
The fidicuals, 'nasion', 'lpa', 'rpa' must be specified in
the montage file. Useful for points captured using Polhemus FastSCAN.
Default is False.
Returns
-------
montage : instance of Montage
The montage.
See Also
--------
read_dig_montage : To read subject-specific digitization information.
Notes
-----
Built-in montages are not scaled or transformed by default.
Montages can contain fiducial points in addition to electrode
locations, e.g. ``biosemi-64`` contains 67 total channels.
.. versionadded:: 0.9.0
"""
if path is None:
path = op.join(op.dirname(__file__), 'data', 'montages')
if not op.isabs(kind):
supported = ('.elc', '.txt', '.csd', '.sfp', '.elp', '.hpts', '.loc',
'.locs', '.eloc')
montages = [op.splitext(f) for f in os.listdir(path)]
montages = [m for m in montages if m[1] in supported and kind == m[0]]
if len(montages) != 1:
raise ValueError('Could not find the montage. Please provide the '
'full path.')
kind, ext = montages[0]
fname = op.join(path, kind + ext)
else:
kind, ext = op.splitext(kind)
fname = op.join(path, kind + ext)
if ext == '.sfp':
# EGI geodesic
with open(fname, 'r') as f:
lines = f.read().replace('\t', ' ').split("\n")
ch_names_, pos = [], []
for ii, line in enumerate(lines):
line = line.strip().split()
if len(line) > 0: # skip empty lines
if len(line) != 4: # name, x, y, z
raise ValueError("Malformed .sfp file in line " + str(ii))
this_name, x, y, z = line
ch_names_.append(this_name)
pos.append([float(cord) for cord in (x, y, z)])
pos = np.asarray(pos)
elif ext == '.elc':
# 10-5 system
ch_names_ = []
pos = []
with open(fname) as fid:
for line in fid:
if 'Positions\n' in line:
break
pos = []
for line in fid:
if 'Labels\n' in line:
break
pos.append(list(map(float, line.split())))
for line in fid:
if not line or not set(line) - set([' ']):
break
ch_names_.append(line.strip(' ').strip('\n'))
pos = np.array(pos)
elif ext == '.txt':
# easycap
try: # newer version
data = np.genfromtxt(fname, dtype='str', skip_header=1)
except TypeError:
data = np.genfromtxt(fname, dtype='str', skiprows=1)
ch_names_ = list(data[:, 0])
theta, phi = data[:, 1].astype(float), data[:, 2].astype(float)
x = 85. * np.cos(np.deg2rad(phi)) * np.sin(np.deg2rad(theta))
y = 85. * np.sin(np.deg2rad(theta)) * np.sin(np.deg2rad(phi))
z = 85. * np.cos(np.deg2rad(theta))
pos = np.c_[x, y, z]
elif ext == '.csd':
# CSD toolbox
dtype = [('label', 'S4'), ('theta', 'f8'), ('phi', 'f8'),
('radius', 'f8'), ('x', 'f8'), ('y', 'f8'), ('z', 'f8'),
('off_sph', 'f8')]
try: # newer version
table = np.loadtxt(fname, skip_header=2, dtype=dtype)
except TypeError:
table = np.loadtxt(fname, skiprows=2, dtype=dtype)
ch_names_ = table['label']
theta = (2 * np.pi * table['theta']) / 360.
phi = (2 * np.pi * table['phi']) / 360.
pos = _sphere_to_cartesian(theta, phi, r=1.0)
pos = np.asarray(pos).T
elif ext == '.elp':
# standard BESA spherical
dtype = np.dtype('S8, S8, f8, f8, f8')
try:
data = np.loadtxt(fname, dtype=dtype, skip_header=1)
except TypeError:
data = np.loadtxt(fname, dtype=dtype, skiprows=1)
az = data['f2']
horiz = data['f3']
radius = np.abs(az / 180.)
angles = np.array(
[90. - h if a >= 0. else -90. - h for h, a in zip(horiz, az)])
sph_phi = (0.5 - radius) * 180.
sph_theta = angles
azimuth = sph_theta / 180.0 * np.pi
elevation = sph_phi / 180.0 * np.pi
r = 85.
y, x, z = _sphere_to_cartesian(azimuth, elevation, r)
pos = np.c_[x, y, z]
ch_names_ = data['f1'].astype(np.str)
elif ext == '.hpts':
# MNE-C specified format for generic digitizer data
dtype = [('type', 'S8'), ('name', 'S8'), ('x', 'f8'), ('y', 'f8'),
('z', 'f8')]
data = np.loadtxt(fname, dtype=dtype)
pos = np.vstack((data['x'], data['y'], data['z'])).T
ch_names_ = data['name'].astype(np.str)
elif ext in ('.loc', '.locs', '.eloc'):
ch_names_ = np.loadtxt(fname, dtype='S4',
usecols=[3]).astype(np.str).tolist()
dtype = {'names': ('angle', 'radius'), 'formats': ('f4', 'f4')}
angle, radius = np.loadtxt(fname,
dtype=dtype,
usecols=[1, 2],
unpack=True)
sph_phi, sph_theta = _topo_to_sphere(angle, radius)
azimuth = sph_theta / 180.0 * np.pi
elevation = sph_phi / 180.0 * np.pi
r = np.ones((len(ch_names_), ))
x, y, z = _sphere_to_cartesian(azimuth, elevation, r)
pos = np.c_[-y, x, z]
else:
raise ValueError('Currently the "%s" template is not supported.' %
kind)
selection = np.arange(len(pos))
if unit == 'mm':
pos /= 1e3
elif unit == 'cm':
pos /= 1e2
elif unit != 'm':
raise ValueError("'unit' should be either 'm', 'cm', or 'mm'.")
if transform:
names_lower = [name.lower() for name in list(ch_names_)]
if ext == '.hpts':
fids = ('2', '1', '3') # Alternate cardinal point names
else:
fids = ('nz', 'lpa', 'rpa')
missing = [name for name in fids if name not in names_lower]
if missing:
raise ValueError("The points %s are missing, but are needed "
"to transform the points to the MNE coordinate "
"system. Either add the points, or read the "
"montage with transform=False. " % missing)
nasion = pos[names_lower.index(fids[0])]
lpa = pos[names_lower.index(fids[1])]
rpa = pos[names_lower.index(fids[2])]
neuromag_trans = get_ras_to_neuromag_trans(nasion, lpa, rpa)
pos = apply_trans(neuromag_trans, pos)
if ch_names is not None:
sel, ch_names_ = zip(*[(i, e) for i, e in enumerate(ch_names_)
if e in ch_names])
sel = list(sel)
pos = pos[sel]
selection = selection[sel]
else:
ch_names_ = list(ch_names_)
kind = op.split(kind)[-1]
return Montage(pos=pos, ch_names=ch_names_, kind=kind, selection=selection)
def post_process_txt_labels(txtdir,
wavdir=None,
output_folder=None,
output_tsv=None,
min_dur_event=0.250,
min_dur_inter=0.150,
background_label=False,
rm_nOn_nOff=True):
""" clean the .txt files of each file. It is the same processing as the real data
- overlapping events of the same class are mixed
- if silence < 150ms between two conscutive events of the same class, they are mixed
- if event < 250ms, the event lasts 250ms
Args:
txtdir: str, directory path where the XXX.txt files are.
wavdir: str, directory path where the associated XXX.wav audio files are (associated with .txt files)
output_folder: str, optional, folder in which to put the checked files
output_tsv: str, optional, tsv with all the annotations concatenated
min_dur_event: float, optional in sec, minimum duration of an event
min_dur_inter: float, optional in sec, minimum duration between 2 events
background_label: bool, whether to include the background label in the annotations.
rm_nOn_nOff: bool, whether to delete the additional _nOn _nOff at the end of labels.
Returns:
None
"""
logger = create_logger(__name__ + "/" +
inspect.currentframe().f_code.co_name)
if wavdir is None:
wavdir = txtdir
fix_count = 0
logger.info(
"Correcting annotations ... \n"
"* annotations with negative duration will be removed\n" +
"* annotations with duration <250ms will be extended on the offset side)"
)
if output_folder is not None:
create_folder(output_folder)
df_single = pd.DataFrame() # only useful if output_csv defined
if background_label:
list_files = glob.glob(osp.join(txtdir, "*.jams"))
else:
list_files = glob.glob(osp.join(txtdir, "*.txt"))
if len(list_files) == 0:
list_files = glob.glob(osp.join(txtdir, '*.jams'))
out_extension = '.txt'
for fn in list_files:
logger.debug(fn)
df, length_sec = get_data(
fn,
osp.join(wavdir,
osp.splitext(osp.basename(fn))[0] + '.wav'),
background_label=background_label)
df, fc = _post_process_labels_file(df, length_sec, min_dur_event,
min_dur_inter, rm_nOn_nOff)
fix_count += fc
if output_folder is not None:
filepath = os.path.splitext(
os.path.basename(fn))[0] + out_extension
df[['onset', 'offset',
'event_label']].to_csv(osp.join(output_folder, filepath),
header=False,
index=False,
sep="\t")
if output_tsv is not None:
df['filename'] = osp.join(
osp.splitext(osp.basename(fn))[0] + '.wav')
df_single = df_single.append(
df[['filename', 'onset', 'offset', 'event_label']],
ignore_index=True)
if output_tsv:
save_tsv(df_single, output_tsv)
logger.info(f"{fix_count} problems Fixed")
