def test_global_check_and_location_preprocessors_should_be_applied_after_global_custom_preprocessors(self):
ui = self._create_ui()
ui.add_library('lib')
ui.enable_location_preprocessing()
ui.enable_check_preprocessing()
ui.add_preprocessor(TestPreprocessor())
file_name = self.create_entity_file()
ui.add_source_files(file_name, 'lib')
pp_source = Template("""\
-- check_relation(a = b, line_num => 1, file_name => "$file", \
auto_msg => "Relation a = b failed! Left is " & to_string(a) & ". Right is " & to_string(b) & ".");
library vunit_lib;
context vunit_lib.vunit_context;
entity $entity is
end entity;
architecture arch of $entity is
begin
log("Hello World", line_num => 11, file_name => "$file");
check_relation(1 /= 2, line_num => 12, file_name => "$file", \
auto_msg => "Relation 1 /= 2 failed! Left is " & to_string(1) & ". Right is " & to_string(2) & ".");
report "Here I am!";
end architecture;
""")
with open(join(self._preprocessed_path, 'lib', basename(file_name))) as fread:
self.assertEqual(fread.read(), pp_source.substitute(entity='ent0', file=basename(file_name)))
def _preprocess(self, library_name, file_name, preprocessors):
"""
Preprocess file_name within library_name using explicit preprocessors
if preprocessors is None then use implicit globally defined processors
"""
# @TODO dependency checking etc...
if preprocessors is None:
preprocessors = [self._location_preprocessor, self._check_preprocessor]
preprocessors = [p for p in preprocessors if p is not None]
preprocessors = self._external_preprocessors + preprocessors
if len(preprocessors) == 0:
return file_name
code = ostools.read_file(file_name)
for preprocessor in preprocessors:
code = preprocessor.run(code, basename(file_name))
pp_file_name = join(self._preprocessed_path, library_name, basename(file_name))
idx = 1
while ostools.file_exists(pp_file_name):
LOGGER.debug("Preprocessed file exists '%s', adding prefix", pp_file_name)
pp_file_name = join(self._preprocessed_path,
library_name, "%i_%s" % (idx, basename(file_name)))
idx += 1
ostools.write_file(pp_file_name, code)
return pp_file_name
def _copy_contents(dst_dir, contents):
items = {"dirs": set(), "files": set()}
for path in contents:
if isdir(path):
items['dirs'].add(path)
elif isfile(path):
items['files'].add(path)
dst_dir_name = basename(dst_dir)
if dst_dir_name == "src" and len(items['dirs']) == 1:
copytree(list(items['dirs']).pop(), dst_dir, symlinks=True)
else:
makedirs(dst_dir)
for d in items['dirs']:
copytree(d, join(dst_dir, basename(d)), symlinks=True)
if not items['files']:
return
if dst_dir_name == "lib":
dst_dir = join(dst_dir, mkdtemp(dir=dst_dir))
for f in items['files']:
copyfile(f, join(dst_dir, basename(f)))
def _find_in_most_specific_archive(self, path):
file_name = basename(path)
dir_name = dirname(path)
while len(dir_name) > 0:
archive_name = join(self.data_root, dir_name + archive_extension)
if access(archive_name, R_OK) and \
archive_name not in self.open_archives:
self.open_archives[archive_name] = ZipFile(archive_name, "r")
if archive_name in self.open_archives:
# print "expecting to find file %s in archive %s" % \
# (file_name, archive_name)
return self.open_archives[archive_name].open(file_name)
# XXX open() method is for ZipFile; would be different for
# Tar if we want to change archive type.
file_name = join(basename(dir_name), file_name)
dir_name = dirname(dir_name)
raise IOError(
"File not found (%s), not even in archive; data_root is %s" % \
(path, self.data_root)
)
def vars2png(self, wrfout_path, dom_id, ts_esmf, vars):
"""
Postprocess a list of scalar fields into KMZ files.
:param wrfout_path: WRF file to process
:param dom_id: the domain identifier
:param ts_esmf: time stamp in ESMF format
:param vars: list of variables to process
"""
# open the netCDF dataset
d = nc4.Dataset(wrfout_path)
# extract ESMF string times and identify timestamp of interest
times = [''.join(x) for x in d.variables['Times'][:]]
if ts_esmf not in times:
raise PostprocError("Invalid timestamp %s" % ts_esmf)
tndx = times.index(ts_esmf)
# build one KMZ per variable
for var in vars:
try:
outpath_base = os.path.join(self.output_path, self.product_name + ("-%02d-" % dom_id) + ts_esmf + "-" + var)
if var in ['WINDVEC']:
raster_path, coords = self._vector2png(d, var, tndx, outpath_base)
raster_name = osp.basename(raster_path)
self._update_manifest(dom_id, ts_esmf, var, { 'raster' : raster_name, 'coords' : coords})
else:
raster_path, cb_path, coords = self._scalar2png(d, var, tndx, outpath_base)
mf_upd = { 'raster' : osp.basename(raster_path), 'coords' : coords}
if cb_path is not None:
mf_upd['colorbar'] = osp.basename(cb_path)
self._update_manifest(dom_id, ts_esmf, var, mf_upd)
except Exception as e:
logging.warning("Exception %s while postprocessing %s for time %s into PNG" % (e.message, var, ts_esmf))
logging.warning(traceback.print_exc())
def submit(self, root=None, force=False, repo=None):
"""Create a dataset for the bundle, then add a resource for each of the
extracts listed in the bundle.yaml file"""
import databundles.util as du
if repo:
self.repo_name = repo
self.set_api()
from os.path import basename
ckb = self.remote.update_or_new_bundle_extract(self.bundle)
sent = set()
self.remote.put_package(ckb)
for doc in self.bundle.config.group('about').get('documents',[]):
self.store_document(ckb, doc)
for extract_data in self.generate_extracts(root=root):
file_ = self._do_extract(extract_data, force=force)
if file_ not in sent:
r = self._send(ckb, extract_data,file_)
sent.add(file_)
url = r['ckan_url']
self.bundle.log("Submitted {} to {}".format(basename(file_), url))
else:
self.bundle.log("Already processed {}, not sending.".format(basename(file_)))
return True
def cpMCNPproject(directory):
wkdir=getcwd()
if checkifMCNPproject(directory,1)==1:
return 1
elif checkifMCNPproject(wkdir,2)==2:
return 2
else:
cards = [ path.join(directory,"cards/parameters.part"),
path.join(directory,"cards/materials.part"),
path.join(directory,"cards/source.part"),
path.join(directory,"cards/tallies.part"),
path.join(directory,"cards/traslations.part")]
geom = [ path.join(directory,"geom/cells.part"),
path.join(directory,"geom/surfaces.part")]
for card in cards:
try:
copyfile(card, path.join(wkdir, "cards/",path.basename(card)))
except Exception as e:
print "\n\033[1;34mMCNPmanager cp error:\033[1;32m %s \033[0m\n" % (e)
for g in geom:
try:
copyfile(g, path.join(wkdir, "geom/",path.basename(g)))
except Exception as e:
print "\n\033[1;34mMCNPmanager cp error:\033[1;32m %s \033[0m\n" % (e)
return 0
def remote_copy(files, host, user, path, transfer_tool='ssh'):
if transfer_tool == 'paramiko':
import paramiko
pyhrf.verbose(1, 'Copying files to remote destination %s@%s:%s ...' \
%(host,user,path))
ssh = paramiko.SSHClient()
known_hosts_file = os.path.join("~", ".ssh", "known_hosts")
ssh.load_host_keys(os.path.expanduser(known_hosts_file))
ssh.connect(host, username=user)
sftp = ssh.open_sftp()
for f in files:
remotepath = op.join(path,op.basename(f))
pyhrf.verbose(2, f + ' -> ' + remotepath + ' ...')
flocal = open(f)
remote_file = sftp.file(remotepath, "wb")
remote_file.set_pipelined(True)
remote_file.write(flocal.read())
flocal.close()
remote_file.close()
sftp.close()
ssh.close()
else:
sfiles = string.join(['"%s"'%f for f in files], ' ')
scp_cmd = 'scp -C %s "%s@%s:%s"' %(sfiles, user, host, path)
pyhrf.verbose(1, 'Data files transfer with scp ...')
pyhrf.verbose(2, scp_cmd)
if os.system(scp_cmd) != 0:
raise Exception('Error while scp ...')
pyhrf.verbose(1, 'Copy done!')
return [op.join(path,op.basename(f)) for f in files]
def compile_timestamped_transcript_files(json_filenames):
"""
`json_filenames` is a list of filepaths with this filename format:
00900-01000.json
where the left-number represents the starting time offset and
the right-number represents the ending time, in seconds
Each file in this list follows the Watson API standard JSON response
Returns: a dictionary that is the result of concatenating all the json files
into one, with "results" pointing to a list of all returned responses.
To maintain compatibility with Watson's API response, "result_index" key
is included and is set to 0, i.e. it'd be as if the resulting dictionary
is the response returned when sending an entire unbroken soundstream to Watson
"""
compiled_results = []
compiled_dict = {'results': compiled_results, "result_index": 0}
filenames = sorted(json_filenames, key=lambda x: int(basename(x).split('-')[0]))
for fn in filenames:
start_offset_sec = int(basename(fn).split('-')[0])
with open(fn) as f:
data = json.load(f)
for result in data['results']:
for x in result.get('word_alternatives'):
x['start_time'] += start_offset_sec
x['end_time'] += start_offset_sec
for alt in result.get('alternatives'):
for ts in alt['timestamps']:
# each timestamp object is a list:
# ["hi", 9.93, 10.11]
ts[1] += start_offset_sec
ts[2] += start_offset_sec
compiled_results.append(result)
return compiled_dict
def ZipIt(request, Qobject6):
import zipfile
# remove old
import os
try:
pathtoold = PathMaker(request.user.username, 'results.zip')
os.remove(pathtoold)
except OSError:
pass
#from Results.models import dbResults
# Make Results object
# query = 'SELECT * FROM Inputs_dbresults WHERE username = "******" ORDER BY id DESC LIMIT 1'
# Qobject6 = dbResults.objects.raw(query)[0]
# select table columns to send; fibril.pdb, LayerLines.jpg and parameters.txt
fibfile = str(Qobject6.fibrilPDB)
LLout = str(Qobject6.LLoutputPic)
inten = 'static_in_pro/media_root/' + str(Qobject6.intensity)
parampath = PathMaker(request.user.username, 'parameters.txt')
# give file a name and location in user's dir
filename = 'results.zip'
Path = PathMaker(request.user.username, filename)
# remove paths
from os.path import basename
# buffer = StringIO()
zipped = zipfile.ZipFile(Path, 'w')
zipped.write(fibfile, basename(fibfile))
zipped.write(LLout, basename(LLout))
zipped.write(inten, basename(inten))
zipped.write(parampath, basename(parampath))
zipped.close()
def create_tag(self):
current_stack = inspect.stack()
stackid = 0
while basename(current_stack[stackid][1]) == 'environment.py' or basename(current_stack[stackid][1]) == 'nfs4client.py':
stackid = stackid + 1
test_name = '%s:%s' % (basename(current_stack[stackid][1]), current_stack[stackid][3])
return test_name
def print_usage(actions):
"""Print the usage information. (Help screen)"""
actions = actions.items()
actions.sort()
print 'usage: %s <action> [<options>]' % basename(sys.argv[0])
print ' %s --help' % basename(sys.argv[0])
print
print 'actions:'
for name, (func, doc, arguments) in actions:
print ' %s:' % name
for line in doc.splitlines():
print ' %s' % line
if arguments:
print
for arg, shortcut, default, argtype in arguments:
if isinstance(default, bool):
print ' %s' % (
(shortcut and '-%s, ' % shortcut or '') + '--' + arg
)
else:
print ' %-30s%-10s%s' % (
(shortcut and '-%s, ' % shortcut or '') + '--' + arg,
argtype, default
)
print
def __init__(self, current_file_path=''):
"""
FileDialog constructor.
Args:
current_file_path: the current directory or path to the open flow graph
"""
if not current_file_path: current_file_path = path.join(DEFAULT_FILE_PATH, NEW_FLOGRAPH_TITLE + Preferences.file_extension())
if self.type == OPEN_FLOW_GRAPH:
FileDialogHelper.__init__(self, gtk.FILE_CHOOSER_ACTION_OPEN, 'Open a Flow Graph from a File...')
self.add_and_set_filter(get_flow_graph_files_filter())
self.set_select_multiple(True)
elif self.type == SAVE_FLOW_GRAPH:
FileDialogHelper.__init__(self, gtk.FILE_CHOOSER_ACTION_SAVE, 'Save a Flow Graph to a File...')
self.add_and_set_filter(get_flow_graph_files_filter())
self.set_current_name(path.basename(current_file_path))
elif self.type == SAVE_CONSOLE:
FileDialogHelper.__init__(self, gtk.FILE_CHOOSER_ACTION_SAVE, 'Save Console to a File...')
self.add_and_set_filter(get_text_files_filter())
file_path = path.splitext(path.basename(current_file_path))[0]
self.set_current_name(file_path) #show the current filename
elif self.type == SAVE_IMAGE:
FileDialogHelper.__init__(self, gtk.FILE_CHOOSER_ACTION_SAVE, 'Save a Flow Graph Screen Shot...')
self.add_and_set_filter(get_image_files_filter())
current_file_path = current_file_path + IMAGE_FILE_EXTENSION
self.set_current_name(path.basename(current_file_path)) #show the current filename
elif self.type == OPEN_QSS_THEME:
FileDialogHelper.__init__(self, gtk.FILE_CHOOSER_ACTION_OPEN, 'Open a QSS theme...')
self.add_and_set_filter(get_qss_themes_filter())
self.set_select_multiple(False)
self.set_current_folder(path.dirname(current_file_path)) #current directory
def to_xml(self):
q = Element('dictionary')
q.attrib["value"] = basename(dirname(self.dct))
r = SubElement(q, "revision",
value=str(self._svn_revision(dirname(self.dct))),
timestamp=datetime.utcnow().isoformat(),
checksum=self._checksum(open(self.dct, 'rb').read()))
s = SubElement(r, 'corpus')
s.attrib["value"] = basename(self.fn)
s.attrib["checksum"] = self._checksum(open(self.fn, 'rb').read())
SubElement(r, 'percent').text = "%.2f" % self.get_coverage()
SubElement(r, 'total').text = str(len(self.get_words()))
SubElement(r, 'known').text = str(len(self.get_known_words()))
SubElement(r, 'unknown').text = str(len(self.get_unknown_words()))
wrx = re.compile(r"\^(.*)/")
s = SubElement(r, 'top')
for word, count in self.get_top_unknown_words():
SubElement(s, 'word', count=str(count)).text = wrx.search(word).group(1)
s = SubElement(r, 'system')
SubElement(s, 'time').text = "%.4f" % self.timer
return ("coverage", etree.tostring(q))
def copy_static_entry(source, targetdir, builder, context={},
exclude_matchers=(), level=0):
"""Copy a HTML builder static_path entry from source to targetdir.
Handles all possible cases of files, directories and subdirectories.
"""
if exclude_matchers:
relpath = relative_path(builder.srcdir, source)
for matcher in exclude_matchers:
if matcher(relpath):
return
if path.isfile(source):
target = path.join(targetdir, path.basename(source))
if source.lower().endswith('_t') and builder.templates:
# templated!
fsrc = open(source, 'r', encoding='utf-8')
fdst = open(target[:-2], 'w', encoding='utf-8')
fdst.write(builder.templates.render_string(fsrc.read(), context))
fsrc.close()
fdst.close()
else:
copyfile(source, target)
elif path.isdir(source):
if level == 0:
for entry in os.listdir(source):
if entry.startswith('.'):
continue
copy_static_entry(path.join(source, entry), targetdir,
builder, context, level=1,
exclude_matchers=exclude_matchers)
else:
target = path.join(targetdir, path.basename(source))
if path.exists(target):
shutil.rmtree(target)
shutil.copytree(source, target)
def eMailCSV(fName):
#This allows the uset to automatically send an e-mail containing the CSV file via e-mail
send_from = "From_Email"
send_to = "To_Email"
subject = "Email_Subject"
header = "To: To_Email\n From: From_Email\n Subject: Email_Subject\n\n\n"
text = "Email_Text"
eServer = "email.server.ext"
ePort = portNumber
msg = MIMEMultipart()
msg['From']=send_from
msg['To'] = send_to
msg['Date'] =formatdate(localtime=True)
msg['Subject'] = subject
msg.attach(MIMEText(text))
with open(fName, "rb") as fil:
msg.attach(MIMEApplication(
fil.read(),
Content_Disposition='attachment; filename="%s"' % basename(fName),
Name=basename(fName)
))
smtp = smtplib.SMTP(eServer,ePort)
smtp.ehlo()
#smtp.start_ssl()
smtp.starttls()
smtp.ehlo
smtp.login('Login_Name', 'Login_Password')
smtp.sendmail(send_from, send_to, msg.as_string())
smtp.close()
def _get_matfile_data():
# compare to Arielle's in the .mat file
mat_file = "sub01_session_1_raw_ROI_timeseries.mat"
mat_file = osp.join(osp.dirname(osp.realpath(__file__)), mat_file)
# to_check.keys() == ['all_rois', 'time_series',
# '__globals__', 'Nvox', '__header__', '__version__']
to_check = sio.loadmat(mat_file)
nvox = to_check['Nvox'][0]
nb_runs = to_check['time_series'].shape[2] # has shape (time, rois, nb_runs)
assert nb_runs == 4
# make a dict for nvox
check_nvox = {}
for idx, roi in enumerate(to_check['all_rois']):
k, _ = osp.splitext(osp.basename(roi[0][0]))
check_nvox[k] = nvox[idx]
# make a dict for signals
arielle_runs = []
for run in range(nb_runs):
check_signals = {}
for idx, roi in enumerate(to_check['all_rois']):
k = osp.splitext(osp.basename(roi[0][0]))[0]
check_signals[k] = to_check['time_series'][:,idx,run]
arielle_runs.append(check_signals)
return check_nvox, arielle_runs
def main():
DCT_H = 30
DCT_W = 40
a = 1
while a < len(argv):
if argv[a] == '-h':
DCT_H = int(argv[a+1])
i += 2
elif argv[a] == '-w':
DCT_W = int(argv[a+1])
i += 2
else:
stderr.write('Unknown option: %s\n' % argv[a])
return 1
for fname in stdin:
fname=fname.strip()
if len(fname) == 0: continue
f = basename(fname)[:3] # frame
us = basename(dirname(fname)).split('_')
u, s = us[0], us[1]
dct = reduce(lambda x,y: x+y,
GetDCT(fname)[:DCT_H,:DCT_W].tolist(), [])
odir = 'data/features/video/%s/%s' % (u, s)
if not exists(odir):
system('mkdir -p %s' % odir)
fdct = open('%s/%s.dct' % (odir, f), 'w')
for i in dct:
fdct.write('%f\n' % i)
fdct.close()
return 0
def createChimeraScript(self, volume, pdb):
""" Create a chimera script to visualize a pseudoatoms pdb
obteined from a given EM 3d volume.
A property will be set in the pdb object to
store the location of the script.
"""
pseudoatoms = pdb.getFileName()
scriptFile = pseudoatoms + '_chimera.cmd'
pdb._chimeraScript = String(scriptFile)
sampling = volume.getSamplingRate()
radius = sampling * self.pseudoAtomRadius.get()
fnIn = volume.getFileName()
localInputFn = self._getBasePath(fnIn)
createLink(fnIn, localInputFn)
fhCmd = open(scriptFile, 'w')
fhCmd.write("open %s\n" % basename(pseudoatoms))
fhCmd.write("rangecol bfactor,a 0 white 1 red\n")
fhCmd.write("setattr a radius %f\n" % radius)
fhCmd.write("represent sphere\n")
fhCmd.write("open %s\n" % basename(localInputFn))
threshold = 0.01
if self.maskMode == NMA_MASK_THRE:
self.maskThreshold.get()
xdim = volume.getDim()[0]
origin = xdim / 2
fhCmd.write("volume #1 level %f transparency 0.5 voxelSize %f originIndex %d\n" % (threshold, sampling, origin))
fhCmd.close()
def get(url, output=None):
if output is None:
output = P.basename(urlparse(url).path)
base = output
else:
base = P.basename(output)
with file(output, 'wb') as f:
try:
r = urllib2.urlopen(url)
except urllib2.HTTPError, e:
raise HelperError('urlopen', None, '%s: %s' % (url, e))
current = 0
size = int(r.info().get('Content-Length', -1))
while True:
block = r.read(16384)
if not block:
break
current += len(block)
if size < 0:
info('\rDownloading %s: %i kB' % (base, current/1024.), end='')
else:
info('\rDownloading %s: %i / %i kB (%0.2f%%)' % (base, current/1024., size/1024., current*100./size), end='')
f.write(block)
info('\nDone')
def _remove_res(self, res):
LEN_PREFIX = self.db.LEN_PREFIX
res_id, last_updated, name, res_data = self.res_index[res.area_path]
# res_data: {lang -> blobname -> ilk -> toplevelnames}
for lang, tfifb in res_data.items():
dbfile_and_res_id_from_blobname = self.blob_index[lang]
for blobname, toplevelnames_from_ilk in tfifb.items():
# Update 'blob_index' for $lang.
dbfile, res_id = dbfile_and_res_id_from_blobname[blobname]
del dbfile_and_res_id_from_blobname[blobname]
# Remove ".blob" file (and associated caches).
pattern = join(self.base_dir, safe_lang_from_lang(lang),
dbfile+".*")
try:
for path in glob(pattern):
log.debug("fs-write: remove catalog %s blob file '%s'",
lang, basename(path))
os.remove(path)
except EnvironmentError as ex:
#XXX If get lots of these, then try harder. Perhaps
# creating a zombies area, or creating a list of
# them: self.db.add_zombie(dbpath).
#XXX THis isn't a correct analysis: the dbfile may just
# not have been there.
log.warn("could not remove dbfile '%s' (%s '%s'): "
"leaving zombie", dbpath, lang, blobname)
# Update 'toplevel*_index' for $lang.
# toplevelname_index: {lang -> ilk -> toplevelname -> res_id -> blobnames}
# toplevelprefix_index: {lang -> ilk -> prefix -> res_id -> toplevelnames}
for ilk, toplevelnames in six.iteritems(toplevelnames_from_ilk):
try:
bfrft = self.toplevelname_index[lang][ilk]
for toplevelname in toplevelnames:
del bfrft[toplevelname][res_id]
if not bfrft[toplevelname]:
del bfrft[toplevelname]
except KeyError as ex:
self.db.corruption("CatalogsZone._remove_res",
"error removing top-level names of ilk '%s' for "
"'%s' resource from toplevelname_index: %s"
% (ilk, basename(res.path), ex),
"ignore")
try:
tfrfp = self.toplevelprefix_index[lang][ilk]
for toplevelname in toplevelnames:
prefix = toplevelname[:LEN_PREFIX]
del tfrfp[prefix][res_id]
if not tfrfp[prefix]:
del tfrfp[prefix]
except KeyError as ex:
self.db.corruption("CatalogsZone._remove_res",
"error removing top-level name of ilk '%s' for "
"'%s' resource from toplevelprefix_index: %s"
% (ilk, basename(res.path), ex),
"ignore")
del self.res_index[res.area_path]
def mainFunc():
parser = argparse.ArgumentParser(description='Run Elastix registration protocol for all images in the directory')
parser.add_argument('--refDir', '-r', dest='refDir', required = True, \
help='The directory containing the reference images.')
parser.add_argument('--floatFile', '-f', dest='floatFile', required = True, \
help='Path to the floating image.')
parser.add_argument('--outDir', '-o', dest='outDir', required = False, \
help='Path to store the output images/parameters (default: current dir)', default=os.getcwd())
parser.add_argument('--atlas', '-a', dest='atlas', required = False, \
help='Path to the atlas segmentation file which will be resampled with the CPP file from the registration.', default=None)
args = parser.parse_args()
refImgs = [join(args.refDir, File) for File in listdir(args.refDir)]
refImgs = [img for img in refImgs if isfile(img) and img.endswith('.nii')]
if not refImgs:
print('Couldn\'t find any reference images')
return
if not path.isfile(args.floatFile):
print('Coudln\'t find the float image')
refImgs.sort(key=str.lower)
refFloatPairs = [[refImg, args.floatFile] for refImg in refImgs]
f3dParStr = paramListToShortString(f3d_params)
aladinParStr = paramListToShortString(aladin_params)
for rfPair in refFloatPairs:
baseName = basename(rfPair[0])[:-4]+'_'+basename(rfPair[1])[:-4]
currOutDir = join(args.outDir,baseName)
mkdir(currOutDir)
elastixLogPath = join(currOutDir,basename+'_LOG.txt')
elastixCommand = elastixExec+' -f '+rfPair[0]+' -m '+rfPair[1]+' -p '.join(elastixParams)+' -o '+currOutDir
elastixLog = ''
try:
elastixLog = ccall(elastixCommand, shell=True, stderr=STDOUT)
except CalledProcessError as err:
writeAndDie(err.output, elastixLogPath)
with open(elastixLogPath, 'w') as f:
f.write(elastixLog)
transformParameterFiles = ['TransformParameters.0.txt', 'TransformParameters.1.txt']
transformParameterFiles = [join(currOutDir,tpFile) for tpFile in transformParameterFiles]
for tpFilePath in transformParameterFiles:
with open(tpFilePath,'r') as tpFile:
tpCont = tpFile.read()
tpCont = tpCont.replace('(FinalBSplineInterpolationOrder 3)', '(FinalBSplineInterpolationOrder 1)')
with open(tpFilePath,'w') as tpFile:
tpCont = tpFile.write(tpCont)
if args.atlas is not None:
atlasOutDir = join(currOutDir,'atlas')
mkdir(atlasOutDir)
trfxCmd = trfxExec+' -in '+args.atlas+' -out '+atlasOutDir+' tp '+transformParameterFiles[-1]
try:
resampleLog = ccall(trfxCmd, shell=True, stderr=STDOUT)
except CalledProcessError as err:
writeAndDie(err.output, join(atlasOutDir,'ERR.txt'))
def generate(self):
self.resources.win_to_unix()
source_files = []
for r_type, n in CoIDE.FILE_TYPES.iteritems():
for file in getattr(self.resources, r_type):
source_files.append({"name": basename(file), "type": n, "path": file})
header_files = []
for r_type, n in CoIDE.FILE_TYPES2.iteritems():
for file in getattr(self.resources, r_type):
header_files.append({"name": basename(file), "type": n, "path": file})
libraries = []
for lib in self.resources.libraries:
l, _ = splitext(basename(lib))
libraries.append(l[3:])
if self.resources.linker_script is None:
self.resources.linker_script = ""
ctx = {
"name": self.program_name,
"source_files": source_files,
"header_files": header_files,
"include_paths": self.resources.inc_dirs,
"scatter_file": self.resources.linker_script,
"library_paths": self.resources.lib_dirs,
"object_files": self.resources.objects,
"libraries": libraries,
"symbols": self.get_symbols(),
}
target = self.target.lower()
# Project file
self.gen_file("coide_%s.coproj.tmpl" % target, ctx, "%s.coproj" % self.program_name)
def test_validate_demux_file_infer(self):
demux_fp, _, out_dir = self._generate_files({'s1': 'SKB2.640194',
's2': 'SKM4.640180'})
prep_info = {"1.SKB2.640194": {"not_a_run_prefix": "s1"},
"1.SKM4.640180": {"not_a_run_prefix": "s2"},
"1.SKB3.640195": {"not_a_run_prefix": "s3"},
"1.SKB6.640176": {"not_a_run_prefix": "s4"}}
files = {'preprocessed_demux': [demux_fp]}
job_id = self._create_template_and_job(
prep_info, files, "Demultiplexed")
obs_success, obs_ainfo, obs_error = _validate_demux_file(
self.qclient, job_id, prep_info, out_dir, demux_fp)
self.assertTrue(obs_success)
name = splitext(basename(demux_fp))[0]
exp_fastq_fp = join(out_dir, "%s.fastq" % name)
exp_fasta_fp = join(out_dir, "%s.fasta" % name)
exp_demux_fp = join(out_dir, basename(demux_fp))
filepaths = [
(exp_fastq_fp, 'preprocessed_fastq'),
(exp_fasta_fp, 'preprocessed_fasta'),
(exp_demux_fp, 'preprocessed_demux')]
exp = [ArtifactInfo(None, "Demultiplexed", filepaths)]
self.assertEqual(obs_ainfo, exp)
self.assertEqual(obs_error, "")
with File(exp_demux_fp) as f:
self.assertItemsEqual(f.keys(), ["1.SKB2.640194", "1.SKM4.640180"])
def rename_file(self, fname):
"""Rename file"""
path, valid = QInputDialog.getText(self, _("Rename"), _("New name:"), QLineEdit.Normal, osp.basename(fname))
if valid:
path = osp.join(osp.dirname(fname), to_text_string(path))
if path == fname:
return
if osp.exists(path):
if (
QMessageBox.warning(
self,
_("Rename"),
_("Do you really want to rename <b>%s</b> and " "overwrite the existing file <b>%s</b>?")
% (osp.basename(fname), osp.basename(path)),
QMessageBox.Yes | QMessageBox.No,
)
== QMessageBox.No
):
return
try:
misc.rename_file(fname, path)
self.parent_widget.renamed.emit(fname, path)
return path
except EnvironmentError as error:
QMessageBox.critical(
self,
_("Rename"),
_("<b>Unable to rename file <i>%s</i></b>" "<br><br>Error message:<br>%s")
% (osp.basename(fname), to_text_string(error)),
)
def append_deps_rootpath(dep_modules_roots, search_depth=10):
"""
Append all paths described in PackageInfo.dep_modules_roots into the sys.path,
so any module in the package can be called as a main entry,
with successfully importing dependent modules of dependent package in an easy scheme:
'import <identifiable_package_root>.<sub>.<target_module>',
Which could a more intuitive usage of module import.
Be sure this function is called for the main entry script with all outer dependent
package names in the parameter 'dep_modules_roots'
"""
check_path = ospath.dirname(ospath.abspath(__file__))
dep_dirs = []
dep_remains = list(dep_modules_roots)
for i in range(search_depth):
check_path = ospath.dirname(check_path)
check_name = ospath.basename(check_path)
for dep_name in dep_remains:
if dep_name == check_name:
dep_dirs.append(check_path)
dep_remains.remove(dep_name)
if not dep_remains:
break
if dep_dirs:
for dep_dir in dep_dirs:
sys.path.append(dep_dir)
print(BColors.BLUE
+ "Append path of package:'{pkg}'".format(pkg=ospath.basename(dep_dir))
+ " for package:'{name}' to sys.path as dependent modules root."
.format(name=PackageInfo.package_name)
+ BColors.ENDC)
return [name for name in dep_modules_roots if name not in dep_remains]
else:
return None
def _load_rbo(self):
"""Load APC2015rbo dataset"""
dataset_dir = osp.join(this_dir, 'dataset/APC2015rbo/berlin_samples')
img_glob = osp.join(dataset_dir, '*_bin_[A-L].jpg')
desc = 'rbo'
for img_file in tqdm.tqdm(glob.glob(img_glob), ncols=80, desc=desc):
basename = osp.splitext(osp.basename(img_file))[0]
# apply mask, crop and save
bin_mask_file = re.sub('.jpg$', '.pbm', img_file)
bin_mask = imread(bin_mask_file, mode='L')
where = np.argwhere(bin_mask)
roi = where.min(0), where.max(0) + 1
id_ = osp.join('rbo', basename)
dataset_index = len(self.ids) - 1
self.datasets['rbo'].append(dataset_index)
mask_glob = re.sub('.jpg$', '_*.pbm', img_file)
mask_files = [None] * self.n_class
for mask_file in glob.glob(mask_glob):
mask_basename = osp.splitext(osp.basename(mask_file))[0]
label_name = re.sub(basename + '_', '', mask_basename)
if label_name == 'shelf':
continue
mask_files[self.target_names.index(label_name)] = mask_file
self.ids.append(id_)
self.rois.append(roi)
self.img_files.append(img_file)
self.mask_files.append(mask_files)
def test_compare_triples():
for mime, fext in MIME_TYPES.items():
dump_path = path.join(DUMP_DIR, path.basename(mime))
for url in URLs:
if six.PY2:
fname = '%s.%s' % (path.basename(urlparse.urlparse(url).path), fext)
else:
fname = '%s.%s' % (path.basename(urlparse(url).path), fext)
fname = path.join(dump_path, fname)
req = Request(url)
req.add_header('Accept', mime)
res = urlopen(req)
g_fdp.parse(data=res.read(), format=mime)
g_dump.parse(fname, format=mime)
both, first, second = graph_diff(g_fdp, g_dump)
n_first = len(first)
# n_second = len(second)
# n_both = len(both)
assert_equals(
n_first, 0, '{} triple(s) different from reference:\n\n{}===\n{}\n'.format(
n_first, first.serialize(format='turtle'), second.serialize(format='turtle')))
def __init__(self, images, delivery_types=None):
"""
Parameters
----------
images : iterable (list, tuple, etc)
A sequence of paths to the image files.
delivery_types : iterable, None
If None (default), the image paths names must follow the `Naming Convention <http://pylinac.readthedocs.org/en/latest/vmat_docs.html#naming-convention>`_.
If the image paths do not follow the naming convention, a 2-element string sequence for ``delivery_types`` must be passed in. E.g. ``['open', 'dmlc']``.
"""
self.settings = Settings('', 1.5)
# error checks
if len(images) != 2:
raise ValueError("Exactly 2 images (open, DMLC) must be passed")
if delivery_types and len(delivery_types) != 2:
raise ValueError("Delivery types must be 2 elements long")
if delivery_types is None:
delivery_types = []
# walk over images and load the open and DMLC fields
for img, deliv in zip_longest(images, delivery_types, fillvalue=''):
if OPEN in img.lower() or OPEN == deliv.lower():
self.image_open = image.load(img)
elif DMLC in img.lower() or DMLC == deliv.lower():
self.image_dmlc = image.load(img)
else:
raise ValueError("Image file names must follow the naming convention (e.g. 'DRGS_open.dcm'), or the delivery types must be passed explicitly")
# try to determine test type
if all(DRGS in osp.basename(img).lower() for img in images):
self.settings.test_type = DRGS
elif all(DRMLC in osp.basename(img).lower() for img in images):
self.settings.test_type = DRMLC
def gmap_setup(gsnap_dir, out_dir, ref_fasta):
ref_base = op.splitext(op.basename(ref_fasta))[0]
ref_dir = op.dirname(ref_fasta)
ref_name = op.basename(ref_base)
# have to cd to the out_dir because gsnap writes to cwd.
cmd = "set -e\n cd %(ref_dir)s && \n"
cmd += "gmap_build"
cmd += " -k 12 -D %(ref_dir)s -d %(ref_base)s %(ref_fasta)s > %(out_dir)s/gmap_build.log && "
cmd += "\ncmetindex -d %(ref_base)s -F %(ref_dir)s > gmap_cmetindex.log 2> gmap_cmetindex.error.log"
cmd %= locals()
print >>sys.stderr, "[ command ] $", cmd
cmd_last = op.join(out_dir, "ran_gsnap_setup.sh")
rerun = False
if not op.exists(cmd_last) or not is_up_to_date_b(ref_fasta, cmd_last) or not is_same_cmd(cmd, cmd_last):
fh = open(cmd_last, "w")
print >>fh, cmd
fh.close()
rerun = True
elif is_up_to_date_b(ref_fasta, cmd_last) and not is_same_cmd(cmd, cmd_last):
fh = open(cmd_last, "w")
print >>fh, cmd
fh.close()
rerun = True
# TODO: check time-stamp
rerun = True
if rerun:
p = Popen(cmd.replace('\n', ' '), shell=True)
print >>sys.stderr, "^ executing gmap/gsnap setup^"
if p.wait() != 0:
pass
else:
print >>sys.stderr, "gsnap setup stuff is up to date, re-using"
return ref_base
dir1= os.getcwd()
for root, dirs, files in os.walk(dir1):
for file in files:
if file.endswith('.csv'):
#os.chdir(dir)
with open(file) as f:
csv_reader= csv.reader(f,delimiter=";")
print dir1
dir2 = dir1 + "\\" + basename(file)+"docs"
print dir2
os.mkdir(dir2)
os.chdir(dir2)
for row in csv_reader:
# urls=i
txt = []
#txt.append("url:")
#txt.append(str(row[0]))
#txt.append("\n")
try:
r = requests.get(row[0])
# Email: [email protected] % # Homepage: https://www.researchgate.net/profile/Thieu_Nguyen6 % # Github: https://github.com/thieunguyen5991 % # -------------------------------------------------------------------------------------------------------% from os.path import splitext, basename, realpath from sklearn.model_selection import ParameterGrid from models.main.traditional_rnn import Cnn1 from utils.IOUtil import _load_dataset__ from utils.Settings import * from utils.Settings import cnn1_final as param_grid if SPF_RUN_TIMES == 1: all_model_file_name = SPF_LOG_FILENAME else: # If runs with more than 1, like stability test --> name of the models ==> such as: rnn1hl.csv all_model_file_name = str(splitext(basename(realpath(__file__)))[0]) def train_model(item): root_base_paras = { "data_original": dataset, "train_split": SPF_TRAIN_SPLIT, # should use the same in all test "data_window": data_window, # same "scaling": SPF_SCALING, # minmax or std "feature_size": SPF_FEATURE_SIZE, # same, usually : 1 "network_type": SPF_3D_NETWORK, # RNN-based: 3D, others: 2D "n_runs": SPF_RUN_TIMES, # 1 or others "log_filename": all_model_file_name, "path_save_result": SPF_PATH_SAVE_BASE + SPF_DATA_FILENAME[loop] + "/", "draw": SPF_DRAW, "log": SPF_LOG
def get_object_id(self):
return basename(self.__path), self.get_current_version()
def main():
warnings.warn("This script is aimed to demonstrate how to convert the\n"
"JSON file to a single image dataset, and not to handle\n"
"multiple JSON files to generate a real-use dataset.")
parser = argparse.ArgumentParser()
parser.add_argument('json_file')
parser.add_argument('-o', '--out', default=None)
args = parser.parse_args()
json_file = args.json_file
alist = os.listdir(json_file)
for i in range(0, len(alist)):
path = os.path.join(json_file, alist[i])
data = json.load(open(path))
out_dir = osp.basename(path).replace('.', '_')
out_dir = osp.join(osp.dirname(path), out_dir)
if not osp.exists(out_dir):
os.mkdir(out_dir)
if data['imageData']:
imageData = data['imageData']
else:
imagePath = os.path.join(os.path.dirname(path), data['imagePath'])
with open(imagePath, 'rb') as f:
imageData = f.read()
imageData = base64.b64encode(imageData).decode('utf-8')
img = utils.img_b64_to_arr(imageData)
label_name_to_value = {'_background_': 0}
for shape in data['shapes']:
label_name = shape['label']
if label_name in label_name_to_value:
label_value = label_name_to_value[label_name]
else:
label_value = len(label_name_to_value)
label_name_to_value[label_name] = label_value
# label_values must be dense
label_values, label_names = [], []
for ln, lv in sorted(label_name_to_value.items(), key=lambda x: x[1]):
label_values.append(lv)
label_names.append(ln)
assert label_values == list(range(len(label_values)))
lbl = utils.shapes_to_label(img.shape, data['shapes'], label_name_to_value)
captions = ['{}: {}'.format(lv, ln)
for ln, lv in label_name_to_value.items()]
lbl_viz = utils.draw_label(lbl, img, captions)
PIL.Image.fromarray(img).save(osp.join(out_dir, 'img.png'))
utils.lblsave(osp.join(out_dir, 'label.png'), lbl)
PIL.Image.fromarray(lbl_viz).save(osp.join(out_dir, 'label_viz.png'))
with open(osp.join(out_dir, 'label_names.txt'), 'w') as f:
for lbl_name in label_names:
f.write(lbl_name + '\n')
warnings.warn('info.yaml is being replaced by label_names.txt')
info = dict(label_names=label_names)
with open(osp.join(out_dir, 'info.yaml'), 'w') as f:
yaml.safe_dump(info, f, default_flow_style=False)
print('Saved to: %s' % out_dir)
{
'dbtag':'Vb',
'unit':'V',
'description':'Battery voltage',
'lb':3.7,
'ub':5.5,
'interval':60*60,
},
{
'dbtag':'SoC',
'unit':'%',
'description':'State of Charge',
'lb':30, # more like a warning than a valid range check
'ub':100,
'interval':60*60,
},
]
if '__main__' == __name__:
for c in conf:
print('- - -')
for k, v in c.items():
print(k, ':' ,v)
import sys
sys.path.append('../..')
from os.path import basename
from storage.storage2 import create_table
create_table(conf, basename(__file__).split('.')[0].replace('_', '-'))
from os import listdir
from os.path import basename, dirname
# Import all the game managers
__all__ = list()
for f in listdir(dirname(__file__)):
if f[-3:] == '.py' and not f.endswith('__init__.py'):
__all__.append(basename(f)[:-3])
resampler = param
else:
log("Invalid value for resampler, ignoring")
elif argname in ("--out-sample-rate", "-out-sample-rate"):
if (isint(param)):
outsamplerate = int(param)
else:
log("Invalid value for output sample rate, ignoring")
elif argname in ("--filter-append", "-filter-append"):
filter_append = param
elif isdir(argname):
path = abspath(argname)
break
elif isfile(argname):
concatfile = realpath(argname)
convfile = "binaural_%s" % basename(concatfile)
wdir = dirname(concatfile)
doconcat = False
domakecue = False
dosplit = False
log("Enabling single file mode for '{filename}' (output filename: {convfile})"
.format(filename=argname, convfile=convfile))
else:
log("Unknown argument: %s" % arg)
if path is None:
path = abspath(".")
if (doconcat or dovolgain or dobconv) and not which(ffmpeg):
fatal("Wrong FFmpeg path: %s" % ffmpeg)
def primaryName(fileName):
return path.basename(fileName).split(".")[0]
if not line:
return None
return tuple(map(str.strip, line.split(';')))
def parse_file(file):
entries = [
entry for entry in (parse_entry(line) for line in file)
if entry is not None
]
return entries
ualtest = sys.argv[1]
test_path = sys.argv[2]
test_name = path.basename(test_path)
with open(test_path, 'r', encoding='utf-8') as file:
test_cases = parse_file(file)
def run_ualtest(text, *args):
# Run command.
command = [ualtest]
command.extend(args)
result = subprocess.run(command,
input=text,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
def getNameWithoutExtension(MovieNameWithExtension):
extLenght = len(path.splitext( path.basename(MovieNameWithExtension) )[1])
return MovieNameWithExtension[: -extLenght]
uid = '%s_%s_%s@git2ical' % \
(md5(author)[0:8], md5(day)[0:8], repo_hash[0:8])
add_event(summary, day, desc, uid)
else:
for day in daydata:
summary = u'%s: %d' % \
(repo_name, sum(map(len, daydata[day].itervalues())))
subjects = []
authors = daydata[day].keys()
authors.sort()
for author in authors:
subjects.extend(['', u'%s:' % author])
commits = daydata[day][author]
subjects.extend(map(lambda c: u'— %s' % c.subject, commits))
desc = u'\r\n'.join(subjects)
uid = '%s_%s@git2ical' % (md5(day)[0:8], repo_hash[0:8])
add_event(summary, day, desc, uid)
return cal
if __name__ == '__main__':
split_authors, last_days, repo, outfile = parse_cmdline()
repo_hash = md5(repo)
repo_name = basename(outfile).rsplit('.', 1)[0]
daydata = load_commits(repo, last_days)
cal = make_cal(daydata, split_authors, repo_name, repo_hash)
f = open(outfile, 'w')
f.write(cal.as_string())
f.close()
def run(config, MED_DATA_list):
bound_lick_counter = 0
color_counter = 0
plot_total_counter = []
bout_counter = 0
for name, info in config.experiments.items():
dir_name = path.join(config.path, info['folder'])
experiment_files = listdir(dir_name)
MED_DATA = [data for data in MED_DATA_list
if path.basename(data.file) in experiment_files]
if not MED_DATA:
continue
if 'raster' in info:
signal_in_trial = []
for data in MED_DATA:
res = raster(data, **info['raster'])
signal_in_trial.append(res / (len(data.Licks) if hasattr(data, 'Licks') else 1))
if 'raster psth' in info:
signal_in_trial = []
for data in MED_DATA:
res = raster(data, **info['raster psth'])
signal_in_trial.append(res / (len(data.Licks) if hasattr(data, 'Licks') else 1))
if 'ordered raster' in info:
m = len(MED_DATA)
n = max(len(info['ordered raster'].get('valve', 'Sucrose')) for data in MED_DATA)
label_x = info['ordered raster']['labels']
if isinstance(label_x, str):
label_x = [label_x]
assert len(label_x) == n, f"{len(label_x)} labels doesn't match the number of signals ({n})"
signal_per_conc = np.zeros(n, dtype=float)
for data in MED_DATA:
res = ordered_raster_PSTH(data, **info['ordered raster'])
# print(res)
if len(res) < n:
tmp = np.zeros(n, dtype=float)
# tmp[:len(res)] = res
res = tmp
signal_per_conc += res
signal_per_conc /= m
if info['ordered raster'].get('average_licks'):
plt.figure('Average rewarded licks')
plt.title('Average rewarded licks')
plt.bar(range(n), signal_per_conc, tick_label=label_x, color=background_color)
# plt.ylim([0, max(signal_per_conc)+5])
if 'trials_per_day' in info:
# sort MED_DATA by mouse in a dictionary
mouse_data = group_by_mouse(MED_DATA)
trials_per_day(mouse_data)
if 'close loop' in info:
mouse_data = group_by_mouse(MED_DATA)
bound_licks = {}
for mouse, data_list in mouse_data.items():
bound_licks[mouse] = {}
for data in data_list:
res = raster(data, **info['close loop'])
if hasattr(data, 'Licks'):
index = res / len(data.Licks)
else:
index = 0
bound_licks[mouse][data.start_date] = index
plt.figure(f'Bound Licks')
for mouse, licks_with_laser in bound_licks.items():
licks = [licks for date, licks in sorted(licks_with_laser.items())]
i = bound_lick_counter + 1
days = len(licks)
plt.plot(range(i, i + days), licks, marker='o', label=f'{mouse} {name}')
bound_lick_counter += days
plt.xlabel('Day')
plt.ylabel('Licks in laser period / Total licks')
plt.legend(loc=0)
plt.title('Bound Licks')
if 'open loop' in info:
mouse_data = group_by_mouse(MED_DATA)
bound_licks = {}
for mouse, data_list in mouse_data.items():
bound_licks[mouse] = {}
for data in data_list:
res = open_loop(data, **info['open loop'])
if hasattr(data, 'Licks'):
index = res / len(data.Licks)
else:
index = 0
bound_licks[mouse][data.start_date] = index
plt.figure(f'Bound Licks')
for mouse, licks_with_laser in bound_licks.items():
licks = [licks for date, licks in sorted(licks_with_laser.items())]
i = bound_lick_counter + 1
days = len(licks)
plt.plot(range(i, i + days), licks, marker='o', label=f'{mouse} {name}')
bound_lick_counter += days
plt.xlabel('Day')
plt.ylabel('Licks in laser period / Total licks')
plt.legend(loc=0)
plt.title('Bound Licks')
if 'open loop color task' in info:
mouse_data = group_by_mouse(MED_DATA)
bound_licks = {}
for mouse, data_list in mouse_data.items():
bound_licks[mouse] = {}
for data in data_list:
res = open_loop_color_task(data, **info['open loop color task'])
if hasattr(data, 'Licks'):
index = res / len(data.Licks)
else:
index = 0
bound_licks[mouse][data.start_date] = index
plt.figure(f'Bound Licks')
for mouse, licks_with_laser in bound_licks.items():
licks = [licks for date, licks in sorted(licks_with_laser.items())]
i = bound_lick_counter + 1
days = len(licks)
plt.plot(range(i, i + days), licks, marker='o', label=f'{mouse} {name}')
bound_lick_counter += days
plt.xlabel('Day')
plt.ylabel('Licks in laser period / Total licks')
plt.legend(loc=0)
plt.title('Bound Licks')
if 'spike raster psth' in info:
for data in MED_DATA:
spike_raster_PSTH(data, **info['spike raster psth'])
if 'plot' in info:
plot = info['plot']
if isinstance(plot, str):
plot = [plot]
for n, signal in enumerate(plot):
plt.figure(f'Total {signal}')
plt.title(MED_DATA[-1].subject)
label = name
# color = plot_colors[color_counter]
y = [len(getattr(data, signal)) if hasattr(data, signal) else 0
for data in MED_DATA]
try:
i = plot_total_counter[n]
except IndexError:
plot_total_counter.append(0)
i = 0
i += 1
days = len(y)
x = range(i, i + days)
plt.bar(x, y, label=label)
plt.xlabel('Day')
plt.ylabel(signal)
plt.legend()
# color_counter += 1
plot_total_counter[n] += days
if 'bouts' in info:
signal = info['bouts']
y = [get_bouts(getattr(data, signal)) if hasattr(data, signal) else 0
for data in MED_DATA]
plt.figure('Bouts')
plt.title(MED_DATA[-1].subject)
i = bout_counter + 1
days = len(y)
x = range(i, i + days)
plt.bar(x, y, label=name)
plt.xlabel('Day')
plt.ylabel(f'Bouts of {signal}')
plt.legend()
bout_counter += days
plt.show()
return
from os import listdir
from os.path import dirname, basename
# Automatically import every class inside the commands package. Magic! :D
__all__ = [basename(f)[:-3] for f in listdir(dirname(__file__)) if f[-3:] == ".py" and not f.endswith("__init__.py")]
def main():
args = parse_args()
cfg = Config.fromfile(args.config)
if args.options is not None:
cfg.merge_from_dict(args.options)
# set cudnn_benchmark
if cfg.get('cudnn_benchmark', False):
torch.backends.cudnn.benchmark = True
# work_dir is determined in this priority: CLI > segment in file > filename
if args.work_dir is not None:
# update configs according to CLI args if args.work_dir is not None
cfg.work_dir = args.work_dir
elif cfg.get('work_dir', None) is None:
# use config filename as default work_dir if cfg.work_dir is None
cfg.work_dir = osp.join('./work_dirs',
osp.splitext(osp.basename(args.config))[0])
if args.resume_from is not None:
cfg.resume_from = args.resume_from
if args.gpu_ids is not None:
cfg.gpu_ids = args.gpu_ids
else:
cfg.gpu_ids = range(1) if args.gpus is None else range(args.gpus)
if args.autoscale_lr:
# apply the linear scaling rule (https://arxiv.org/abs/1706.02677)
cfg.optimizer['lr'] = cfg.optimizer['lr'] * len(cfg.gpu_ids) / 8
# init distributed env first, since logger depends on the dist info.
if args.launcher == 'none':
distributed = False
else:
distributed = True
init_dist(args.launcher, **cfg.dist_params)
# create work_dir
mmcv.mkdir_or_exist(osp.abspath(cfg.work_dir))
# init the logger before other steps
timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime())
log_file = osp.join(cfg.work_dir, f'{timestamp}.log')
logger = get_root_logger(log_file=log_file, log_level=cfg.log_level)
# add a logging filter
logging_filter = logging.Filter('mmdet')
logging_filter.filter = lambda record: record.find('mmdet') != -1
# init the meta dict to record some important information such as
# environment info and seed, which will be logged
meta = dict()
# log env info
env_info_dict = collect_env()
env_info = '\n'.join([(f'{k}: {v}') for k, v in env_info_dict.items()])
dash_line = '-' * 60 + '\n'
logger.info('Environment info:\n' + dash_line + env_info + '\n' +
dash_line)
meta['env_info'] = env_info
# log some basic info
logger.info(f'Distributed training: {distributed}')
logger.info(f'Config:\n{cfg.pretty_text}')
# set random seeds
if args.seed is not None:
logger.info(f'Set random seed to {args.seed}, '
f'deterministic: {args.deterministic}')
set_random_seed(args.seed, deterministic=args.deterministic)
cfg.seed = args.seed
meta['seed'] = args.seed
model = build_detector(cfg.model,
train_cfg=cfg.get('train_cfg'),
test_cfg=cfg.get('test_cfg'))
logger.info(f'Model:\n{model}')
datasets = [build_dataset(cfg.data.train)]
if len(cfg.workflow) == 2:
val_dataset = copy.deepcopy(cfg.data.val)
val_dataset.pipeline = cfg.data.train.pipeline
datasets.append(build_dataset(val_dataset))
if cfg.checkpoint_config is not None:
# save mmdet version, config file content and class names in
# checkpoints as meta data
cfg.checkpoint_config.meta = dict(mmdet_version=__version__,
config=cfg.pretty_text,
CLASSES=datasets[0].CLASSES)
# add an attribute for visualization convenience
model.CLASSES = datasets[0].CLASSES
train_detector(model,
datasets,
cfg,
distributed=distributed,
validate=(not args.no_validate),
timestamp=timestamp,
meta=meta)
def test_type1_raises(path, tmpdir):
out = str(tmpdir / basename(path)) + ".out"
options = Options(path, out)
with pytest.raises(FontParseError):
hintFiles(options)
def create_package(self, filename, debug=False):
tmp_dir = temp_file.make_temp_dir(delete=not debug)
if debug:
print('tmp_dir: %s' % (tmp_dir))
stage_dir = path.join(tmp_dir, 'stage')
files_dir = path.join(stage_dir, 'files')
env_files_dir = path.join(stage_dir, 'env')
file_util.mkdir(files_dir)
file_util.mkdir(env_files_dir)
temp_file.write_temp_files(files_dir, self.files)
temp_file.write_temp_files(env_files_dir, self.env_files)
tmp_compiler_dir = path.join(tmp_dir, 'objects')
cc = compiler(build_target.make_host_build_target())
include_path = []
lib_path = []
static_c_libs = self.objects.get('static_c_libs', [])
for static_c_lib in static_c_libs:
sources, headers = static_c_lib.write_files(tmp_compiler_dir)
include_dir = path.join(tmp_compiler_dir, static_c_lib.filename,
'include')
lib_dir = path.join(tmp_compiler_dir, static_c_lib.filename)
include_path.append(include_dir)
lib_path.append(lib_dir)
cflags = ['-I%s' % (include_dir)]
targets = cc.compile_c([source.path for source in sources],
cflags=cflags)
lib_filename = path.join(tmp_compiler_dir, static_c_lib.filename,
path.basename(static_c_lib.filename))
lib = cc.make_static_lib(lib_filename,
[target.object for target in targets])
file_util.copy(lib, path.join(files_dir, static_c_lib.filename))
for header in headers:
file_util.copy(header.path,
path.join(files_dir, header.filename))
shared_c_libs = self.objects.get('shared_c_libs', [])
for shared_c_lib in shared_c_libs:
sources, headers = shared_c_lib.write_files(tmp_compiler_dir)
include_dir = path.join(tmp_compiler_dir, shared_c_lib.filename,
'include')
lib_dir = path.join(tmp_compiler_dir, shared_c_lib.filename)
include_path.append(include_dir)
lib_path.append(lib_dir)
cflags = ['-I%s' % (include_dir)]
targets = cc.compile_c([source.path for source in sources],
cflags=cflags)
lib_filename = path.join(tmp_compiler_dir, shared_c_lib.filename,
path.basename(shared_c_lib.filename))
lib = cc.make_shared_lib(lib_filename,
[target.object for target in targets])
file_util.copy(lib, path.join(files_dir, shared_c_lib.filename))
for header in headers:
file_util.copy(header.path,
path.join(files_dir, header.filename))
c_programs = self.objects.get('c_programs', [])
for c_program in c_programs:
sources, headers = c_program.write_files(tmp_compiler_dir)
include_dir = path.join(tmp_compiler_dir, c_program.filename,
'include')
lib_dir = path.join(tmp_compiler_dir, c_program.filename)
cflags = ['-I%s' % (include_dir)]
cflags += ['-I%s' % (inc) for inc in include_path]
ldflags = ['-L%s' % (lib_dir)]
ldflags += ['-L%s' % (lib) for lib in lib_path]
ldflags += c_program.ldflags or []
targets = cc.compile_c([source.path for source in sources],
cflags=cflags)
exe_filename = path.join(tmp_compiler_dir, c_program.filename,
path.basename(c_program.filename))
exe = cc.link_exe(exe_filename,
[target.object for target in targets],
ldflags=ldflags)
file_util.copy(exe, path.join(files_dir, c_program.filename))
pkg_desc = package_descriptor(self.metadata.name,
self.metadata.build_version,
properties=self.properties,
requirements=self.requirements)
return package.create_package(filename, pkg_desc,
self.metadata.build_target, stage_dir)
def test_counter_glyphs(tmpdir):
path = "%s/dummy/font.ufo" % DATA_DIR
out = str(tmpdir / basename(path)) + ".out"
options = Options(path, out)
options.vCounterGlyphs = ["m", "M", "T"]
hintFiles(options)
if args.uninstall:
call(["rm", "-f", join(ccls_install_dir, "bin", "ccls")])
chdir(homefolder)
# Install packages only if needed
for pac in packages_for_build:
if not apt_cache[pac].is_installed:
print("Needs to install packages for building CCLS")
call(["sudo", "apt-get", "install", "-y", *packages_for_build])
break
if args.clean:
rmtree(ccls_dir)
if args.build:
if not exists(ccls_dir):
call([
"git", "clone", "--depth=1", "--recursive",
"https://github.com/MaskRay/ccls",
basename(ccls_dir)
])
else:
call(["git", "-C", ccls_dir, "pull"])
chdir(ccls_dir)
call([
"cmake", "-H.", "-BRelease", "-DCMAKE_BUILD_TYPE=Release",
"-DCMAKE_INSTALL_PREFIX={}".format(ccls_install_dir)
])
call(["cmake", "--build", "Release", "--target", "install", "--parallel"])
from os.path import basename, dirname
from pybenzinaconcat.utils.jugutils import jug_main
def _main():
from pybenzinaconcat.index_metadata.index_metadata import main
main()
jug_main(_main, __file__, basename(dirname(__file__)))
def test_ufo(ufo, tmpdir):
out = str(tmpdir / basename(ufo))
options = Options(ufo, out)
hintFiles(options)
assert differ([ufo, out])
def copy_resources(self, *resources, use_symlinks=False):
for path in resources:
resource_dest = op.join(self.resources, op.basename(path))
action = symlink if use_symlinks else copy
action(op.abspath(path), resource_dest)
def test_round_trip(tmpdir):
path_config = op.join(this_folder, '..', 'book_template', '_config.yml')
path_out = op.join(tmpdir.dirpath(), 'tmp_test')
# Custom CSS and JS code
path_js = op.join(path_test_book, "my_js.js")
path_css = op.join(path_test_book, "my_css.css")
# Run the create command
new_name = "test"
new_book(path_out=op.join(path_out, new_name),
config=path_config,
toc=path_toc,
content_folder=path_content,
custom_js=path_js,
custom_css=path_css,
extra_files=[
op.join(path_test_book, 'foo', 'baz.txt'),
op.join(path_test_book, 'foo', 'you')
],
license=path_license)
# Table of contents
old_toc = read(path_toc)
new_toc = read(op.join(path_out, new_name, '_data', 'toc.yml'))
assert old_toc == new_toc
# Config files
with open(path_config, 'r') as ff:
old_config = yaml.load(ff)
with open(op.join(path_out, new_name, '_config.yml'), 'r') as ff:
new_config = yaml.load(ff)
for ii in old_config.keys():
if ii not in ["jupyter_book_version"]:
assert old_config[ii] == new_config[ii]
# License
old_license = read(path_license)
new_license = read(op.join(path_out, 'test', 'content', 'LICENSE.md'))
assert old_license == new_license
# Content
for ifolder, _, ifiles in os.walk(path_content):
for ifile in ifiles:
basename = op.basename(ifile)
# Only check the text files we care about since reading in other files is trickier
if 'LICENSE.md' in basename or all(
ii not in basename for ii in ['.md', '.ipynb', '.html']):
continue
old_content = read(op.join(ifolder, ifile))
new_content = read(
op.join(path_out, 'test', 'content', ifolder, basename))
assert old_content == new_content
# CSS and JS
assert file_contents_equal(
path_js, op.join(path_out, "test", "assets", "custom", "custom.js"))
assert file_contents_equal(
path_css, op.join(path_out, "test", "assets", "custom", "custom.css"))
# Extra files
assert op.exists(op.join(path_out, "test", "baz.txt"))
assert op.exists(op.join(path_out, "test", "you", "bar.txt"))
# This should raise an error because the folder exists now
with pytest.raises(CalledProcessError):
cmd = [
"jupyter-book", "create", new_name, "--config", path_config,
"--toc", path_toc, "--content-folder", path_content, "--license",
path_license, "--out-folder", path_out
]
run(cmd, check=True)
# If we succeed, remove the tmpdir
tmpdir.remove()
def open(init=None, extensions=None):
"""
Creates a DataModel from a number of different types
Parameters
----------
init : shape tuple, file path, file object, astropy.io.fits.HDUList, numpy array, dict, None
- None: A default data model with no shape
- shape tuple: Initialize with empty data of the given shape
- file path: Initialize from the given file (FITS , JSON or ASDF)
- readable file object: Initialize from the given file object
- astropy.io.fits.HDUList: Initialize from the given
`~astropy.io.fits.HDUList`
- A numpy array: A new model with the data array initialized
to what was passed in.
- dict: The object model tree for the data model
extensions : list of AsdfExtension
A list of extensions to the ASDF to support when reading
and writing ASDF files.
Results
-------
model : DataModel instance
"""
from astropy.io import fits
if init is None:
return DataModel(None)
# Send _asn.json files to ModelContainer; avoid shape "cleverness" below
elif (isinstance(init, six.string_types)
and basename(init).split('.')[0].split('_')[-1] == 'asn'):
try:
m = ModelContainer(init, extensions=extensions)
except:
raise TypeError("init ASN not valid for ModelContainer")
return m
elif isinstance(init, DataModel):
# Copy the object so it knows not to close here
return init.__class__(init)
elif isinstance(init, tuple):
for item in init:
if not isinstance(item, int):
raise ValueError("shape must be a tuple of ints")
shape = init
elif isinstance(init, np.ndarray):
shape = init.shape
else:
if isinstance(init, (six.text_type, bytes)) or hasattr(init, "read"):
hdulist = fits.open(init)
elif isinstance(init, fits.HDUList):
hdulist = init
else:
raise TypeError("init must be None, shape tuple, file path, "
"readable file object, or astropy.io.fits.HDUList")
shape = ()
try:
hdu = hdulist[(fits_header_name('SCI'), 1)]
except KeyError:
pass
else:
if hasattr(hdu, 'shape'):
shape = hdu.shape
# Here, we try to be clever about which type to
# return, otherwise, just return a new instance of the
# requested class
if len(shape) == 0:
new_class = DataModel
elif len(shape) == 4:
# It's a RampModel, MIRIRampModel, or QuadModel
try:
dqhdu = hdulist[fits_header_name('DQ')]
except KeyError:
# It's a RampModel or MIRIRampModel
try:
refouthdu = hdulist[fits_header_name('REFOUT')]
except KeyError:
# It's a RampModel
from . import ramp
new_class = ramp.RampModel
else:
# It's a MIRIRampModel
from . import miri_ramp
new_class = miri_ramp.MIRIRampModel
else:
# It's a QuadModel
from . import quad
new_class = quad.QuadModel
elif len(shape) == 3:
# It's a CubeModel
from . import cube
new_class = cube.CubeModel
elif len(shape) == 2:
try:
hdu = hdulist[(fits_header_name('SCI'), 2)]
except (KeyError, NameError):
# It's an ImageModel
from . import image
new_class = image.ImageModel
else:
# It's a MultiSlitModel
from . import multislit
new_class = multislit.MultiSlitModel
else:
raise ValueError("Don't have a DataModel class to match the shape")
return new_class(init, extensions=extensions)
def copy_headers(self, *headers, use_symlinks=False):
for path in headers:
header_dest = op.join(self.headers, op.basename(path))
action = symlink if use_symlinks else copy
action(op.abspath(path), header_dest)
"""
Atomate GUI
"""
from os.path import dirname, basename, isfile, join
import glob
modules = glob.glob(join(dirname(__file__), "*.py"))
__all__ = [
basename(f)[:-3] for f in modules
if isfile(f) and not f.endswith('__init__.py')
]
__version__ = "0.1"
def copy_frameworks(self, *frameworks):
for path in frameworks:
framework_dest = op.join(self.frameworks, op.basename(path))
copy(path, framework_dest)
from os.path import dirname, basename, isfile
import glob
modules = glob.glob(dirname(__file__) + "/*.py")
__all__ = [basename(f)[:-3] for f in modules if isfile(f)
and not f.endswith('__init__.py')]
def start(self, wdir=None, args=None, pythonpath=None):
filename = to_text_string(self.filecombo.currentText())
if wdir is None:
wdir = self._last_wdir
if wdir is None:
wdir = osp.basename(filename)
if args is None:
args = self._last_args
if args is None:
args = []
if pythonpath is None:
pythonpath = self._last_pythonpath
self._last_wdir = wdir
self._last_args = args
self._last_pythonpath = pythonpath
self.datelabel.setText(_('Profiling, please wait...'))
self.process = QProcess(self)
self.process.setProcessChannelMode(QProcess.SeparateChannels)
self.process.setWorkingDirectory(wdir)
self.process.readyReadStandardOutput.connect(self.read_output)
self.process.readyReadStandardError.connect(
lambda: self.read_output(error=True))
self.process.finished.connect(
lambda ec, es=QProcess.ExitStatus: self.finished(ec, es))
self.stop_button.clicked.connect(self.process.kill)
if pythonpath is not None:
env = [
to_text_string(_pth)
for _pth in self.process.systemEnvironment()
]
add_pathlist_to_PYTHONPATH(env, pythonpath)
processEnvironment = QProcessEnvironment()
for envItem in env:
envName, separator, envValue = envItem.partition('=')
processEnvironment.insert(envName, envValue)
self.process.setProcessEnvironment(processEnvironment)
self.output = ''
self.error_output = ''
p_args = ['-m', 'cProfile', '-o', self.DATAPATH]
if os.name == 'nt':
# On Windows, one has to replace backslashes by slashes to avoid
# confusion with escape characters (otherwise, for example, '\t'
# will be interpreted as a tabulation):
p_args.append(osp.normpath(filename).replace(os.sep, '/'))
else:
p_args.append(filename)
if args:
p_args.extend(shell_split(args))
executable = sys.executable
if executable.endswith("spyder.exe"):
# py2exe distribution
executable = "python.exe"
self.process.start(executable, p_args)
running = self.process.waitForStarted()
self.set_running_state(running)
if not running:
QMessageBox.critical(self, _("Error"),
_("Process failed to start"))
def load_datas(rots=None,add_rand=None,add_crop=None,eps=0.1,sample=None,norm=False,**kwargs):
from glob import glob
from PIL import Image
if(rots is None):
rots=[0,90,180,270]
poss=list(glob(path.join(curpth,'datas','pos','*')))
negs=list(glob(path.join(curpth,'datas','neg','*')))
xs=[]
ys=[]
#for i in random.sample(list(poss),70):
allimg=[(i,[eps,1-eps]) for i in poss]+[(j,[1-eps,eps]) for j in negs]
az=list(range(len(allimg)))
#random.shuffle(az)
allimg.sort(key=lambda x:base32(x[0],length=3))
le=len(az)
data=az[le//15:]
test=az[:le//15]
if(sample is not None):
data=random.sample(data,sample)
_xdata=[]
_ydata=[]
_xtest=[]
_ytest=[]
S=dict()
siz=kwargs.get('img_siz',48)
prog=0
tm=time.time()
ims=kwargs.get("img_siz",img_siz)
tot=len(data)*len(rots)+len(test)
for pth,y in [allimg[i] for i in data]:
try:
prog+=len(rots)
print('load%.2f%%,remaining%.1fsec'%(prog*100/tot,(time.time()-tm)/prog*(tot-prog)),end='\r')
im=None
#h=myhash.phashi(im,w=40)
#f=open(pth,'rb')
h=path.basename(path.dirname(pth))+base32(pth,length=16)
#f.close()
if(h in S):
print(pth,'==',S[h])
try:
im.close()
os.remove(pth)
im=Image.open(S[h])
except Exception as e:
print(e)
pass
continue
S[h]=pth
for r in rots:
#azz=[im1]
svname="file=%s,rot=%d,size=%d.npy"%(h,r,ims)
svpth=path.join(curpth,'npy',svname)
if(path.exists(svpth)):
data=np.load(svpth)
else:
if(im is None):
im=Image.open(pth)
im1=im.rotate(r,expand=True)
data=img2input(im1,**kwargs)
np.save(svpth,data)
'''if(add_crop is None):
else:
azz=[im1]+crop_img(im1,add_crop)'''
_xdata.append(data)
_ydata.append(y)
except Exception as e:
print(pth,e)
#traceback.print_exc()
continue
#for i in random.sample(list(negs),70):
for pth,y in [allimg[i] for i in test]:
try:
prog+=1
print('load%.2f%%,remaining%.1fsec'%(prog*100/tot,(time.time()-tm)/prog*(tot-prog)),end='\r')
h=path.basename(path.dirname(pth))+base32(pth,length=16)
svname="file=%s,rot=%d,size=%d.npy"%(h,0,ims)
svpth=path.join(curpth,'npy',svname)
if(path.exists(svpth)):
data=np.load(svpth)
else:
im=Image.open(pth)
data=img2input(im,**kwargs)
np.save(svpth,data)
_xtest.append(data)
_ytest.append(y)
except Exception as e:
print(pth,e)
if(add_rand is not None):
_xdata=np.array(_xdata,np.float32)
az=_xdata+(np.random.rand(*_xdata.shape)-0.5)*add_rand
_xdata=np.concatenate((_xdata,az),axis=0)
_ydata=np.array(_ydata)
_ydata=np.concatenate((_ydata,_ydata),axis=0)
if(norm):
_xdata=(np.array(_xdata,np.float16)-127.5)/127.5
_xtest=(np.array(_xtest,np.float16)-127.5)/127.5
if(not isinstance(_xdata,np.ndarray)):
_xdata=np.array(_xdata,np.float16)
return _xdata,np.array(_ydata),np.array(_xtest),np.array(_ytest)
def header(path):
test_name = osp.basename(path)
print(
"\n".join(("\n", "*" * 40, "Starting %s test in %s" % (test_name, path), "*" * 40)),
flush=True,
)
