def read_results(dirnames):
results = defaultdict(dict)
for dirname in dirnames:
pattern = joinpath(dirname, "results", "translate",
"*", "*", "translate.log")
for result_file in glob(pattern):
runtime = read_runtime(result_file)
result_dir, _ = splitpath(result_file)
domain_dir, problem = splitpath(result_dir)
_, domain = splitpath(domain_dir)
task = "%s:%s" % (domain, problem)
results[task][dirname] = runtime
return results
def read_results(dirnames):
results = defaultdict(dict)
for dirname in dirnames:
pattern = joinpath(dirname, "results", "translate", "*", "*",
"translate.log")
for result_file in glob(pattern):
runtime = read_runtime(result_file)
result_dir, _ = splitpath(result_file)
domain_dir, problem = splitpath(result_dir)
_, domain = splitpath(domain_dir)
task = "%s:%s" % (domain, problem)
results[task][dirname] = runtime
return results
def complete_files(self, text, state):
str_delim = text[0]
path = text[1:]
if path.startswith("~/"):
path = expanduser("~/") + path[2:]
elif path.startswith("~"):
i = path.find(pathsep)
if i > 0:
path = expanduser(path[:i]) + path[i:]
else:
return [
str_delim + "~" + i[0] + pathsep
for i in getpwall()
if i[0].startswith(path[1:])
][state]
dir, fname = splitpath(path)
if not dir:
dir = os.curdir
return [
str_delim + joinpath(dir, i)
for i in os.listdir(dir)
if i.startswith(fname)
][state]
def dll_path(info):
"""
Path to the compiled model defined by *info*.
"""
from os.path import join as joinpath, split as splitpath, splitext
basename = splitext(splitpath(info['filename'])[1])[0]
return joinpath(DLL_PATH, basename+'.so')
def splitfullpath(path):
if len(path) == 0:
return list()
head, tail = splitpath(path)
base = splitfullpath(head)
base.append(tail)
return base
def sftp(self, local_path, remote_path, mkdir=True):
# from https://gist.github.com/johnfink8/2190472
oldcwd = os.getcwd()
sftp = self.client.open_sftp()
try:
remote_path = PurePosixPath(remote_path)
parent, child = splitpath(local_path)
os.chdir(parent)
for d, _, files in os.walk(child):
d = PurePath(d)
try:
remote_dir = remote_path / d
print("make", remote_dir)
if not mkdir:
# skip first mkdir
mkdir = True
else:
sftp.mkdir(str(remote_dir))
except:
raise
for fname in files:
local_fname = d / fname
remote_fname = remote_dir / fname
print(local_fname.relative_to(child), "->", remote_fname)
sftp.put(str(local_fname), str(remote_fname))
finally:
os.chdir(oldcwd)
sftp.close()
def isinpath(program: str) -> bool:
"""Test if the specified application is in the system PATH
>>> isinpath('sh')
True
>>> isinpath('env')
True
>>> isinpath('firefox')
True
>>> isinpath('FIREFOX')
False
>>> isinpath('not_in_path')
False
>>> isinpath('/usr/bin')
False
>>> isinpath('/usr/bin/')
False
>>> isinpath('/usr')
False
"""
fpath = splitpath(program)[0]
if fpath and isfile(program) and access(program, X_OK):
return True
envpath = environ[r'PATH'].split(pathsep)
for filepath in envpath:
exe_file = joinpath(filepath.strip(r'"'), program)
if isfile(exe_file) and access(exe_file, X_OK):
return True
return False
def splitfullpath(path):
if len(path) == 0:
return list()
head, tail = splitpath(path)
base = splitfullpath(head)
base.append(tail)
return base
def __call__(self, oldPath):
head, tail = splitpath(oldPath)
headParts = head.split(sep)
if not headParts:
raise ValueError('Directory part is empty for entry "%s"' %
oldPath)
headParts[0] = self.newName
return sep.join(headParts + [tail])
def make_observer(file_name, handler):
full_path = abspath(file_name)
folder, _ = splitpath(full_path)
observer = Observer()
observer.schedule(handler, folder, False)
observer.daemon = True
return observer
def make_observer(file_name, handler):
full_path = abspath(file_name)
folder, _ = splitpath(full_path)
observer = Observer()
observer.schedule(handler, folder, False)
observer.daemon = True
return observer
def __init__(self, maildir, filesystem=None):
self.logger = logging.getLogger("MdClient.%s" % maildir)
foldername = maildir.split("/")[-1]
base = splitpath(maildir)[0]
self.folder = MdFolder(
foldername if foldername else "", base=base,
filesystem=filesystem) if filesystem else MdFolder(
foldername if foldername else "", base=base)
def __init__(self, here, open, joinpath, splitpath):
'''
:param str here:
'''
make = lambda p: Path(p, open=open, joinpath=joinpath, splitpath=splitpath)
self.joinpath = lambda there: make(joinpath(here, there))
self.open = lambda **kwargs: open(here, **kwargs)
self._parent = lambda: make(splitpath(here)[0])
self._path = here
def __call__(self, oldPath): head, tail = splitpath(oldPath) headParts = head.split(sep) if not headParts: raise ValueError( 'Directory part is empty for entry "%s"' % oldPath ) headParts[0] = self.newName return sep.join(headParts + [ tail ])
def __init__(self, maildir, filesystem=None):
self.logger = logging.getLogger("MdClient.%s" % maildir)
foldername = maildir.split("/")[-1]
base = splitpath(maildir)[0]
self.folder = MdFolder(
foldername if foldername else "",
base=base,
filesystem=filesystem
) if filesystem else MdFolder(foldername if foldername else "",
base=base
)
def produce_tex(template, filename, context, compile=True):
"""Генерирует новый .tex-файл РПД, подставляя данные контекста в шаблон.
Если compile==True, вызывает xelatex на полученном файле"""
fn = splitpath(filename)[1]
tex_file = splitext(fn)[0] + '.tex'
output = open(tex_file, 'w', encoding='utf-8')
output.write(template.render(**context))
output.close()
if compile:
sleep(0.5)
execute(['xelatex', '--synctex=1', tex_file])
def load(cls, filename: str):
"""
Restore the schedule object from the json file
:param filename: The file to read the data from
:return: Schedule
"""
with open(filename, "rb") as file:
tables_dict: dict[str, dict] = json.load(file)
grade: str = splitext(splitpath(filename)[1])[0]
tables = dict()
for name, table in tables_dict.items():
tables[name] = pd.DataFrame.from_records(table['data'],
columns=table['columns'])
return cls(grade, tables)
def dll_path(model_info, dtype="double"):
"""
Path to the compiled model defined by *model_info*.
"""
basename = splitext(splitpath(model_info['filename'])[1])[0]
if np.dtype(dtype) == generate.F32:
basename += "32"
elif np.dtype(dtype) == generate.F64:
basename += "64"
else:
basename += "128"
# Hack to find precompiled dlls
path = joinpath(generate.DATA_PATH, '..', 'compiled_models', basename+'.so')
if os.path.exists(path):
return path
return joinpath(DLL_PATH, basename+'.so')
def dll_path(model_info, dtype="double"):
"""
Path to the compiled model defined by *model_info*.
"""
basename = splitext(splitpath(model_info['filename'])[1])[0]
if np.dtype(dtype) == generate.F32:
basename += "32"
elif np.dtype(dtype) == generate.F64:
basename += "64"
else:
basename += "128"
basename += ARCH + ".so"
# Hack to find precompiled dlls
path = joinpath(generate.DATA_PATH, '..', 'compiled_models', basename)
if os.path.exists(path):
return path
return joinpath(DLL_PATH, basename)
def checklockacct():
#Check if user should be unlocked and unlock them
logkpr('checklockacct called')
#Find *.lock in VAR['TIMEKPRDIR']
s = VAR['TIMEKPRDIR'] + '/*.lock'
l = glob(s)
for f in l:
#Get username from filename - os.path.split
u = splitpath(f)[1].replace('.lock', '')
lastmodified = getmtime(f) #Get last modified time from username.lock file
#Get time when lock should be lifted
dtlock = float(lastmodified + getlocklasts())
dtnow = float(timenow())
#If time now is great than or equal to the time when lock should be lifted
if dtnow >= dtlock:
logkpr('checklockacct: %s should be unlocked, unlocking..' % u)
unlockuser(u)
logkpr('checklockacct: removing %s.lock file..' % u)
remove(f)
def checklockacct():
#Check if user should be unlocked and unlock them
logkpr('checklockacct called')
#Find *.lock in VAR['TIMEKPRDIR']
s = VAR['TIMEKPRDIR'] + '/*.lock'
l = glob(s)
for f in l:
#Get username from filename - os.path.split
u = splitpath(f)[1].replace('.lock', '')
lastmodified = getmtime(f) #Get last modified time from username.lock file
#Get time when lock should be lifted
dtlock = float(lastmodified + getlocklasts())
dtnow = float(timenow())
#If time now is great than or equal to the time when lock should be lifted
if dtnow >= dtlock:
logkpr('checklockacct: %s should be unlocked, unlocking..' % u)
unlockuser(u)
logkpr('checklockacct: removing %s.lock file..' % u)
remove(f)
def __init__(self, uri):
pr = urlparse(uri.replace("\\", "/"))
self.scheme = pr.scheme if pr.scheme is not None else ""
self.user = pr.username if pr.username is not None else ""
self.password = pr.password if pr.password is not None else ""
self.server = pr.hostname if pr.hostname is not None else ""
self.port = str(pr.port) if pr.port is not None else ""
self.dirpath, fullfile = splitpath(pr.path)
if self.dirpath is None:
self.dirpath = ""
self.file, ext = splitext(fullfile)
if self.file is None:
self.file = ""
if len(ext) > 0:
self.ext = ext[1:]
else:
self.ext = ""
self.query = RestUri.parse_args(pr.query)
if self.query is None:
self.query = {}
self.frag = pr.fragment if pr.fragment is not None else ""
def _isPackagePath(fpath):
# Determine if a FilePath-like object is a Python package. TODO: deal with
# __init__module.(so|dll|pyd)?
extless = fpath.splitext()[0]
basend = splitpath(extless)[1]
return basend == "__init__"
sys.exit(1)
playlist = open(playlist_path, 'r')
lines = playlist.readlines()
track_list = []
for l in lines:
l = l.strip()
# Ignore comments and empty lines
if not l.startswith('#') and len(l) > 0:
# Check that the file is real
track_list.append(l)
playlist.close()
# Proceed each track
for track in track_list:
file_type = splitext(track)[1].lower()
(file_path, file_name) = splitpath(track)
file_name = file_name.split(file_type)[0]
file_type = file_type[1:]
# MP3s are just copied as is
if file_type == 'mp3':
# Download (copy) the track
run(""" cp "%s" ./""" % track)
print " DOWNLOADED - %s" % track
# FLACs are re-encoded as AAC
elif file_type == 'flac':
wave_file = abspath("./%s.wav" % file_name)
run("""flac --output-name="%s" --decode "%s" """ % (wave_file, track))
print " DECODED - %s" % track
run("""faac -q 80 "%s" """ % wave_file)
import os
import sys
import xmlrunner
import unittest
import imp
from os.path import abspath, dirname, split as splitpath, join as joinpath
import logging
logger = logging.getLogger(__name__)
if not logger.root.handlers:
import logging.config
LOGGER_CONFIG_FILE = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'logging.ini')
logging.config.fileConfig(LOGGER_CONFIG_FILE, disable_existing_loggers=False)
if len(sys.argv) < 2:
logger.error("Use %s <filename to test>",sys.argv[0])
sys.exit(-1)
run_py = joinpath(dirname(dirname(abspath(__file__))), 'run.py')
run = imp.load_source('sasview_run', run_py)
run.prepare()
#print "\n".join(sys.path)
test_path,test_file = splitpath(abspath(sys.argv[1]))
print("=== testing:",sys.argv[1])
#print test_path, test_file
sys.argv = [sys.argv[0]]
os.chdir(test_path)
sys.path.insert(0, test_path)
test = imp.load_source('tests',test_file)
unittest.main(test, testRunner=xmlrunner.XMLTestRunner(output='logs'))
#!/usr/bin/env python3
from os import remove
from os.path import abspath, splitext, split as splitpath
from shutil import rmtree
from subprocess import call
from sys import argv, executable as pythonexe
py_filename = argv[1].replace('\\','/')
path,fname = splitpath(py_filename)
name,ext = splitext(fname)
is_console = (ext != '.pyw')
s = open('export_exe.spec','r').read()
s = s.format(py_filename=py_filename, name=name, is_console=is_console)
open('_export_exe.spec','w').write(s)
try:
rmtree('dist')
except:
pass
call(pythonexe +' ../python35/Scripts/pyinstaller-script.py _export_exe.spec')
remove('_export_exe.spec')
rmtree('build')
call('explorer '+abspath('dist'), shell=True)
print('EXE export done.')
def _make_filename(item):
src = item.content['src']
ext = splitext(urlparse(src).path)[1]
return splitpath(item.url)[1] + ext
def yieldFragment(self):
# List that stores the components we've added
components = []
# Stack that stores directories we've already visited
stack = []
# List that stores the virtual directories we added
virtualstack = []
yield self.indent(
'<?xml version="1.0" encoding="utf-8"?>'
)
yield self.startElement(
'Wix', xmlns = 'http://schemas.microsoft.com/wix/2006/wi'
)
yield self.startElement('Fragment')
yield self.startElement('DirectoryRef', Id = self.directoryRef)
# Add virtual directories
if self.virtualDir:
head = self.virtualDir
while head:
joinedPath = head
head, tail_ = splitpath(head)
virtualstack.insert(0, joinedPath)
for path in virtualstack:
componentId = 'directory_' + str(uuid4()).replace('-', '_')
yield self.startElement(
'Directory', Id = componentId, Name = basename(path)
)
# Walk the provided file list
firstDir = True
for dirpath, dirnames, filenames in self.fileGenerator:
# Remove excluded sub-directories
for exclusion in excludedDirectories:
if exclusion in dirnames:
dirnames.remove(exclusion)
if firstDir:
firstDir = False
else:
# Handle directory hierarchy appropriately
while stack:
popped = stack.pop()
if isParentDir(dirpath, popped):
stack.append(popped)
break
else:
yield self.endElement('Directory')
# Enter new directory
stack.append(dirpath)
yield self.startElement(
'Directory', Id = makeDirectoryId(newGuid()),
Name = basename(dirpath)
)
# Remove excluded files
for exclusion in excludedFiles:
if exclusion in filenames:
filenames.remove(exclusion)
# Process files
for filename in filenames:
guid = newGuid()
componentId = makeComponentId(guid)
sourcePath = joinpath(dirpath, filename)
yield self.startElement(
'Component', Id = componentId, Guid = guid,
DiskId = '1', Win64 = self.win64
)
yield self.startElement(
'File', Id = makeFileId(guid), Name = filename,
Source = sourcePath
)
yield self.endElement('File')
yield self.endElement('Component')
components.append(componentId)
# Drain pushed physical directories
while stack:
popped = stack.pop()
yield self.endElement('Directory')
# Drain pushed virtual directories
while virtualstack:
popped = virtualstack.pop()
yield self.endElement('Directory')
yield self.endElement('DirectoryRef')
yield self.endElement('Fragment')
# Emit ComponentGroup
yield self.startElement('Fragment')
yield self.startElement('ComponentGroup', Id = self.componentGroup)
for component in components:
yield self.startElement('ComponentRef', Id = component)
yield self.endElement('ComponentRef')
yield self.endElement('ComponentGroup')
yield self.endElement('Fragment')
yield self.endElement('Wix')
#!/usr/bin/env python
import os
import sys
import xmlrunner
import unittest
import imp
from os.path import abspath, dirname, split as splitpath, join as joinpath
run_py = joinpath(dirname(dirname(abspath(__file__))), 'run.py')
run = imp.load_source('sasview_run', run_py)
run.prepare()
#print "\n".join(sys.path)
test_path, test_file = splitpath(abspath(sys.argv[1]))
print "=== testing:", sys.argv[1]
#print test_path, test_file
sys.argv = [sys.argv[0]]
os.chdir(test_path)
sys.path.insert(0, test_path)
test = imp.load_source('tests', test_file)
unittest.main(test, testRunner=xmlrunner.XMLTestRunner(output='logs'))
def render(tpl_path, context):
path, filename = splitpath(tpl_path)
return jinja2.Environment(loader=jinja2.FileSystemLoader(
path or './')).get_template(filename).render(context).replace(
u'\ufeff', '')
def yieldFragment(self):
# List that stores the components we've added
components = []
# Stack that stores directories we've already visited
stack = []
# List that stores the virtual directories we added
virtualstack = []
yield self.indent(
'<?xml version="1.0" encoding="utf-8"?>'
)
yield self.startElement(
'Wix', xmlns = 'http://schemas.microsoft.com/wix/2006/wi'
)
yield self.startElement('Fragment')
yield self.startElement('DirectoryRef', Id = self.directoryRef)
# Add virtual directories
if self.virtualDir:
head = self.virtualDir
while head:
joinedPath = head
head, tail_ = splitpath(head)
virtualstack.insert(0, joinedPath)
for path in virtualstack:
componentId = 'directory_' + str(uuid4()).replace('-', '_')
yield self.startElement(
'Directory', Id = componentId, Name = basename(path)
)
# Walk the provided file list
firstDir = True
for dirpath, dirnames, filenames in self.fileGenerator:
# Remove excluded sub-directories
for exclusion in excludedDirectories:
if exclusion in dirnames:
dirnames.remove(exclusion)
if firstDir:
firstDir = False
else:
# Handle directory hierarchy appropriately
while stack:
popped = stack.pop()
if isParentDir(dirpath, popped):
stack.append(popped)
break
else:
yield self.endElement('Directory')
# Enter new directory
stack.append(dirpath)
yield self.startElement(
'Directory', Id = makeDirectoryId(newGuid()),
Name = basename(dirpath)
)
# Remove excluded files
for exclusion in excludedFiles:
if exclusion in filenames:
filenames.remove(exclusion)
# Process files
for filename in filenames:
guid = newGuid()
componentId = makeComponentId(guid)
sourcePath = joinpath(dirpath, filename)
yield self.startElement(
'Component', Id = componentId, Guid = guid,
DiskId = '1', Win64 = self.win64
)
yield self.startElement(
'File', Id = makeFileId(guid), Name = filename,
Source = sourcePath
)
yield self.endElement('File')
yield self.endElement('Component')
components.append(componentId)
# Drain pushed physical directories
while stack:
popped = stack.pop()
yield self.endElement('Directory')
# Drain pushed virtual directories
while virtualstack:
popped = virtualstack.pop()
yield self.endElement('Directory')
yield self.endElement('DirectoryRef')
yield self.endElement('Fragment')
# Emit ComponentGroup
yield self.startElement('Fragment')
yield self.startElement('ComponentGroup', Id = self.componentGroup)
for component in components:
yield self.startElement('ComponentRef', Id = component)
yield self.endElement('ComponentRef')
yield self.endElement('ComponentGroup')
yield self.endElement('Fragment')
yield self.endElement('Wix')
def _isPackagePath(fpath):
# Determine if a FilePath-like object is a Python package. TODO: deal with
# __init__module.(so|dll|pyd)?
extless = fpath.splitext()[0]
basend = splitpath(extless)[1]
return basend == "__init__"
def goto(self, cmdl):
""" Moves current dictionary position and working directory (if appropriate). """
from os import chdir
from os.path import exists, join, split as splitpath, isdir
from .explore import explore
from . import get_shell
from pylada import interactive
shell = get_shell(self)
if interactive.jobfolder is None:
print "No current job-folders."
return
if len(cmdl.split()) == 0:
if interactive.jobfolder_path is None:
print "Current position in job folder:", interactive.jobfolder.name
else:
print "Current position in job folder:", interactive.jobfolder.name
print "Filename of job-folder: ", interactive.jobfolder_path
return
args = cmdl.split()
if len(args) > 1:
print "Invalid argument to goto {0}.".format(cmdl)
return
# if no argument, then print current job data.
if len(args) == 0:
explore(self, "")
return
# cases to send to iterate.
if args[0] == "next": return iterate(self, "")
elif args[0] == "previous": return iterate(self, "previous")
elif args[0] == "reset": return iterate(self, "reset")
# case for which precise location is given.
try:
result = interactive.jobfolder[args[0]]
except KeyError as e:
print e
return
interactive.jobfolder = result
if 'jobparams' in shell.user_ns:
shell.user_ns['jobparams'].view = interactive.jobfolder.name
if 'collect' in shell.user_ns:
shell.user_ns['collect'].view = interactive.jobfolder.name
good = not interactive.jobfolder.is_tagged
if good:
for value in interactive.jobfolder.values():
good = not value.is_tagged
if good: break
if not good:
print '**** Current job-folders and sub-folders are all off; ' \
'jobparams (except onoff) and collect will not work.'
return
if interactive.jobfolder_path is None: return
dir = join(
splitpath(interactive.jobfolder_path)[0],
interactive.jobfolder.name[1:])
if exists(dir): chdir(dir)
else:
print "In {0}, but no corresponding directory on disk." \
.format(interactive.jobfolder.name.split('/')[-2])
return
def goto(self, cmdl):
""" Moves current dictionary position and working directory (if appropriate). """
from os import chdir
from os.path import exists, join, split as splitpath, isdir
from .explore import explore
from . import get_shell
from pylada import interactive
shell = get_shell(self)
if interactive.jobfolder is None:
print "No current job-folders."
return
if len(cmdl.split()) == 0:
if interactive.jobfolder_path is None:
print "Current position in job folder:", interactive.jobfolder.name
else:
print "Current position in job folder:", interactive.jobfolder.name
print "Filename of job-folder: ", interactive.jobfolder_path
return
args = cmdl.split()
if len(args) > 1:
print "Invalid argument to goto {0}.".format(cmdl)
return
# if no argument, then print current job data.
if len(args) == 0:
explore(self, "")
return
# cases to send to iterate.
if args[0] == "next": return iterate(self, "")
elif args[0] == "previous": return iterate(self, "previous")
elif args[0] == "reset": return iterate(self, "reset")
# case for which precise location is given.
try: result = interactive.jobfolder[args[0]]
except KeyError as e:
print e
return
interactive.jobfolder = result
if 'jobparams' in shell.user_ns:
shell.user_ns['jobparams'].view = interactive.jobfolder.name
if 'collect' in shell.user_ns:
shell.user_ns['collect'].view = interactive.jobfolder.name
good = not interactive.jobfolder.is_tagged
if good:
for value in interactive.jobfolder.values():
good = not value.is_tagged
if good: break
if not good:
print '**** Current job-folders and sub-folders are all off; ' \
'jobparams (except onoff) and collect will not work.'
return
if interactive.jobfolder_path is None: return
dir = join( splitpath(interactive.jobfolder_path)[0],
interactive.jobfolder.name[1:] )
if exists(dir): chdir(dir)
else:
print "In {0}, but no corresponding directory on disk." \
.format(interactive.jobfolder.name.split('/')[-2])
return
def _split_path(fpath):
fpath = realpath(fpath)
return splitpath(fpath)