def compareDirectories(new, old, relpath = ""):
log("XXX new " + new + " old "+old + " relpath "+relpath)
newroot, newdirs, newfiles = next(os.walk(new))
oldroot, olddirs, oldfiles = next(os.walk(old))
log("new dirs: " + ",".join(newdirs))
all_dirs = list(set(newdirs) | set(olddirs))
log("all dirs: " + ",".join(all_dirs))
common_dirs = list(set(newdirs) & set(olddirs))
log("common dirs: "+ ",".join(common_dirs))
missing_new_dirs = list(set(newdirs) - set(common_dirs))
log("missing new dirs: "+ ",".join(missing_new_dirs))
compareFiles(newroot,oldroot,newfiles, oldfiles, relpath)
for missing_dir in missing_new_dirs:
full_path = os.path.join(diff_root,relpath)
full_path = os.path.join(full_path,missing_dir)
log("full " + full_path)
mkpath(full_path)
src = os.path.join(newroot,missing_dir)
log("missing dir src "+src)
copy_tree(src, full_path)
for common_dir in common_dirs:
next_new_dir = os.path.join(newroot,common_dir)
next_old_dir = os.path.join(oldroot,common_dir)
newrelpath = os.path.join(relpath,common_dir);
log("newrelpath "+newrelpath)
compareDirectories(next_new_dir, next_old_dir, newrelpath)
def createMoFile(self):
try:
mkpath(self.LOCALEDIR)
copy_file(self.POFILEPATH, self.LOCALEDIR)
cwd = os.getcwd()
os.chdir(self.LOCALEDIR)
if self.USE_MSGFMT_BINARY:
# The msgfmt binary that ships as part of GNU gettext tools
# is more robust then the Python version and includes
# error checking capabilities.
moFile = self.POFILE[:-2] + "mo"
exp = ["msgfmt", "-c", "--check-accelerators", "-o%s" % moFile,
self.POFILE]
else:
# The msgfmt gettext binary is not installed by default on
# Windows and OS X. The Python version of msgfmt is included
# however with Chandler.
msgfmt = os.path.join(self.CHANDLERHOME, "tools", "msgfmt.py")
exp = [self.PYTHON, msgfmt, self.POFILE]
result = build_lib.runCommand(exp, timeout=TIMEOUT, logger=ignore, ignorepreexec=True)
os.chdir(cwd)
if result != 0:
raise Exception(' '.join(exp) + ' failed with error code %d' % result)
except Exception, e:
self.raiseError("Unable to create mo file from %s': %s." % (self.POFILEPATH, e))
def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True,
mode='w'):
"""Create a zip file from all the files under 'base_dir'. The output
zip file will be named 'base_dir' + ".zip". Uses either the "zipfile"
Python module (if available) or the InfoZIP "zip" utility (if installed
and found on the default search path). If neither tool is available,
raises DistutilsExecError. Returns the name of the output zip file.
"""
import zipfile
mkpath(os.path.dirname(zip_filename), dry_run=dry_run)
log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)
def visit(z, dirname, names):
for name in names:
path = os.path.normpath(os.path.join(dirname, name))
if os.path.isfile(path):
p = path[len(base_dir) + 1:]
if not dry_run:
z.write(path, p)
log.debug("adding '%s'" % p)
compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
if not dry_run:
z = zipfile.ZipFile(zip_filename, mode, compression=compression)
for dirname, dirs, files in os.walk(base_dir):
visit(z, dirname, files)
z.close()
else:
for dirname, dirs, files in os.walk(base_dir):
visit(None, dirname, files)
return zip_filename
def export_site(build_dir, static_files_dir, template_dir, locale_dir, locale_domain, template_vars):
if os.path.exists(build_dir):
shutil.rmtree(build_dir)
print "copying static files"
shutil.copytree(static_files_dir, build_dir)
print "generating locale redirectors"
redirector = LocaleRedirectorServer(template_dir)
for dirpath, dirnames, filenames in os.walk(template_dir):
files = [os.path.join(dirpath, filename)[len(template_dir) + 1 :] for filename in filenames]
for relpath in files:
print " %s" % (relpath)
abspath = os.path.join(template_dir, relpath)
env = {"PATH_INFO": "/%s" % relpath}
if relpath.endswith("index.html"):
env["PATH_INFO"] = env["PATH_INFO"][: -len("index.html")]
mimetype, contents = redirector.handle_file_as_jinja2_template(env, template_dir, abspath)
dest_path = os.path.join(build_dir, relpath)
mkpath(os.path.dirname(dest_path))
open(dest_path, "w").write(contents)
server = LocalizedTemplateServer(template_dir, locale_dir, locale_domain, template_vars)
locales = find_locales(locale_dir, locale_domain, NULL_LOCALE)
for hyphenated_locale in locales:
print "processing localization '%s'" % hyphenated_locale
env = {}
server.maybe_apply_translation(env, unhyphenate(hyphenated_locale))
for dirpath, dirnames, filenames in os.walk(template_dir):
files = [os.path.join(dirpath, filename)[len(template_dir) + 1 :] for filename in filenames]
for relpath in files:
print " %s/%s" % (hyphenated_locale, relpath)
abspath = os.path.join(template_dir, relpath)
mimetype, contents = server.handle_file_as_jinja2_template(env, template_dir, abspath)
dest_path = os.path.join(build_dir, hyphenated_locale, relpath)
mkpath(os.path.dirname(dest_path))
open(dest_path, "w").write(contents)
print "done.\n\nyour new static site is located at:\n%s" % build_dir
def copydir_run_2to3(src, dest, template=None, fixer_names=None,
options=None, explicit=None):
"""Recursively copy a directory, only copying new and changed files,
running run_2to3 over all newly copied Python modules afterward.
If you give a template string, it's parsed like a MANIFEST.in.
"""
from distutils.dir_util import mkpath
from distutils.file_util import copy_file
from distutils.filelist import FileList
filelist = FileList()
curdir = os.getcwd()
os.chdir(src)
try:
filelist.findall()
finally:
os.chdir(curdir)
filelist.files[:] = filelist.allfiles
if template:
for line in template.splitlines():
line = line.strip()
if not line: continue
filelist.process_template_line(line)
copied = []
for filename in filelist.files:
outname = os.path.join(dest, filename)
mkpath(os.path.dirname(outname))
res = copy_file(os.path.join(src, filename), outname, update=1)
if res[1]: copied.append(outname)
run_2to3([fn for fn in copied if fn.lower().endswith('.py')],
fixer_names=fixer_names, options=options, explicit=explicit)
return copied
def copy_static_scaffold(output_dir):
source_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'atramhasis', 'static'))
dest_dir = os.path.join(os.path.dirname(__file__), 'atramhasis', 'scaffolds', output_dir, '+package+', 'static')
dir_util.copy_tree(os.path.join(source_dir, 'css'), os.path.join(dest_dir, 'css'), update=True)
dir_util.copy_tree(os.path.join(source_dir, 'img'), os.path.join(dest_dir, 'img'), update=True)
dir_util.copy_tree(os.path.join(source_dir, 'js'), os.path.join(dest_dir, 'js'), update=True)
dir_util.copy_tree(os.path.join(source_dir, 'scss', 'atramhasis'), os.path.join(dest_dir, 'scss', 'atramhasis'),
update=True)
dir_util.mkpath(os.path.join(dest_dir, 'admin'))
file_util.copy_file(
os.path.join(source_dir, 'admin', '.bowerrc'),
os.path.join(dest_dir, 'admin', '.bowerrc'),
update=True
)
file_util.copy_file(
os.path.join(source_dir, 'admin', 'bower.json'),
os.path.join(dest_dir, 'admin', 'bower.json'),
update=True
)
file_util.copy_file(
os.path.join(source_dir, 'admin', 'Gruntfile.js'),
os.path.join(dest_dir, 'admin', 'Gruntfile.js'),
update=True
)
file_util.copy_file(
os.path.join(source_dir, 'admin', 'package.json'),
os.path.join(dest_dir, 'admin', 'package.json'),
update=True
)
def constructPreBuiltHTMITFakeroot(self):
"""
Construct fakeroot from prebuilt htm-it
:returns: SHA of the products repo in the fakeroot
:rtype: tuple
"""
config = self.config
logger = self.logger
productsDirectory = self.productsDirectory
logger.debug("Creating %s", productsDirectory)
mkpath(productsDirectory)
copy_tree(config.productsDir, productsDirectory)
iteration = git.getCommitCount(productsDirectory, logger=logger)
with changeToWorkingDir(productsDirectory):
actualSHA = git.getCurrentSha(logger=logger)
# Set extra python path
self.setPythonPath()
# Clean HTM-IT Scripts
self.cleanScripts()
# Purge anything not whitelisted
self.purgeBlacklistedStuff()
return (iteration, actualSHA)
def generate_default_shinken_file():
# The default file must have good values for the directories:
# etc, var and where to push scripts that launch the app.
templatefile = "bin/default/shinken.in"
build_base = 'build'
outfile = os.path.join(build_base, "bin/default/shinken")
#print('generating %s from %s', outfile, templatefile)
mkpath(os.path.dirname(outfile))
bin_path = default_paths['bin']
# Read the template file
f = open(templatefile)
buf = f.read()
f.close
# substitute
buf = buf.replace("$ETC$", default_paths['etc'])
buf = buf.replace("$VAR$", default_paths['var'])
buf = buf.replace("$RUN$", default_paths['run'])
buf = buf.replace("$LOG$", default_paths['log'])
buf = buf.replace("$SCRIPTS_BIN$", bin_path)
# write out the new file
f = open(outfile, "w")
f.write(buf)
f.close()
def distutils_extension(self, tmpdir='build', verbose=True):
from distutils.dir_util import mkpath
from .recompiler import recompile
#
if not hasattr(self, '_assigned_source'):
if hasattr(self, 'verifier'): # fallback, 'tmpdir' ignored
return self.verifier.get_extension()
raise ValueError("set_source() must be called before"
" distutils_extension()")
module_name, source, source_extension, kwds = self._assigned_source
if source is None:
raise TypeError("distutils_extension() is only for C extension "
"modules, not for dlopen()-style pure Python "
"modules")
mkpath(tmpdir)
ext, updated = recompile(self, module_name,
source, tmpdir=tmpdir, extradir=tmpdir,
source_extension=source_extension,
call_c_compiler=False, **kwds)
if verbose:
if updated:
sys.stderr.write("regenerated: %r\n" % (ext.sources[0],))
else:
sys.stderr.write("not modified: %r\n" % (ext.sources[0],))
return ext
def save_state(state, data_dir):
state_file = state_file_path(data_dir)
new_state = "{0}".format(state)
print('Saving migration state "{0}" to {1}'.format(new_state, state_file))
mkpath(os.path.dirname(state_file))
with open(state_file, "w") as f:
f.write(new_state)
def the_end(site_post):
for site_id, li in site_post.iteritems():
for i in Q.SiteHost.equal_to("site", DB.Site.create_without_data(site_id)).find():
host = i.get('host')
path = join(CONFIG.SITEMAP_PATH, host)
mkpath(join(path, "sitemap"))
sitemap(path, host, li)
def bootstrap():
# Set PETSC_DIR and PETSC_ARCH
PETSC_DIR = os.path.abspath(os.getcwd())
PETSC_ARCH = get_platform() + '-python'
os.environ['PETSC_DIR'] = PETSC_DIR
os.environ['PETSC_ARCH'] = PETSC_ARCH
sys.path.insert(0, os.path.join(PETSC_DIR, 'config'))
# Generate package __init__.py file
from distutils.dir_util import mkpath
pkgdir = os.path.join('config', 'pypi')
if not os.path.exists(pkgdir): mkpath(pkgdir)
pkgfile = os.path.join(pkgdir, '__init__.py')
fh = open(pkgfile, 'wt')
fh.write(init_py)
fh.close()
# Simple-minded lookup for MPI and mpi4py
mpi4py = mpicc = None
try:
import mpi4py
conf = mpi4py.get_config()
mpicc = conf.get('mpicc')
except ImportError: # mpi4py is not installed
mpi4py = None
mpicc = os.environ.get('MPICC') or find_executable('mpicc')
except AttributeError: # mpi4py is too old
pass
if ('setuptools' in sys.modules):
metadata['zip_safe'] = False
if not mpi4py and mpicc:
metadata['install_requires']= ['mpi4py>=1.2.2']
def build_data_filename(job, filetype):
"""
Generates an absolute path to a file for the job and type.
"""
path = '/tmp/%s/data%s' % (job, EXTENSIONS[filetype])
mkpath(os.path.dirname(path))
return os.path.abspath(path)
def test_mkpath_with_custom_mode(self):
umask = os.umask(2)
os.umask(umask)
mkpath(self.target, 448)
self.assertEqual(stat.S_IMODE(os.stat(self.target).st_mode), 448 & ~umask)
mkpath(self.target2, 365)
self.assertEqual(stat.S_IMODE(os.stat(self.target2).st_mode), 365 & ~umask)
def make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0):
"""Create a (possibly compressed) tar file from all the files under
'base_dir'. 'compress' must be "gzip" (the default), "compress",
"bzip2", or None. Both "tar" and the compression utility named by
'compress' must be on the default program search path, so this is
probably Unix-specific. The output tar file will be named 'base_dir' +
".tar", possibly plus the appropriate compression extension (".gz",
".bz2" or ".Z"). Return the output filename.
"""
# XXX GNU tar 1.13 has a nifty option to add a prefix directory.
# It's pretty new, though, so we certainly can't require it --
# but it would be nice to take advantage of it to skip the
# "create a tree of hardlinks" step! (Would also be nice to
# detect GNU tar to use its 'z' option and save a step.)
compress_ext = {"gzip": ".gz", "bzip2": ".bz2", "compress": ".Z"}
# flags for compression program, each element of list will be an argument
compress_flags = {"gzip": ["-f9"], "compress": ["-f"], "bzip2": ["-f9"]}
if compress is not None and compress not in compress_ext.keys():
raise ValueError, "bad value for 'compress': must be None, 'gzip', or 'compress'"
archive_name = base_name + ".tar"
mkpath(os.path.dirname(archive_name), dry_run=dry_run)
cmd = ["tar", "-cf", archive_name, base_dir]
spawn(cmd, dry_run=dry_run)
if compress:
spawn([compress] + compress_flags[compress] + [archive_name], dry_run=dry_run)
return archive_name + compress_ext[compress]
else:
return archive_name
def build_extension(self, ext, copy=True):
self.announce("building '%s' in %s"%(ext[1], ext[0]))
if os.name == 'nt':
# same as in config.py here: even on 'nt' we have to
# use posix paths because we run in a cygwin shell at this point
path = self.build_temp.replace('\\', '/') + '/' + ext[0]
temp_target = self.build_temp.replace('\\', '/') + '/' + ext[0]
else:
path = os.path.join(self.build_temp, ext[0])
temp_target = os.path.join(self.build_temp, ext[0])
make = os.environ.get('MAKE', 'make')
command = '%s -C "%s" %s'%(make, path, ext[1])
spawn(['sh', '-c', command], self.verbose, self.dry_run)
#The extension may not be compiled. For now just skip it.
if copy and os.path.isfile(os.path.join(temp_target, ext[1])):
if self.inplace: build_path = ext[0]
else: build_path = os.path.join(self.build_lib, ext[0])
mkpath (build_path, 0777, self.verbose, self.dry_run)
copy_file(os.path.join(path, ext[1]),
os.path.join(build_path, ext[1]),
1, 1, 0, None, self.verbose, self.dry_run)
def make_zipfile(base_name, base_dir, verbose = 0, dry_run = 0):
try:
import zipfile
except ImportError:
zipfile = None
zip_filename = base_name + '.zip'
mkpath(os.path.dirname(zip_filename), dry_run=dry_run)
if zipfile is None:
if verbose:
zipoptions = '-r'
else:
zipoptions = '-rq'
try:
spawn(['zip',
zipoptions,
zip_filename,
base_dir], dry_run=dry_run)
except DistutilsExecError:
raise DistutilsExecError, "unable to create zip file '%s': could neither import the 'zipfile' module nor find a standalone zip utility" % zip_filename
else:
log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)
if not dry_run:
zip = zipfile.ZipFile(zip_filename, 'w', compression=zipfile.ZIP_DEFLATED)
for dirpath, dirnames, filenames in os.walk(base_dir):
for name in filenames:
path = os.path.normpath(os.path.join(dirpath, name))
if os.path.isfile(path):
zip.write(path, path)
log.info("adding '%s'" % path)
zip.close()
return zip_filename
def _move(self, instance=None, **kwargs):
if hasattr(instance, 'get_upload_to'):
src = getattr(instance, self.attname)
if src:
src = str(src)
m = re.match(r"%s/(.*)" % self.upload_to, src)
if m:
upload_path = instance.get_upload_to(self.attname)
dst = "%s%s" % (
upload_path,
m.groups()[0]
)
basedir = os.path.join(
settings.MEDIA_ROOT,
os.path.dirname(dst)
)
fromdir = os.path.join(
settings.MEDIA_ROOT,
src
)
mkpath(basedir)
shutil.move(fromdir,
os.path.join(basedir,
m.groups()[0])
)
setattr(instance, self.attname, dst)
instance.save()
if self.signal:
self.signal(instance)
def __init__(self, *args, **kwargs):
super(Flask, self).__init__(*args, **kwargs)
self.request_class = TrexRequest
# Add trex/templates to the jinja2 search path
self.jinja_loader.searchpath.append(os.path.join(os.path.dirname(__file__), 'templates'))
self.settings.readfp(codecs.open(os.path.join(self.root_path, '..', 'trex', 'base.ini'), 'r', 'utf8'))
self.settings.readfp(codecs.open(os.path.join(self.root_path, 'default.ini'), 'r', 'utf8'))
self.check_default_config()
self.settings.readfp(codecs.open(os.path.join(self.root_path, 'local.ini'), 'r', 'utf8'))
self.check_local_config()
# Set up logging directory target. Later this can be configured
self.log_directory = os.path.abspath(os.path.join(self.root_path, '..', 'logs'))
mkpath(self.log_directory)
self.init_jinja()
self.exception_reporter = FlaskExceptionReporter(self)
FlaskCDN(self)
trex.support.model.settings = self.settings
self.init_application()
if self.settings.getboolean('server', 'opcode_audit'):
from trex.support.audit import TrexAudit
TrexAudit(self)
def __init__(self):
self.config_path = os.path.join(
os.path.expanduser("~"), '.config', 'congabonga')
dir_util.mkpath(self.config_path)
self.ui = CongaUi()
self.ui.connect('tab-changed', self.on_ui_tab_changed)
self.ui.connect('player-seek', self.on_ui_player_seek)
self.ui.connect('player-pause', self.on_ui_player_pause)
self.ui.connect('player-next', self.on_ui_player_next)
self.ui.connect('player-previous', self.on_ui_player_previous)
self.ui.connect('player-play', self.on_ui_player_play)
self.ui.connect('playlist-activated', self.on_ui_playlist_activated)
self.ui.connect('artist-activated', self.on_ui_artist_activated)
self.ui.connect('album-activated', self.on_ui_album_activated)
self.ui.connect('search', self.on_ui_search)
self.ui.connect('library-switch-toggled', self.on_ui_library_switch_toggled)
self.library = Library(self.config_path)
self.library.connect('updated', self.on_library_updated)
self.library.connect('plugins-loaded', self.on_library_plugins_loaded)
self.library.load()
self.player = CongaPlayer()
self.player.connect('playing', self.on_player_playing)
self.player.connect('stopped', self.on_player_stopped)
self.player.connect('next', self.on_player_next)
self.player.connect('position-updated', self.on_player_position_updated)
def _build_dependencies(self, transform_dir):
target_dir = os.path.join(self.venv_dir,
'lib/python2.7/site-packages/biokbase')
mkpath(target_dir)
touch(os.path.join(target_dir, '__init__.py'))
shutil.copytree(
os.path.join(transform_dir, "lib/biokbase/Transform"),
os.path.join(self.venv_dir,
"lib/python2.7/site-packages/biokbase/Transform"))
print("Pulling git dependencies")
for dep in KBASE_DEPENDENCIES:
print("Checking out " + GITHUB_URL + dep)
repo = os.path.split(dep)[1]
gitdir = os.path.join(self.working_dir, repo)
git.Git().clone(GITHUB_URL + dep, gitdir)
self._copy_deps(gitdir)
shutil.rmtree(os.path.join(gitdir))
scripts_dir = os.path.join(transform_dir, "plugins/scripts/")
bin_dir = os.path.join(self.venv_dir, "bin/")
for root, _, files in os.walk(scripts_dir):
for file_ in files:
filepath = os.path.join(root, file_)
print("copy from {0} {1}".format(filepath, bin_dir))
shutil.copy(filepath, bin_dir)
self._make_executable_for_all(os.path.join(bin_dir, file_))
self._make_wrapper(bin_dir, file_)
def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=None, mode="w"):
"""Create a zip file from all the files under 'base_dir'. The output
zip file will be named 'base_dir' + ".zip". Uses either the "zipfile"
Python module (if available) or the InfoZIP "zip" utility (if installed
and found on the default search path). If neither tool is available,
raises DistutilsExecError. Returns the name of the output zip file.
"""
import zipfile
mkpath(os.path.dirname(zip_filename), dry_run=dry_run)
log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)
def visit(z, dirname, names):
for name in names:
path = os.path.normpath(os.path.join(dirname, name))
if os.path.isfile(path):
p = path[len(base_dir) + 1 :]
if not dry_run:
z.write(path, p)
log.debug("adding '%s'" % p)
if compress is None:
compress = sys.version >= "2.4" # avoid 2.3 zipimport bug when 64 bits
compression = [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED][bool(compress)]
if not dry_run:
z = zipfile.ZipFile(zip_filename, mode, compression=compression)
os.path.walk(base_dir, visit, z)
z.close()
else:
os.path.walk(base_dir, visit, None)
return zip_filename
def copy_data_tree(src, dst, exclude_pattern=['(RCS|CVS|\.svn)', '.*\~']):
"""
Copy an entire directory tree 'src' to a new location 'dst'.
:param exclude_pattern: a list of pattern to exclude.
"""
names = os.listdir(src)
mkpath(dst)
outfiles = []
for p in exclude_pattern:
names = [x for x in names if not (re.match(p, x))]
for n in names:
src_name = os.path.join(src, n)
dst_name = os.path.join(dst, n)
if os.path.isdir(src_name):
ret = copy_data_tree(src_name, dst_name, exclude_pattern)
outfiles += ret
else:
shutil.copy2(src_name, dst_name)
outfiles.append(dst_name)
return outfiles
def make_lib_archive(self, zip_filename, base_dir, verbose=0,
dry_run=0):
# Like distutils "make_archive", except we can specify the
# compression to use - default is ZIP_STORED to keep the
# runtime performance up.
# Also, we don't append '.zip' to the filename.
from distutils.dir_util import mkpath
mkpath(os.path.dirname(zip_filename), dry_run=dry_run)
if self.compressed:
compression = zipfile.ZIP_DEFLATED
else:
compression = zipfile.ZIP_STORED
if not dry_run:
z = zipfile.ZipFile(zip_filename, "w",
compression=compression)
save_cwd = os.getcwd()
os.chdir(base_dir)
for dirpath, dirnames, filenames in os.walk('.'):
for fn in filenames:
path = os.path.normpath(os.path.join(dirpath, fn))
if os.path.isfile(path):
z.write(path, path)
os.chdir(save_cwd)
z.close()
return zip_filename
def configure_libffi(self):
if sys.platform == "win32":
return
if LIBFFI_SOURCES == None:
return
src_dir = os.path.abspath(LIBFFI_SOURCES)
# Building libffi in a path containing spaces doesn't work:
self.build_temp = self.build_temp.replace(" ", "")
build_dir = os.path.join(self.build_temp, 'libffi')
if not self.force and self.fix_extension(build_dir):
return
mkpath(build_dir)
config_args = []
# Pass empty CFLAGS because we'll just append the resulting CFLAGS
# to Python's; -g or -O2 is to be avoided.
cmd = "cd %s && env CFLAGS='' '%s/configure' %s" \
% (build_dir, src_dir, " ".join(config_args))
print 'Configuring static FFI library:'
print cmd
res = os.system(cmd)
if res:
print "Failed"
sys.exit(res)
assert self.fix_extension(build_dir), "Could not find libffi after building it"
def run_old(self):
"""Run the distutils command"""
# check whether we can execute rpmbuild
if not self.dry_run:
try:
devnull = open(os.devnull, 'w')
subprocess.Popen(['rpmbuild', '--version'],
stdin=devnull, stdout=devnull)
except OSError:
raise DistutilsError("Cound not execute rpmbuild. Make sure "
"it is installed and in your PATH")
mkpath(self.dist_dir)
# build command: to get the build_base
cmdbuild = self.get_finalized_command("build")
cmdbuild.verbose = self.verbose
self.build_base = cmdbuild.build_base
self._populate_rpm_topdir(self.rpm_base)
for name, rpm_spec in self.rpm_specs.items():
self._prepare_distribution(name)
self._create_rpm(rpm_name=name, spec=rpm_spec)
if not self.keep_temp:
remove_tree(self.build_base, dry_run=self.dry_run)
def build_man_page(self):
"""Build man pages for all installable programs."""
self.announce("building man pages")
descriptions = {}
descriptions['synopsis'] = """simple frontend to the Synopsis framework, a multi-language source code introspection tool that
provides a variety of representations for the parsed code, to
enable further processing such as documentation extraction,
reverse engineering, and source-to-source translation."""
descriptions['sxr-server'] = """the Synopsis Cross-Reference http server. Allows users
to query and browse cross-referenced source code."""
help2man = find_executable('help2man')
if not help2man:
self.warn("cannot build man pages")
return
gzip = find_executable('gzip')
section = 1
man_dir = 'share/man/man%d'%section
mkpath(man_dir, 0777, self.verbose, self.dry_run)
for s in ['synopsis', 'sxr-server']:
command = [help2man, '-N', '-n', descriptions[s]]
executable = os.path.join('scripts', s)
output = '%s/%s.%d'%(man_dir, s, section)
command += ['-o', output, executable]
spawn(command)
if gzip:
spawn(['gzip', '-f', output])
def copy_doc(path, name, pkgdir, exts=TEXT_EXTS, language=None, dry_run=0,
copy_tree=copy_tree, copy_file=copy_file, mkpath=mkpath):
if path is None:
return
is_string = hasattr(path, 'getvalue')
if is_string:
ext = '.txt'
else:
ext = os.path.splitext(path)[1].lower()
if ext == '':
ext = '.txt'
if ext not in exts:
raise ValueError('Invalid extension for %s' % (path,))
destdir = os.path.join(pkgdir, 'Contents', 'Resources')
if language is not None:
destdir = os.path.join(destdir, language + '.lproj')
mkpath(destdir)
dest = os.path.join(destdir, name + ext)
if is_string:
if not dry_run:
f = open(dest, 'w')
f.write(path.getvalue())
f.close()
elif ext == '.rtfd':
copy_tree(path, dest)
else:
copy_file(path, dest)
def create_config():
logger.info("Creating default config file %s"%config_file())
cnf=Config(os.path.join(config_folder(),HISTORY_FILE), URL_NOT_SET,os.path.join(config_folder(),TORRENTS_DIR))
dir_util.mkpath(config_folder())
with open(os.path.join(config_folder(),CONFIG_FILE),'w') as f:
yaml.dump(cnf,f)
return cnf
def mkdir_if_not_exists(dir):
"""Makes the specified directory if it doesn't exist.
:dir: The directory name to be created.
:returns: None
"""
if not os.path.isdir(dir):
mkpath(dir)
parser.add_argument('--image_dir', default='keypoint_test_a_images_20180103')
parser.add_argument('--json_file',
default='keypoint_test_a_annotations_20180103.json')
parser.add_argument('--confidence', type=float, default=0.5)
args = parser.parse_args()
# set variable paths to images and json file
image_dir = os.path.join(args.data_dir, args.image_dir)
json_file = os.path.join(args.data_dir, args.json_file)
annos = json.load(open(json_file, 'r'))
print('annos', len(annos))
target_annotation_dir = os.path.join(args.out_dir)
mkpath(target_annotation_dir)
for idx, anno in enumerate(annos):
# Print status.
if (idx + 1) % 1000 == 0 or (idx + 1) == len(annos):
print(str(idx + 1) + ' / ' + str(len(annos)) + "test")
# read images
image_path = image_dir + "/" + anno['image_id'] + '.jpg'
image = cv2.imread(image_dir + "/" + anno['image_id'] + '.jpg',
cv2.IMREAD_COLOR)
print(image_path)
out_file = target_annotation_dir + "/" + anno['image_id'] + '.txt'
f = open(out_file, 'w')
max_im_shrink = (0x7fffffff / 200.0 / (image.shape[0] * image.shape[1])
def generate_mod(py_file):
log.info("generating cffi module %r" % py_file)
mkpath(os.path.dirname(py_file))
updated = recompiler.make_py_source(ffi, module_name, py_file)
if not updated:
log.info("already up-to-date")
def build(skip):
"""Build custom Python2 from source."""
# Step 1: change into our build directory
os.chdir(PYTHON_BUILD_DIR)
# Don't compile Python if the skip option is passed
if not skip:
# Step 1.5: Add extra modules
setup_dist = os.path.join(PYTHON_BUILD_DIR, 'Modules/Setup.dist')
with open(setup_dist, "a") as f:
log.debug("Adding additional modules to be included...")
f.write("_socket socketmodule.c timemodule.c\n")
f.write("_ssl _ssl.c -DUSE_SSL "
"-I{0}/include -I{0}/include/openssl -L{0}/lib "
"-lssl -lcrypto".format(OPENSSL_INSTALL_PATH))
# Step 2: Run the Configure setup of Python to set correct paths
os.chdir(PYTHON_BUILD_DIR)
if os.path.isdir(PYTHON2_INSTALL):
shutil.rmtree(PYTHON2_INSTALL, ignore_errors=True)
mkpath(PYTHON2_INSTALL)
log.info("Configuring Python...")
cmd = [
'./configure',
'--prefix={}'.format(PYTHON2_INSTALL),
# 'CPPFLAGS=-I{}/include'.format(OPENSSL_INSTALL_PATH),
# 'LDFLAGS=-L{}/lib'.format(OPENSSL_INSTALL_PATH),
'--enable-shared',
'--enable-toolbox-glue',
'--with-ensurepip=install',
'--enable-ipv6',
'--with-threads',
'--datarootdir={}/share'.format(PYTHON2_INSTALL),
'--datadir={}/share'.format(PYTHON2_INSTALL),
]
runner.Popen(cmd, stdout=sys.stdout)
# Step 3: compile Python. this will take a while.
# FIXME: We need to check return codes.
log.info("Compiling Python. This will take a while time...")
log.detail("Running Python make routine...")
cmd = ['/usr/bin/make']
runner.Popen(cmd, stdout=sys.stdout)
sys.stdout.flush() # does this help?
log.debug("Create some temp files thats")
log.detail("Running Python make install routine...")
cmd = ['/usr/bin/make', 'install']
runner.Popen(cmd, stdout=sys.stdout)
sys.stdout.flush() # does this help?
# Step 4: Install pip + requirements
os.chdir(os.path.join(PYTHON2_INSTALL, 'bin'))
# Update pip to latest
log.info("Upgrading pip...")
cmd = ['./pip', 'install', '--upgrade', 'pip']
runner.Popen(cmd, stdout=sys.stdout)
# Install all pip modules from requirements.txt
log.info("Install requirements...")
requirements = os.path.join(CURRENT_DIR, 'requirements.txt')
cmd = ['./python2.7', '-m', 'pip', 'install', '-r', requirements]
runner.Popen(cmd, stdout=sys.stdout)
else:
log.info("Python compile skipped due to -skip option")
def build_library(self,
library,
pkg_config_name,
local_source=None,
supports_non_srcdir_builds=True):
log.info("checking if library '%s' is installed", library)
try:
build_args = self.pkgconfig(pkg_config_name)
log.info("found '%s' installed, using it", library)
except CalledProcessError:
# If local_source is not specified, then immediately fail.
if local_source is None:
raise DistutilsExecError("library '%s' is not installed",
library)
log.info("building library '%s' from source", library)
# Determine which compilers we are to use, and what flags.
# This is based on what distutils.sysconfig.customize_compiler()
# does, but that function has a problem that it doesn't produce
# necessary (e.g. architecture) flags for C++ compilers.
cc, cxx, opt, cflags = get_config_vars('CC', 'CXX', 'OPT',
'CFLAGS')
cxxflags = cflags
if 'CC' in os.environ:
cc = os.environ['CC']
if 'CXX' in os.environ:
cxx = os.environ['CXX']
if 'CFLAGS' in os.environ:
cflags = opt + ' ' + os.environ['CFLAGS']
if 'CXXLAGS' in os.environ:
cxxflags = opt + ' ' + os.environ['CXXFLAGS']
# Use a subdirectory of build_temp as the build directory.
build_temp = os.path.realpath(
os.path.join(self.build_temp, library))
# Destination for headers and libraries is build_clib.
build_clib = os.path.realpath(self.build_clib)
# Create build directories if they do not yet exist.
mkpath(build_temp)
mkpath(build_clib)
if not supports_non_srcdir_builds:
self._stage_files_recursive(local_source, build_temp)
# Run configure.
cmd = [
'/bin/sh',
os.path.join(os.path.realpath(local_source),
'configure'), '--prefix=' + build_clib,
'--disable-shared', '--with-pic', '--disable-maintainer-mode'
]
log.info('%s', ' '.join(cmd))
check_call(cmd,
cwd=build_temp,
env=dict(self._environ,
CC=cc,
CXX=cxx,
CFLAGS=cflags,
CXXFLAGS=cxxflags))
# Run make install.
cmd = ['make', 'install']
log.info('%s', ' '.join(cmd))
check_call(cmd, cwd=build_temp, env=self._environ)
build_args = self.pkgconfig(pkg_config_name)
return build_args
def mkpath(self, name, mode=0777):
mkpath(name, mode, dry_run=self.dry_run)
files_differ = True
#remove original file
os.unlink(original_file)
else:
files_differ = True
if files_differ:
#Files differ, so overwrite original with candidate
os.rename(candidate_file, original_file)
#
# generate compound code
#
mkpath(os.path.join(ROOT_DIR, "include/obj"))
mkpath(os.path.join(ROOT_DIR, "include/gen"))
mkpath(os.path.join(ROOT_DIR, "src/obj"))
mkpath(os.path.join(ROOT_DIR, "src/gen"))
for n in compound_names:
x = compound_types[n]
# skip natively implemented types
# if x.niflibtype: continue
if x.name in NATIVETYPES.keys(): continue
if n[:3] == 'ns ': continue
if not GENALLFILES and not x.cname in GENBLOCKS:
continue
def read_args_from_stdin_and_run():
''' Main executable function to train and evaluate classifier.
Post Condition
--------------
AUC and other eval info printed to stdout.
Trained classifier saved ???.
'''
if not sys.stdin.isatty():
for line in sys.stdin.readlines():
line = line.strip()
sys.argv.extend(line.split(' '))
parser = argparse.ArgumentParser()
parser.add_argument(
'--dataset_path',
default='/tmp/',
type=str,
help="Path to folder containing:" +
" *.npy files: X_train, y_train, P_train"
" *.txt files: X_colnames.txt and y_colnames.txt")
parser.add_argument(
'--pretrained_clf_path',
default='/tmp/',
type=str,
help="Path to folder to hold output from classifier. Includes:" +
" perf_metric*.txt files: auc_train.txt & auc_test.txt" +
" settings.txt: description of all settings to reproduce.")
parser.add_argument(
'--output_path',
default=None,
type=str,
help='Place to write output files. Defaults to pretrained_clf_path if not specified.')
parser.add_argument(
'--split_names',
default='test')
parser.add_argument(
'--split_nicknames',
default=None)
parser.add_argument(
'--features_path',
default='/tmp/',
type=str,
help="Path to folder with SSAMfeat*.npy files")
parser.add_argument(
'--target_arr_name',
default='Y',
type=str,
)
parser.add_argument(
'--target_names',
default='all',
type=str,
help='Name of response/intervention to test.' +
' To try specific interventions, write names separated by commas.' +
' To try all interventions, use special name "all"')
parser.add_argument(
'--top_k',
default=3, type=int,
help='Num of predictions to take to compute hitrate')
parser.add_argument(
'--oracle_dataset_path',
default=None, type=str,
help='Path to train arrays used as oracle knowledge for predictions')
parser.add_argument(
'--seed_bootstrap',
default=42,
type=int,
help='Seed for bootstrap')
parser.add_argument(
'--n_bootstraps',
default=5000,
type=int,
help='Number of samples for bootstrap conf. intervals')
parser.add_argument(
'--bootstrap_stratify_pos_and_neg',
default=True,
type=int,
help='Whether to stratify examples or not')
args, unk_list = parser.parse_known_args()
arg_dict = vars(args)
dataset_path = arg_dict['dataset_path']
if arg_dict['output_path'] is None:
output_path = arg_dict['pretrained_clf_path']
else:
output_path = arg_dict['output_path']
if not os.path.exists(output_path):
mkpath(output_path)
if arg_dict['oracle_dataset_path'] is None:
arg_dict['oracle_dataset_path'] = dataset_path
# Set default seed for numpy
# Might be used by cross-validation, etc.
np.random.seed(8675309)
clf_opt_list = list()
# Read parsed args from plain-text file
# so we can exactly reproduce pretrained classifier
try:
with open(os.path.join(arg_dict['pretrained_clf_path'], 'settings.txt'), 'r') as f:
for line in f.readlines():
line = line.strip()
clf_opt_list.append(line.split(' = '))
clf_opts = dict(clf_opt_list)
except IOError as e:
clf_opts = dict()
clf_opts['feature_arr_names'] = 'X' # DUMMY!
clf_opts['classifier_name'] = arg_dict['pretrained_clf_path']
feat_path_list = [
arg_dict['dataset_path'],
arg_dict['features_path']]
pprint('[run_classifier says:] Loading dataset ...')
start_time = time.time()
feature_arr_names = clf_opts['feature_arr_names'].split(',')
pprint('feature_arr_names:')
feat_colnames_by_arr = OrderedDict()
for feat_arr_name in feature_arr_names:
pprint(feat_arr_name)
cur_feat_colnames = None
for feat_path in feat_path_list:
colname_fpath = os.path.join(
feat_path,
feat_arr_name + '_colnames.txt')
if os.path.exists(colname_fpath):
cur_feat_colnames = \
[unicode(feat_arr_name + ":") + s
for s in load_list_of_unicode_from_txt(colname_fpath)]
break
feat_colnames_by_arr[feat_arr_name] = cur_feat_colnames
target_arr_name = arg_dict['target_arr_name']
all_target_names = load_list_of_strings_from_txt(os.path.join(
arg_dict['dataset_path'],
target_arr_name + '_colnames.txt'))
target_names = arg_dict['target_names']
if target_names == 'all':
target_names = all_target_names
target_cols = np.arange(len(all_target_names)).tolist()
else:
target_names = target_names.split(',')
target_cols = list()
for name in target_names:
assert name in all_target_names
target_cols.append(all_target_names.index(name))
datasets_by_split = OrderedDict()
split_names = arg_dict['split_names'].split(',')
if arg_dict['split_nicknames'] is None:
split_nicknames = split_names
else:
split_nicknames = arg_dict['split_nicknames'].split(',')
alias_by_split = OrderedDict()
for nickname, split_name in zip(split_nicknames,split_names):
datasets_by_split[nickname] = dict()
split_dataset = datasets_by_split[nickname]
alias_by_split[nickname] = split_name
# Load Y
dense_fpath = os.path.join(
dataset_path,
target_arr_name + "_%s.npy" % split_name)
y = np.asarray(np.load(dense_fpath), order='C', dtype=np.int32)
if y.ndim < 2:
y = y[:,np.newaxis]
assert y.ndim == 2
assert y.shape[1] == len(all_target_names)
split_dataset['y'] = y[:, target_cols]
assert split_dataset['y'].shape[1] == len(target_cols)
# Load X
x_list = list()
for feat_arr_name in feature_arr_names:
x_cur = None
def fpath_generator():
for feat_path in feat_path_list:
for sname in [nickname, split_name]:
dense_fpath = os.path.join(
feat_path, feat_arr_name + "_" + sname + ".npy")
sparse_fpath = os.path.join(
feat_path, feat_arr_name + "_csr_" + sname + ".npz")
yield dense_fpath, sparse_fpath
ds_path_list = [pair for pair in fpath_generator()]
for ii, (dense_fpath, sparse_fpath) in enumerate(ds_path_list):
try:
if os.path.exists(sparse_fpath):
x_cur = load_csr_matrix(sparse_fpath)
assert np.all(np.isfinite(x_cur.data))
break
else:
x_cur = np.asarray(
np.load(dense_fpath),
order='C', dtype=np.float64)
if x_cur.ndim < 2:
x_cur = np.atleast_2d(x_cur).T
assert np.all(np.isfinite(x_cur))
break
except IOError as e:
if ii == len(ds_path_list) - 1:
# Couldn't find desired file in any feat_path
raise e
else:
# Try the next feat_path in the list
pass
if x_cur is not None:
if feat_colnames_by_arr[feat_arr_name] is not None:
feat_dim = len(feat_colnames_by_arr[feat_arr_name])
assert x_cur.shape[1] == feat_dim
else:
# Add dummy colnames
feat_dim = x_cur.shape[1]
n_sig_digits = np.maximum(
3, int(np.ceil(np.log10(feat_dim))))
fmt_str = "%s_%0" + str(n_sig_digits) + "d"
feat_colnames_by_arr[feat_arr_name] = [
fmt_str % (feat_arr_name, fid)
for fid in range(feat_dim)]
x_list.append(x_cur)
if isinstance(x_list[0], np.ndarray):
split_dataset['x'] = np.hstack(x_list)
else:
split_dataset['x'] = scipy.sparse.hstack(x_list, format='csr')
assert split_dataset['x'].ndim == 2
assert split_dataset['x'].shape[0] == split_dataset['y'].shape[0]
assert (
isinstance(split_dataset['x'], np.ndarray)
or isinstance(split_dataset['x'], scipy.sparse.csr_matrix)
)
if split_name == split_names[0]:
# Flatten feat colnames into single list
feat_colnames = sum(feat_colnames_by_arr.values(), [])
assert isinstance(feat_colnames, list)
assert len(feat_colnames) == split_dataset['x'].shape[1]
print('y colnames: %s' % ' '.join(target_names))
if len(feat_colnames) > 10:
print('x colnames: %s ... %s' % (' '.join(feat_colnames[:5]), ' '.join(feat_colnames[-5:])))
else:
print('x colnames: %s' % ' '.join(feat_colnames))
print('---- %5s dataset summary' % split_name)
print('%9d total examples' % y.shape[0])
print('y : %d x %d targets' % split_dataset['y'].shape)
print('x : %d x %d features' % split_dataset['x'].shape)
for c in xrange(len(target_names)):
y_c = split_dataset['y'][:,c]
print('target %s : frac pos %.3f' % (target_names[c], np.mean(y_c)))
print(' %6d pos examples' % np.sum(y_c == 1))
print(' %6d neg examples' % np.sum(y_c == 0))
elapsed_time = time.time() - start_time
print('[run_classifier says:] dataset loaded after %.2f sec.' % elapsed_time)
print('[eval_pretrained_classifier says:] eval multilabel')
eval_pretrained_avg_clf(
classifier_name=clf_opts['classifier_name'],
classifier_path=arg_dict['pretrained_clf_path'],
datasets_by_split=datasets_by_split,
alias_by_split=alias_by_split,
dataset_path=arg_dict['dataset_path'],
oracle_dataset_path=arg_dict['oracle_dataset_path'],
feat_colnames=feat_colnames,
output_path=output_path,
target_arr_name=target_arr_name,
label_names=all_target_names,
feature_arr_names=feature_arr_names,
seed_bootstrap=arg_dict['seed_bootstrap'],
n_bootstraps=arg_dict['n_bootstraps'],
bootstrap_stratify_pos_and_neg=arg_dict['bootstrap_stratify_pos_and_neg'],
)
elapsed_time = time.time() - start_time
print('[eval_pretrained_classifier says:] completed after %.2f sec' % (elapsed_time))
def byte_compile(py_files,
optimize=0,
force=0,
target_dir=None,
verbose=1,
dry_run=0,
direct=None):
if direct is None:
direct = (__debug__ and optimize == 0)
# "Indirect" byte-compilation: write a temporary script and then
# run it with the appropriate flags.
if not direct:
from tempfile import mktemp
from distutils.util import execute, spawn
script_name = mktemp(".py")
if verbose:
print("writing byte-compilation script '%s'" % script_name)
if not dry_run:
script = open(script_name, "w")
script.write("""
from pluginbuilder.util import byte_compile
from modulegraph.modulegraph import *
files = [
""")
for f in py_files:
script.write(repr(f) + ",\n")
script.write("]\n")
script.write("""
byte_compile(files, optimize=%r, force=%r,
target_dir=%r,
verbose=%r, dry_run=0,
direct=1)
""" % (optimize, force, target_dir, verbose))
script.close()
cmd = [sys.executable, script_name]
if optimize == 1:
cmd.insert(1, "-O")
elif optimize == 2:
cmd.insert(1, "-OO")
spawn(cmd, verbose=verbose, dry_run=dry_run)
execute(os.remove, (script_name, ),
"removing %s" % script_name,
verbose=verbose,
dry_run=dry_run)
else:
from py_compile import compile
from distutils.dir_util import mkpath
for mod in py_files:
# Terminology from the py_compile module:
# cfile - byte-compiled file
# dfile - purported source filename (same as 'file' by default)
if mod.filename == mod.identifier:
cfile = os.path.basename(mod.filename)
dfile = cfile + (__debug__ and 'c' or 'o')
else:
cfile = mod.identifier.replace('.', os.sep)
if mod.packagepath:
dfile = cfile + os.sep + '__init__.py' + (__debug__ and 'c'
or 'o')
else:
dfile = cfile + '.py' + (__debug__ and 'c' or 'o')
if target_dir:
cfile = os.path.join(target_dir, dfile)
if force or newer(mod.filename, cfile):
if verbose:
print("byte-compiling %s to %s" % (mod.filename, dfile))
if not dry_run:
mkpath(os.path.dirname(cfile))
suffix = os.path.splitext(mod.filename)[1]
if suffix in ('.py', '.pyw'):
zfile, pth = path_to_zip(mod.filename)
if zfile is None:
compile(mod.filename, cfile, dfile)
else:
fn = dfile + '.py'
open(fn, 'wb').write(get_zip_data(zfile, pth))
compile(mod.filename, cfile, dfile)
os.unlink(fn)
elif suffix in PY_SUFFIXES:
# Minor problem: This will happily copy a file
# <mod>.pyo to <mod>.pyc or <mod>.pyc to
# <mod>.pyo, but it does seem to work.
copy_file_data(mod.filename, cfile)
else:
raise RuntimeError \
("Don't know how to handle %r" % mod.filename)
else:
if verbose:
print("skipping byte-compilation of %s to %s" % \
(mod.filename, dfile))
RedNose.score = 201 # above coverage
nose_argv = ['', '-s', '-v', '--exe', '--rednose', '--nologcapture']
if '--collect' in sys.argv:
sys.argv.append('--collect-only')
if '--collect-only' in sys.argv: # this is a user trying simply to view the available tests. no need xunit.
CTRexScenario.is_test_list = True
xml_arg = ''
else:
xml_name = 'unit_test.xml'
if CTRexScenario.setup_dir:
CTRexScenario.setup_name = os.path.basename(
CTRexScenario.setup_dir)
xml_name = 'report_%s.xml' % CTRexScenario.setup_name
xml_arg = '--xunit-file=%s/%s' % (CTRexScenario.report_dir, xml_name)
mkpath(CTRexScenario.report_dir)
sys_args = sys.argv[:]
for i, arg in enumerate(sys.argv):
if 'log-path' in arg:
nose_argv += ['--nologcapture']
else:
for tests_type in CTRexScenario.test_types.keys():
if tests_type in arg:
specific_tests = True
CTRexScenario.test_types[tests_type].append(
arg[arg.find(tests_type):])
sys_args.remove(arg)
if not specific_tests:
for key in ('--func', '--functional'):
def copy_tree(src,
dst,
preserve_mode=1,
preserve_times=1,
preserve_symlinks=0,
update=0,
verbose=0,
dry_run=0,
condition=None):
"""
Copy an entire directory tree 'src' to a new location 'dst'. Both
'src' and 'dst' must be directory names. If 'src' is not a
directory, raise DistutilsFileError. If 'dst' does not exist, it is
created with 'mkpath()'. The end result of the copy is that every
file in 'src' is copied to 'dst', and directories under 'src' are
recursively copied to 'dst'. Return the list of files that were
copied or might have been copied, using their output name. The
return value is unaffected by 'update' or 'dry_run': it is simply
the list of all files under 'src', with the names changed to be
under 'dst'.
'preserve_mode' and 'preserve_times' are the same as for
'copy_file'; note that they only apply to regular files, not to
directories. If 'preserve_symlinks' is true, symlinks will be
copied as symlinks (on platforms that support them!); otherwise
(the default), the destination of the symlink will be copied.
'update' and 'verbose' are the same as for 'copy_file'.
"""
assert isinstance(src, str), repr(src)
assert isinstance(dst, str), repr(dst)
from distutils.dir_util import mkpath
from distutils.file_util import copy_file
from distutils.dep_util import newer
from distutils.errors import DistutilsFileError
from distutils import log
if condition is None:
condition = skipscm
if not dry_run and not os_path_isdir(src):
raise DistutilsFileError("cannot copy tree '%s': not a directory" %
src)
try:
names = os_listdir(src)
except os.error as xxx_todo_changeme:
(errno, errstr) = xxx_todo_changeme.args
if dry_run:
names = []
else:
raise DistutilsFileError("error listing files in '%s': %s" %
(src, errstr))
if not dry_run:
mkpath(dst)
outputs = []
for n in names:
src_name = os.path.join(src, n)
dst_name = os.path.join(dst, n)
if (condition is not None) and (not condition(src_name)):
continue
if preserve_symlinks and os_path_islink(src_name):
link_dest = os_readlink(src_name)
log.info("linking %s -> %s", dst_name, link_dest)
if not dry_run:
if update and not newer(src, dst_name):
pass
else:
if os_path_islink(dst_name):
os.remove(dst_name)
os.symlink(link_dest, dst_name)
outputs.append(dst_name)
elif os_path_isdir(src_name):
outputs.extend(
copy_tree(src_name,
dst_name,
preserve_mode,
preserve_times,
preserve_symlinks,
update,
dry_run=dry_run,
condition=condition))
else:
copy_file(src_name,
dst_name,
preserve_mode,
preserve_times,
update,
dry_run=dry_run)
outputs.append(dst_name)
return outputs
def mkpath(self, fn):
mkpath(fn, dry_run=self.dry_run, verbose=self.verbose)
def finalize_options(self):
# type: (None) -> None
"""Append custom openssl include file and library linking options."""
build_ext.build_ext.finalize_options(self)
self.openssl_default = None
self.set_undefined_options('build', ('openssl', 'openssl'))
if self.openssl is None:
self.openssl = self.openssl_default
self.set_undefined_options('build', ('bundledlls', 'bundledlls'))
self.libraries = ['ssl', 'crypto']
if sys.platform == 'win32':
self.libraries = ['ssleay32', 'libeay32']
if self.openssl and openssl_version(self.openssl, 0x10100000, True):
self.libraries = ['libssl', 'libcrypto']
self.swig_opts.append('-D_WIN32')
# Swig doesn't know the version of MSVC, which causes errors in e_os2.h
# trying to import stdint.h. Since python 2.7 is intimately tied to
# MSVC 2008, it's harmless for now to define this. Will come back to
# this shortly to come up with a better fix.
self.swig_opts.append('-D_MSC_VER=1500')
if sys.version_info[:1] >= (3,):
self.swig_opts.append('-py3')
log.debug('self.include_dirs = %s', self.include_dirs)
log.debug('self.library_dirs = %s', self.library_dirs)
if self.openssl is not None:
log.debug('self.openssl = %s', self.openssl)
openssl_library_dir = os.path.join(self.openssl, 'lib')
openssl_include_dir = os.path.join(self.openssl, 'include')
self.library_dirs.append(openssl_library_dir)
self.include_dirs.append(openssl_include_dir)
log.debug('self.include_dirs = %s', self.include_dirs)
log.debug('self.library_dirs = %s', self.library_dirs)
if platform.system() == "Linux":
# For RedHat-based distros, the '-D__{arch}__' option for
# Swig needs to be normalized, particularly on i386.
mach = platform.machine().lower()
if mach in ('i386', 'i486', 'i586', 'i686'):
arch = '__i386__'
elif mach in ('ppc64', 'powerpc64', 'ppc64le', 'ppc64el'):
arch = '__powerpc64__'
elif mach in ('ppc', 'powerpc'):
arch = '__powerpc__'
else:
arch = '__%s__' % mach
self.swig_opts.append('-D%s' % arch)
if mach in ('ppc64le', 'ppc64el'):
self.swig_opts.append('-D_CALL_ELF=2')
self.swig_opts.extend(['-I%s' % i for i in self.include_dirs])
# Some Linux distributor has added the following line in
# /usr/include/openssl/opensslconf.h:
#
# #include "openssl-x85_64.h"
#
# This is fine with C compilers, because they are smart enough to
# handle 'local inclusion' correctly. Swig, on the other hand, is
# not as smart, and needs to be told where to find this file...
#
# Note that this is risky workaround, since it takes away the
# namespace that OpenSSL uses. If someone else has similarly
# named header files in /usr/include, there will be clashes.
if self.openssl is None:
self.swig_opts.append('-I/usr/include/openssl')
else:
self.swig_opts.append('-I' + os.path.join(openssl_include_dir, 'openssl'))
# swig seems to need the default header file directories
self.swig_opts.extend(['-I%s' % i for i in _get_additional_includes()])
self.swig_opts.append('-includeall')
self.swig_opts.append('-modern')
self.swig_opts.append('-builtin')
# These two lines are a workaround for
# http://bugs.python.org/issue2624 , hard-coding that we are only
# building a single extension with a known path; a proper patch to
# distutils would be in the run phase, when extension name and path are
# known.
self.swig_opts.extend(['-outdir',
os.path.join(os.getcwd(), 'M2Crypto')])
self.include_dirs.append(os.path.join(os.getcwd(), 'SWIG'))
if sys.platform == 'cygwin' and self.openssl is not None:
# Cygwin SHOULD work (there's code in distutils), but
# if one first starts a Windows command prompt, then bash,
# the distutils code does not seem to work. If you start
# Cygwin directly, then it would work even without this change.
# Someday distutils will be fixed and this won't be needed.
self.library_dirs += [os.path.join(self.openssl, 'bin')]
mkpath(os.path.join(self.build_lib, 'M2Crypto'))
def write_rev_file():
mkpath(root_dir)
with open(rev_file_name, 'w') as rev_file:
rev_file.write(rev_value)
def run(self):
log.info("Loading checksums.")
libfgb_pkgdir = "pkgs/libfgb"
checksums_file = os.path.join(libfgb_pkgdir, "checksums.ini")
with open(checksums_file, 'r') as f:
checksums = dict(s.split('=') for s in f.read().split('\n') if s)
log.info("Checking if upstream tar file exists.")
if not os.path.exists("upstream"):
mkpath("upstream")
libfgb_file = os.path.join("upstream", checksums["tarball"])
if not os.path.exists(libfgb_file):
log.warn(
"File %s does not exist. Attempting to download from %s." %
(libfgb_file, UPSTREAM_TAR_URL))
# We use curl to avoid [SSL: CERTIFICATE_VERIFY_FAILED] errors
os.system("curl --insecure --create-dirs -o %s %s" %
(libfgb_file, UPSTREAM_TAR_URL))
if not os.path.exists(libfgb_file):
log.error(
"""Download failed. You may wish to download the file "%s" manually from "%s" and place it in the "upstream/" directory."""
% (checksums["tarball"], UPSTREAM_TAR_URL))
sys.exit(1)
if sha1sum(libfgb_file) != checksums["sha1"]:
log.error("Checksum for file %s is different." % libfgb_file)
sys.exit(1)
log.info("Creating directories.")
if os.path.exists("local"):
remove_tree("local")
mkpath("local/include")
mkpath("local/lib")
tmpdir = "local/var/tmp"
mkpath(tmpdir)
log.info("Extracting tar file.")
import tarfile
tar = tarfile.open(libfgb_file)
tar.extractall(tmpdir)
tar.close()
libfgb_builddir = os.path.join(tmpdir, UPSTREAM_TAR_BASEDIR)
if not os.path.exists(libfgb_builddir):
log.error("Failed to extract files properly.")
sys.exit(1)
log.info("Applying patches.")
os.chdir(libfgb_builddir)
patches_dir = os.path.join(cwd, libfgb_pkgdir, "patches")
patchfiles = [
os.path.join(patches_dir, s)
for s in sorted(os.listdir(patches_dir)) if s.endswith('.patch')
]
for p in patchfiles:
log.info("Applying %s" % p)
if os.system("patch -p1 < '%s'" % p):
log.error("Failed to apply patches.")
sys.exit(1)
os.chdir(cwd)
log.info("Copying include and lib files.")
if UNAME == "Darwin":
FGB_LIBDIR = "macosx"
elif UNAME == "Linux":
FGB_LIBDIR = "x64"
else:
log.error(
"Error installing libfgb: libfgb is not available for this platform."
)
sys.exit(1)
for f in [
"nv/int/protocol_maple.h", "nv/maple/C/call_fgb.h",
"nv/maple/C/call_fgb_basic.h"
]:
copy_file(os.path.join(libfgb_builddir, f), "local/include")
copy_tree(os.path.join(libfgb_builddir, "nv/protocol"),
"local/include")
copy_tree(os.path.join(libfgb_builddir, "nv/maple/C", FGB_LIBDIR),
"local/lib")
print ("OK.")
print ("Compiling BBarolo... ",end="")
sys.stdout.flush()
ret = subprocess.call("make -j%i lib"%mpr.cpu_count(), shell=True, stdout=f)
if ret!=0: sys.exit("\nCompilation failed. Check %s for errors.\n"%logfile)
print ("OK.")
if sys.argv[1]=='sdist':
# If we are creating the sdist package, make a tar with BB source
try: remove_tree("pyBBarolo/BBarolo")
except: pass
try: remove_tree("pyBBarolo/BBarolo.tar.gz")
except: pass
mkpath("pyBBarolo/BBarolo")
subprocess.call("cp -r src/ pyBBarolo/BBarolo/src", shell=True,stdout=f)
subprocess.call("cp -r config/ pyBBarolo/BBarolo/config", shell=True,stdout=f)
subprocess.call("cp -r configure pyBBarolo/BBarolo/", shell=True,stdout=f)
subprocess.call("cp -r Makefile.in pyBBarolo/BBarolo/", shell=True,stdout=f)
subprocess.call("cp -r Makefile.in pyBBarolo/BBarolo/", shell=True,stdout=f)
subprocess.call("rm -rf pyBBarolo/BBarolo/src/Build", shell=True,stdout=f)
subprocess.call("cd pyBBarolo && tar -czvf BBarolo.tar.gz BBarolo", shell=True,stdout=f)
remove_tree("pyBBarolo/BBarolo")
# If we creating the dist, additional file is the tar just created
package_data = {'pyBBarolo': ['*.tar.gz']}
else:
print ("------ Installing pyBBarolo v%s ------"%version)
def build_native_image_image():
image_path = native_image_path(suite_native_image_root())
mx.log('Building native-image executable ' + image_path)
image_dir = dirname(image_path)
mkpath(image_dir)
native_image_on_jvm(['--tool:native-image', '-H:Path=' + image_dir])
def copytree(src,
dst,
ignore_pattern_if_file_exists='*',
preserve_mode=True,
preserve_times=False,
preserve_symlinks=False,
update=False,
verbose=1,
dry_run=0):
"""
A slightly modified version of `distutils.dir_util.copytree`, with the
added parameter `ignore_pattern_if_file_exists` that ignores files if it
matches the given pattern and already exists in the destination directory.
By default, ignores all existing destination files.
"""
if not dry_run and not os.path.isdir(src):
raise DistutilsFileError("cannot copy tree '%s': not a directory" %
src)
try:
names = os.listdir(src)
except OSError as e:
if dry_run:
names = []
else:
raise DistutilsFileError("error listing files in '%s': %s" %
(src, e.strerror)) from e
if not dry_run:
mkpath(dst, verbose=verbose)
outputs = []
outputs_ignored = []
for n in names:
src_name = os.path.join(src, n)
dst_name = os.path.join(dst, n)
if n.startswith('.nfs'):
# skip NFS rename files
continue
if preserve_symlinks and os.path.islink(src_name):
link_dest = os.readlink(src_name)
if verbose >= 1:
log.info("linking %s -> %s", dst_name, link_dest)
if not dry_run:
os.symlink(link_dest, dst_name)
outputs.append(dst_name)
elif os.path.isdir(src_name):
part_outputs, part_outputs_ignored = copytree(
src_name,
dst_name,
ignore_pattern_if_file_exists,
preserve_mode,
preserve_times,
preserve_symlinks,
update,
verbose=verbose,
dry_run=dry_run)
outputs.extend(part_outputs)
outputs_ignored.extend(part_outputs_ignored)
elif not (fnmatch.fnmatch(dst_name, ignore_pattern_if_file_exists)
and os.path.exists(dst_name)):
copy_file(src_name,
dst_name,
preserve_mode,
preserve_times,
update,
verbose=verbose,
dry_run=dry_run)
outputs.append(dst_name)
else:
outputs_ignored.append(dst_name)
return outputs, outputs_ignored
replacements = self._file_replacements(output_fname)
replacements['PERIPHERAL'] = p.to_cpp()
template = string.Template(template_string)
with open(output_fpath, 'w') as f:
print "Writing " + output_fpath + "...",
f.write(template.safe_substitute(replacements))
print "done"
peripheral_header_fnames.append(output_fname)
return peripheral_header_fnames
def generate_header_files(self, output_path=''):
fnames = self._generate_peripheral_files(output_path)
return self._generate_device_file(fnames, output_path)
parser = argparse.ArgumentParser()
parser.add_argument("svd_file", help="SVD input file")
parser.add_argument("out_dir",
help="Directory for generated files",
default='')
args = parser.parse_args()
dev = Device.from_svdfile(args.svd_file, svd_defaults)
if (args.out_dir != ''):
mkpath(args.out_dir)
dev.generate_header_files(args.out_dir)
def draw_results(images,
heats_ground,
heats_result,
pafs_ground,
pafs_result,
masks,
name=''):
"""Save results for debugging.
Parameters
-----------
images : a list of RGB images
heats_ground : a list of keypoint heat maps or None
heats_result : a list of keypoint heat maps or None
pafs_ground : a list of paf vector maps or None
pafs_result : a list of paf vector maps or None
masks : a list of mask for people
"""
# interval = len(images)
for i in range(len(images)):
if heats_ground is not None:
heat_ground = heats_ground[i]
if heats_result is not None:
heat_result = heats_result[i]
if pafs_ground is not None:
paf_ground = pafs_ground[i]
if pafs_result is not None:
paf_result = pafs_result[i]
if masks is not None:
mask = masks[:, :, :, 0]
mask = mask.reshape(hout, wout, 1)
mask1 = np.repeat(mask, n_pos, 2)
mask2 = np.repeat(mask, n_pos * 2, 2)
image = images[i]
fig = plt.figure(figsize=(8, 8))
a = fig.add_subplot(2, 3, 1)
plt.imshow(image)
if pafs_ground is not None:
a = fig.add_subplot(2, 3, 2)
a.set_title('Vectormap_ground')
vectormap = paf_ground * mask2
tmp2 = vectormap.transpose((2, 0, 1))
tmp2_odd = np.amax(np.absolute(tmp2[::2, :, :]), axis=0)
tmp2_even = np.amax(np.absolute(tmp2[1::2, :, :]), axis=0)
# tmp2_odd = tmp2_odd * 255
# tmp2_odd = tmp2_odd.astype(np.int)
plt.imshow(tmp2_odd, alpha=0.3)
# tmp2_even = tmp2_even * 255
# tmp2_even = tmp2_even.astype(np.int)
plt.colorbar()
plt.imshow(tmp2_even, alpha=0.3)
if pafs_result is not None:
a = fig.add_subplot(2, 3, 3)
a.set_title('Vectormap result')
if masks is not None:
vectormap = paf_result * mask2
else:
vectormap = paf_result
tmp2 = vectormap.transpose((2, 0, 1))
tmp2_odd = np.amax(np.absolute(tmp2[::2, :, :]), axis=0)
tmp2_even = np.amax(np.absolute(tmp2[1::2, :, :]), axis=0)
plt.imshow(tmp2_odd, alpha=0.3)
plt.colorbar()
plt.imshow(tmp2_even, alpha=0.3)
if heats_result is not None:
a = fig.add_subplot(2, 3, 4)
a.set_title('Heatmap result')
if masks is not None:
heatmap = heat_result * mask1
else:
heatmap = heat_result
tmp = heatmap
tmp = np.amax(heatmap[:, :, :-1], axis=2)
plt.colorbar()
plt.imshow(tmp, alpha=0.3)
if heats_ground is not None:
a = fig.add_subplot(2, 3, 5)
a.set_title('Heatmap ground truth')
if masks is not None:
heatmap = heat_ground * mask1
else:
heatmap = heat_ground
tmp = heatmap
tmp = np.amax(heatmap[:, :, :-1], axis=2)
plt.colorbar()
plt.imshow(tmp, alpha=0.3)
if masks is not None:
a = fig.add_subplot(2, 3, 6)
a.set_title('Mask')
# print(mask.shape, tmp.shape)
plt.colorbar()
plt.imshow(mask[:, :, 0], alpha=0.3)
# plt.savefig(str(i)+'.png',dpi=300)
# plt.show()
mkpath(config.LOG.vis_path)
plt.savefig(os.path.join(config.LOG.vis_path, '%s%d.png' % (name, i)),
dpi=300)
def mkpath(self, name, mode=0o777):
dir_util.mkpath(name, mode, dry_run=self.dry_run)
"""distutils.archive_util
condition = skipscm
if not dry_run and not os_path_isdir(src):
raise DistutilsFileError, \
"cannot copy tree '%s': not a directory" % src
try:
names = os_listdir(src)
except os.error, (errno, errstr):
if dry_run:
names = []
else:
raise DistutilsFileError, \
"error listing files in '%s': %s" % (src, errstr)
if not dry_run:
mkpath(dst)
outputs = []
for n in names:
src_name = os.path.join(src, n)
dst_name = os.path.join(dst, n)
if (condition is not None) and (not condition(src_name)):
continue
if preserve_symlinks and os_path_islink(src_name):
link_dest = os_readlink(src_name)
log.info("linking %s -> %s", dst_name, link_dest)
if not dry_run:
if update and not newer(src, dst_name):
pass
# Set accepted limit, lim
lims = [1000, 5000, 10000]
distances = []
for dist_measure in ['NRMSE']:
distances.extend(
['{}_{}'.format(t, dist_measure) for t in config['targets']])
distances.append(dist_measure)
# for lim in lims:
for lim in lims:
for d in distances:
print("Working on {}".format(d.upper()))
figPath = "/home/buck06191/Dropbox/phd/Bayesian_fitting/{}/{}/{}/Figures/{}".format(
model_name, DATASET, lim, d)
dir_util.mkpath(figPath)
print("Plotting total histogram")
hist1 = histogram_plot(results, distance=d, frac=1)
hist1.savefig(os.path.join(figPath,
'full_histogram_{}.png'.format(DATASET)),
bbox_inches='tight')
print("Plotting fraction histogram")
hist2 = histogram_plot(results, distance=d, limit=lim)
hist2.savefig(os.path.join(
figPath, 'tol_{}_histogram_{}.png'.format(
str(lim).replace('.', '_'), DATASET)),
bbox_inches='tight')
print("Considering {} lowest values".format(lim))
# print("Generating scatter plot")
# scatter_dist_plot(results, params, tolerance=tol, n_ticks=4)
print("Generating KDE plot")
def do_training(args, module, data_train, data_val, begin_epoch=0):
from distutils.dir_util import mkpath
from log_util import LogUtil
log = LogUtil().getlogger()
mkpath(os.path.dirname(get_checkpoint_path(args)))
#seq_len = args.config.get('arch', 'max_t_count')
batch_size = args.config.getint('common', 'batch_size')
save_checkpoint_every_n_epoch = args.config.getint(
'common', 'save_checkpoint_every_n_epoch')
save_checkpoint_every_n_batch = args.config.getint(
'common', 'save_checkpoint_every_n_batch')
enable_logging_train_metric = args.config.getboolean(
'train', 'enable_logging_train_metric')
enable_logging_validation_metric = args.config.getboolean(
'train', 'enable_logging_validation_metric')
contexts = parse_contexts(args)
num_gpu = len(contexts)
eval_metric = STTMetric(batch_size=batch_size,
num_gpu=num_gpu,
is_logging=enable_logging_validation_metric,
is_epoch_end=True)
# tensorboard setting
loss_metric = STTMetric(batch_size=batch_size,
num_gpu=num_gpu,
is_logging=enable_logging_train_metric,
is_epoch_end=False)
optimizer = args.config.get('optimizer', 'optimizer')
learning_rate = args.config.getfloat('train', 'learning_rate')
learning_rate_annealing = args.config.getfloat('train',
'learning_rate_annealing')
mode = args.config.get('common', 'mode')
num_epoch = args.config.getint('train', 'num_epoch')
clip_gradient = args.config.getfloat('optimizer', 'clip_gradient')
weight_decay = args.config.getfloat('optimizer', 'weight_decay')
save_optimizer_states = args.config.getboolean('train',
'save_optimizer_states')
show_every = args.config.getint('train', 'show_every')
optimizer_params_dictionary = json.loads(
args.config.get('optimizer', 'optimizer_params_dictionary'))
kvstore_option = args.config.get('common', 'kvstore_option')
n_epoch = begin_epoch
is_bucketing = args.config.getboolean('arch', 'is_bucketing')
if clip_gradient == 0:
clip_gradient = None
if is_bucketing and mode == 'load':
model_file = args.config.get('common', 'model_file')
model_name = os.path.splitext(model_file)[0]
model_num_epoch = int(model_name[-4:])
model_path = 'checkpoints/' + str(model_name[:-5])
symbol, data_names, label_names = module(1600)
model = STTBucketingModule(
sym_gen=module,
default_bucket_key=data_train.default_bucket_key,
context=contexts)
data_train.reset()
model.bind(data_shapes=data_train.provide_data,
label_shapes=data_train.provide_label,
for_training=True)
_, arg_params, aux_params = mx.model.load_checkpoint(
model_path, model_num_epoch)
model.set_params(arg_params, aux_params)
module = model
else:
module.bind(data_shapes=data_train.provide_data,
label_shapes=data_train.provide_label,
for_training=True)
if begin_epoch == 0 and mode == 'train':
module.init_params(initializer=get_initializer(args))
lr_scheduler = SimpleLRScheduler(learning_rate=learning_rate)
def reset_optimizer(force_init=False):
optimizer_params = {
'lr_scheduler': lr_scheduler,
'clip_gradient': clip_gradient,
'wd': weight_decay
}
optimizer_params.update(optimizer_params_dictionary)
module.init_optimizer(kvstore=kvstore_option,
optimizer=optimizer,
optimizer_params=optimizer_params,
force_init=force_init)
if mode == "train":
reset_optimizer(force_init=True)
else:
reset_optimizer(force_init=False)
data_train.reset()
data_train.is_first_epoch = True
#tensorboard setting
tblog_dir = args.config.get('common', 'tensorboard_log_dir')
summary_writer = SummaryWriter(tblog_dir)
while True:
if n_epoch >= num_epoch:
break
loss_metric.reset()
log.info('---------train---------')
for nbatch, data_batch in enumerate(data_train):
module.forward_backward(data_batch)
module.update()
# tensorboard setting
if (nbatch + 1) % show_every == 0:
module.update_metric(loss_metric, data_batch.label)
#summary_writer.add_scalar('loss batch', loss_metric.get_batch_loss(), nbatch)
if (nbatch + 1) % save_checkpoint_every_n_batch == 0:
log.info('Epoch[%d] Batch[%d] SAVE CHECKPOINT', n_epoch,
nbatch)
module.save_checkpoint(
prefix=get_checkpoint_path(args) + "n_epoch" +
str(n_epoch) + "n_batch",
epoch=(int(
(nbatch + 1) / save_checkpoint_every_n_batch) - 1),
save_optimizer_states=save_optimizer_states)
# commented for Libri_sample data set to see only train cer
log.info('---------validation---------')
data_val.reset()
eval_metric.reset()
for nbatch, data_batch in enumerate(data_val):
# when is_train = False it leads to high cer when batch_norm
module.forward(data_batch, is_train=True)
module.update_metric(eval_metric, data_batch.label)
# tensorboard setting
val_cer, val_n_label, val_l_dist, _ = eval_metric.get_name_value()
log.info("Epoch[%d] val cer=%f (%d / %d)", n_epoch, val_cer,
int(val_n_label - val_l_dist), val_n_label)
curr_acc = val_cer
summary_writer.add_scalar('CER validation', val_cer, n_epoch)
assert curr_acc is not None, 'cannot find Acc_exclude_padding in eval metric'
data_train.reset()
data_train.is_first_epoch = False
# tensorboard setting
train_cer, train_n_label, train_l_dist, train_ctc_loss = loss_metric.get_name_value(
)
summary_writer.add_scalar('loss epoch', train_ctc_loss, n_epoch)
summary_writer.add_scalar('CER train', train_cer, n_epoch)
# save checkpoints
if n_epoch % save_checkpoint_every_n_epoch == 0:
log.info('Epoch[%d] SAVE CHECKPOINT', n_epoch)
module.save_checkpoint(prefix=get_checkpoint_path(args),
epoch=n_epoch,
save_optimizer_states=save_optimizer_states)
n_epoch += 1
lr_scheduler.learning_rate = learning_rate / learning_rate_annealing
log.info('FINISH')
def run(self):
# Execute sanity checks
warnings = self._directive_checks()
if warnings:
return warnings
# Fetch builder and environment objects
env = self.state_machine.document.settings.env
builder = env.app.builder
# Determine document directory
document_dir = dirname(env.doc2path(env.docname))
# Load content to render
if not self.arguments:
content = '\n'.join(self.content)
else:
# Source file should be relative to document, or absolute to
# configuration directory.
srcfile = self.arguments[0]
if isabs(srcfile):
srcpath = join(env.app.confdir, relpath(srcfile, start='/'))
else:
srcpath = join(document_dir, srcfile)
if not isfile(srcpath):
warning = self.state_machine.reporter.warning(
'{} directive cannot find file {}'.format(
self._get_directive_name(), srcfile),
line=self.lineno)
return [warning]
with open(srcpath, 'rb') as fd:
content = fd.read().decode('utf-8')
# Execute plantweb call
try:
output, frmt, engine, sha = render(content,
engine=self._get_engine_name())
except Exception:
msg = format_exc()
error = nodes.error(
'',
self.state_machine.reporter.error('',
nodes.literal_block('', msg),
line=self.lineno))
return [error]
# Determine filename
filename = '{}.{}'.format(sha, frmt)
imgpath = join(builder.outdir, builder.imagedir, 'plantweb')
# Create images output folder
log.debug('imgpath set to {}'.format(imgpath))
mkpath(imgpath)
# Write content
filepath = join(imgpath, filename)
with open(filepath, 'wb') as fd:
fd.write(output)
log.debug('Wrote image file {}'.format(filepath))
# Default to align center
if 'align' not in self.options:
self.options['align'] = 'center'
# Determine relative path to image from source document directory
filepath_relative = relpath(filepath, document_dir)
log.debug('Image relative path {}'.format(filepath_relative))
# Run Image directive
self.arguments = [filepath_relative]
return Image.run(self)
"""tuple: original resolution of camera."""
RATIO = original_resolution[0] / original_resolution[1]
"""float: aspect ratio of the camera."""
# Initial parameters and variables
SIZE = screen_size[1] # 640
"""int: height of preview image."""
RESOLUTION = (int(SIZE * RATIO), SIZE)
"""tuple: scaled resolution of preview."""
TODAY = datetime.today().strftime("%d-%m-%y")
"""str: today's date for use in naming directories."""
# CURRENT_DIR = path.abspath(path.dirname(__file__))
BASE_DIR = filedialog.askdirectory(initialdir='/media/pi')
"""str: base directory to put photos."""
PHOTO_DIR = path.join(BASE_DIR, "photos", TODAY)
"""str: directory to store images in."""
dir_util.mkpath(PHOTO_DIR)
print("Base directory set to {}".format(PHOTO_DIR))
current_dirs = os.listdir(PHOTO_DIR)
"""list: List of current directories inside `PHOTO_DIR`."""
def get_uid(current_dirs):
""" Get current user ID.
Args:
current_dirs (list of str): List of directories currently in PHOTO_DIR.
Returns:
current_uid (int): Current user ID
"""
# current_uid = 1
if len(current_dirs) != 0:
def build_library(
self,
library,
pkg_config_name,
local_source=None,
supports_non_srcdir_builds=True,
):
log.info("checking if library '%s' is installed", library)
try:
build_args = self.pkgconfig(pkg_config_name)
log.info("found '%s' installed, using it", library)
except CalledProcessError:
# If local_source is not specified, then immediately fail.
if local_source is None:
raise DistutilsExecError("library '%s' is not installed", library)
log.info("building library '%s' from source", library)
env = self.env()
# Determine which compilers we are to use, and what flags.
# This is based on what distutils.sysconfig.customize_compiler()
# does, but that function has a problem that it doesn't produce
# necessary (e.g. architecture) flags for C++ compilers.
cc, cxx, opt, cflags = get_config_vars("CC", "CXX", "OPT", "CFLAGS")
cxxflags = cflags
if "CC" in env:
cc = env["CC"]
if "CXX" in env:
cxx = env["CXX"]
if "CFLAGS" in env:
cflags = opt + " " + env["CFLAGS"]
if "CXXFLAGS" in env:
cxxflags = opt + " " + env["CXXFLAGS"]
# Use a subdirectory of build_temp as the build directory.
build_temp = os.path.realpath(os.path.join(self.build_temp, library))
# Destination for headers and libraries is build_clib.
build_clib = os.path.realpath(self.build_clib)
# Create build directories if they do not yet exist.
mkpath(build_temp)
mkpath(build_clib)
if not supports_non_srcdir_builds:
self._stage_files_recursive(local_source, build_temp)
# Run configure.
cmd = [
"/bin/sh",
os.path.join(os.path.realpath(local_source), "configure"),
"--prefix=" + build_clib,
"--disable-shared",
"--with-pic",
"--disable-maintainer-mode",
]
log.info("%s", " ".join(cmd))
check_call(
cmd,
cwd=build_temp,
env=dict(env, CC=cc, CXX=cxx, CFLAGS=cflags, CXXFLAGS=cxxflags),
)
# Run make install.
cmd = ["make", "install"]
log.info("%s", " ".join(cmd))
check_call(cmd, cwd=build_temp, env=env)
build_args = self.pkgconfig(pkg_config_name)
return build_args
def mkpath(self, name, mode=511):
mkpath(name, mode, dry_run=self.dry_run)
def copy_tree(
src,
dst,
preserve_mode=1,
preserve_times=1,
preserve_symlinks=0,
update=0,
verbose=0,
dry_run=0,
condition=None,
):
"""
Copy an entire directory tree 'src' to a new location 'dst'. Both
'src' and 'dst' must be directory names. If 'src' is not a
directory, raise DistutilsFileError. If 'dst' does not exist, it is
created with 'mkpath()'. The end result of the copy is that every
file in 'src' is copied to 'dst', and directories under 'src' are
recursively copied to 'dst'. Return the list of files that were
copied or might have been copied, using their output name. The
return value is unaffected by 'update' or 'dry_run': it is simply
the list of all files under 'src', with the names changed to be
under 'dst'.
'preserve_mode' and 'preserve_times' are the same as for
'copy_file'; note that they only apply to regular files, not to
directories. If 'preserve_symlinks' is true, symlinks will be
copied as symlinks (on platforms that support them!); otherwise
(the default), the destination of the symlink will be copied.
'update' and 'verbose' are the same as for 'copy_file'.
"""
assert isinstance(src, (str, unicode)), repr(src)
assert isinstance(dst, (str, unicode)), repr(dst)
from distutils import log
from distutils.dep_util import newer
from distutils.dir_util import mkpath
from distutils.errors import DistutilsFileError
src = fsencoding(src)
dst = fsencoding(dst)
if condition is None:
condition = skipscm
if not dry_run and not zipio.isdir(src):
raise DistutilsFileError("cannot copy tree '%s': not a directory" %
src)
try:
names = zipio.listdir(src)
except os.error as exc:
(errno, errstr) = exc.args
if dry_run:
names = []
else:
raise DistutilsFileError("error listing files in '%s': %s" %
(src, errstr))
if not dry_run:
mkpath(dst)
outputs = []
for n in names:
src_name = os.path.join(src, n)
dst_name = os.path.join(dst, n)
if (condition is not None) and (not condition(src_name)):
continue
# Note: using zipio's internal _locate function throws an IOError on
# dead symlinks, so handle it here.
if os.path.islink(src_name) and not os.path.exists(
os.readlink(src_name)):
continue
if preserve_symlinks and zipio.islink(src_name):
link_dest = zipio.readlink(src_name)
log.info("linking %s -> %s", dst_name, link_dest)
if not dry_run:
if update and not newer(src, dst_name):
pass
else:
make_symlink(link_dest, dst_name)
outputs.append(dst_name)
elif zipio.isdir(src_name) and not os.path.isfile(src_name):
# ^^^ this odd tests ensures that resource files that
# happen to be a zipfile won't get extracted.
outputs.extend(
copy_tree(
src_name,
dst_name,
preserve_mode,
preserve_times,
preserve_symlinks,
update,
dry_run=dry_run,
condition=condition,
))
else:
copy_file(
src_name,
dst_name,
preserve_mode,
preserve_times,
update,
dry_run=dry_run,
)
outputs.append(dst_name)
return outputs
def run(self):
"""Run the distutils command"""
log.info("installing library code to %s" % self.bdist_dir)
log.info("generating INFO_SRC and INFO_BIN files")
generate_info_files()
self.dist_name = get_dist_name(self.distribution,
source_only_dist=True,
python_version=get_python_version()[0])
self.dist_target = os.path.join(self.dist_dir, self.dist_name)
log.info("distribution will be available as '%s'" % self.dist_target)
# build command: just to get the build_base
cmdbuild = self.get_finalized_command("build")
self.build_base = cmdbuild.build_base
# install command
install = self.reinitialize_command('install_lib',
reinit_subcommands=1)
install.compile = False
install.warn_dir = 0
install.install_dir = self.bdist_dir
log.info("installing to %s" % self.bdist_dir)
self.run_command('install_lib')
# install extra files
extra_files = {}
for src, dest in extra_files.items():
self.copy_file(src, dest)
# install_egg_info command
cmd_egginfo = self.get_finalized_command('install_egg_info')
cmd_egginfo.install_dir = self.bdist_dir
self.run_command('install_egg_info')
# we need the py2.x converted to py2 in the filename
old_egginfo = cmd_egginfo.get_outputs()[0]
new_egginfo = old_egginfo.replace('-py' + sys.version[:3],
'-py' + get_python_version()[0])
move_file(old_egginfo, new_egginfo)
# create distribution
info_files = [
('README.txt', 'README.txt'),
('LICENSE.txt', 'LICENSE.txt'),
('README.rst', 'README.rst'),
('CONTRIBUTING.rst', 'CONTRIBUTING.rst'),
('docs/INFO_SRC', 'INFO_SRC'),
('docs/INFO_BIN', 'INFO_BIN'),
]
copy_tree(self.bdist_dir, self.dist_target)
pkg_info = mkpath(os.path.join(self.dist_target))
for src, dst in info_files:
if dst is None:
copy_file(src, self.dist_target)
else:
copy_file(src, os.path.join(self.dist_target, dst))
if not self.keep_temp:
remove_tree(self.build_base, dry_run=self.dry_run)
