def find():
# type: () -> str
if sys.platform == "win32":
return which('ffmpeg.exe') or which('bin/ffmpeg.exe')
else:
# I don't know if this works.
return which('ffmpeg') or which('bin/ffmpeg')
def run_command(config):
'''
Run config.cmd. If the command has a TARGET, replace it with config.target.
Else append config.target as an argument.
'''
appcmd = config.cmd
# Split the command into an array of words
if isinstance(appcmd, six.string_types):
appcmd = shlex.split(appcmd)
# If the app is a Cygwin app, TARGET should be a Cygwin path too.
target = config.target
cygcheck, cygpath, kwargs = which('cygcheck'), which('cygpath'), {'universal_newlines': True}
if cygcheck is not None and cygpath is not None:
app_path = check_output([cygpath, '-au', which(appcmd[0])], **kwargs).strip() # nosec
is_cygwin_app = check_output([cygcheck, '-f', app_path], **kwargs).strip() # nosec
if is_cygwin_app:
target = check_output([cygpath, '-au', target], **kwargs).strip() # n osec
# Replace TARGET with the actual target
if 'TARGET' in appcmd:
appcmd = [target if arg == 'TARGET' else arg for arg in appcmd]
else:
appcmd.append(target)
app_log.info('Running %s', ' '.join(appcmd))
if not safe_rmtree(config.target):
app_log.error('Cannot delete target %s. Aborting installation', config.target)
return
proc = Popen(appcmd, bufsize=-1, stdout=sys.stdout, stderr=sys.stderr, **kwargs) # nosec
proc.communicate()
return proc.returncode
def is_available(self):
if not which(self.latexmk):
return False
if self.variant == 'pdflatex':
return bool(which(self.pdflatex))
if self.variant == 'xelatex':
return bool(which(self.xelatex))
def is_available(self):
if not which(self.latexmk):
return False
if self.variant == 'pdflatex':
return bool(which(self.pdflatex))
if self.variant == 'xelatex':
return bool(which(self.xelatex))
def system_information(handler):
'''Handler for system info'''
value, error = {}, {}
try:
import psutil
process = psutil.Process(os.getpid())
value['system', 'cpu-count'] = psutil.cpu_count()
value['system', 'cpu-usage'] = psutil.cpu_percent()
value['system', 'memory-usage'] = psutil.virtual_memory().percent
value['system', 'disk-usage'] = psutil.disk_usage('/').percent
value['gramex', 'memory-usage'] = process.memory_info()[0]
value['gramex', 'open-files'] = len(process.open_files())
except ImportError:
app_log.warning('psutil required for system stats')
error['system', 'cpu-count'] = 'psutil not installed'
error['system', 'cpu-usage'] = 'psutil not installed'
error['system', 'memory-usage'] = 'psutil not installed'
error['system', 'disk-usage'] = 'psutil not installed'
error['gramex', 'memory-usage'] = 'psutil not installed'
error['gramex', 'open-files'] = 'psutil not installed'
try:
import conda
value['conda', 'version'] = conda.__version__,
except ImportError:
app_log.warning('conda required for conda stats')
error['conda', 'version'] = 'conda not installed'
from shutilwhich import which
value['node', 'path'] = which('node')
value['git', 'path'] = which('git')
from gramex.cache import Subprocess
apps = {
('node', 'version'): Subprocess('node --version', shell=True),
('npm', 'version'): Subprocess('npm --version', shell=True),
('yarn', 'version'): Subprocess('yarn --version', shell=True),
('git', 'version'): Subprocess('git --version', shell=True),
}
for key, proc in apps.items():
stdout, stderr = yield proc.wait_for_exit()
value[key] = stdout.strip()
if not value[key]:
error[key] = stderr.strip()
value['python', 'version'] = '{0}.{1}.{2}'.format(*sys.version_info[:3])
value['python', 'path'] = sys.executable
value['gramex', 'version'] = gramex.__version__
value['gramex', 'path'] = os.path.dirname(gramex.__file__)
import pandas as pd
df = pd.DataFrame({'value': value, 'error': error}).reset_index()
df.columns = ['section', 'key'] + df.columns[2:].tolist()
df = df[['section', 'key', 'value',
'error']].sort_values(['section', 'key'])
df['error'] = df['error'].fillna('')
data = gramex.data.filter(df, handler.args)
# TODO: handle _format, _meta, _download, etc just like FormHandler
raise Return(gramex.data.download(data))
def grab(self, keys):
'''Grab specific keys'''
root_window = self.__display.screen().root
keycode = 0
xset_cmd = which("xset")
xmodmap_cmd = which("xmodmap")
if not xset_cmd:
logging.warn(
"xset not found, you might experience some bad key-repeat problems"
)
for key in keys:
keycode = get_keycode(key)
if not keycode:
continue
if xset_cmd:
# FIXME: revert on exit
os.system("%s -r %d" % (xset_cmd, keycode))
if xmodmap_cmd:
cmd_status, cmd_stdout = get_cmd_output("%s -pm" % xmodmap_cmd)
if cmd_status == 0 and cmd_stdout:
lines = cmd_stdout.splitlines()
else:
lines = []
lock_line = [
l.strip().split() for l in lines if l.startswith("lock")
]
num_line = [
l.strip().split() for l in lines if "Num_Lock" in l
]
key_line = [l.strip().split() for l in lines if key in l]
if lock_line:
parts = lock_line[0]
if len(parts) > 1:
self.__caps_lock = parts[1]
if num_line:
parts = num_line[0]
if len(parts) > 1:
self.__num_lock_mod = parts[0]
if key_line:
parts = key_line[0]
if len(parts) > 1:
self.__key_mod = parts[0]
if key == "Caps_Lock":
if not self.__caps_lock:
logging.debug("Caps Lock already disabled!")
else:
self.disable_caps_lock()
atexit.register(self.enable_caps_lock)
ownev = not self.__parent.getModality()
root_window.grab_key(keycode, X.AnyModifier, ownev,
X.GrabModeAsync, X.GrabModeAsync)
return key, keycode
logging.critical("Couldn't find quasimode key")
self.__parent.stop()
return None, None
def skip_if_missing_python_interpreter(python_version_enum):
"""Skip test if missing the necessary version of the Python interpreter"""
file_name = python_version_enum.executable_name
if not shutilwhich.which(file_name):
message = "Missing the necessary version of the Python interpreter: {}".format(
file_name)
pytest.skip(message)
def create_template(self, command, params):
"""
:type command: list[str]
:type params: dist[str, object]
:rtype: drmaa.JobTemplate
"""
jt = self._session.createJobTemplate()
jt.remoteCommand = shutil.which(command[0])
jt.args = command[1:]
if params['num_slots'] > 1:
jt.nativeSpecification = "-pe make {}".format(params['num_slots'])
if params['stdout']:
jt.outputPath = params['stdout']
if params['stderr']:
jt.errorPath = params['stderr']
if params['join_streams']:
jt.joinFiles = True
if params['job_name']:
jt.jobName = params['job_name']
if params['work_dir']:
jt.workingDirectory = params['work_dir']
jt.params = params
return jt
def test_init(self):
init([], AttrDict())
source = os.path.join(variables['GRAMEXPATH'], 'apps', 'init',
'default')
self.check_files(source)
# Ensure templates work
with io.open(os.path.join(self.app_dir, 'gramex.yaml'),
encoding='utf-8') as handle:
line = handle.readline().strip()
ok_('don\'t delete this line' in line)
ok_(re.match(r'# Generated by gramex init 1\.\d+', line))
# If Git LFS is present, ensure that it's set up to track assets/**
if which('git-lfs'):
path = os.path.join(self.app_dir, '.gitattributes')
ok_(os.path.exists(path), 'Git LFS worked')
with open(path, encoding='utf-8') as handle:
ok_('assets/**' in handle.read(), 'Git LFS tracks assets/**')
path = os.path.join(self.app_dir, '.gitignore')
with open(path, encoding='utf-8') as handle:
ok_('assets/**' not in handle.read(),
'.gitignore allows assets/**')
# Else, check that .gitignore does not commit assets/**
else:
path = os.path.join(self.app_dir, '.gitignore')
with open(path, encoding='utf-8') as handle:
ok_('assets/**' in handle.read(),
'.gitignore allows assets/**')
def run(self):
s = None
try:
s = open_command_impl.get_shortcut(self.target)
print s
except Exception:
display_xml_message(u"<p>This shortcut can't be found</p>")
else:
msg = u"Target info of 'open' shortcut '%s'" % self.target
result = "\n".join(
(str(self.target), str(s.target), str(s.shortcut_filename)))
RecentResult.get().push_result(result, msg)
display_xml_message(
u"<p>Target of <command>%s</command> is <command>%s</command>, saved in <command>%s</command></p>"
% (str(self.target), str(s.target), s.shortcut_filename))
try:
# TODO: Implement platform independent
import subprocess
from shutilwhich import which
editor_cmd = os.path.expandvars("$VISUAL")
if not editor_cmd:
editor_cmd = which("gvim")
if editor_cmd:
subprocess.Popen(
[editor_cmd, "--nofork", "--", s.shortcut_filename])
except:
pass
def start_psi4_calculation(path, input_file):
print(input_file)
output_file = input_file.replace('dat', 'out')
input_file = os.path.join(path, input_file)
output_file = os.path.join(path, output_file)
psi4_binary = which('psi4', mode=os.X_OK)
cmd = psi4_binary + ' ' + input_file + ' -o ' + output_file + '2>&1'
os.system(cmd)
def start_psi4_calculation(path, input_file):
print(input_file)
output_file = input_file.replace('dat', 'out')
input_file = os.path.join(path, input_file)
output_file = os.path.join(path, output_file)
psi4_binary = which('psi4', mode=os.X_OK)
cmd = psi4_binary + ' ' + input_file + ' -o ' + output_file + '2>&1'
os.system(cmd)
def _find_full_path(self):
"""Get a full path to the compiler executable."""
full_path = which(self.name)
if full_path is None:
full_path = self.name
return full_path
def installed(b):
''' Returns true if an executable named b exists in the current path.
b may also be a list of binaries.
'''
blist = b if isinstance(b, list) else [
b,
]
return all([which(b) for b in blist])
def collect_info():
gb = opts['git_binary']
git_binary = which(gb)
if not git_binary:
issues.error('Could not find git binary: {}.'.format(gb))
info['git_path'] = git_binary
info['git_tag_name'] = info['ref'].tag_name
def start_psi4_calculation(path, input_file, threads=4):
print(input_file)
output_file = input_file.replace('dat', 'out')
input_file = os.path.join(path, input_file)
output_file = os.path.join(path, output_file)
psi4_binary = which('psi4', mode=os.X_OK)
cmd = psi4_binary + ' ' + input_file + ' -o ' + output_file + ' -n' + threads
process = subprocess.Popen(cmd, stderr=subprocess.PIPE, shell=True)
output = process.communicate()
return(cmd, process.wait(), output)
def start_psi4_calculation(path, input_file, threads=4):
print(input_file)
output_file = input_file.replace('dat', 'out')
input_file = os.path.join(path, input_file)
output_file = os.path.join(path, output_file)
psi4_binary = which('psi4', mode=os.X_OK)
cmd = psi4_binary + ' ' + input_file + ' -o ' + output_file + ' -n' + threads
process = subprocess.Popen(cmd, stderr=subprocess.PIPE, shell=True)
output = process.communicate()
return (cmd, process.wait(), output)
def is_installed(self, custom_name=None): name = self.app_name if not custom_name == None: name = custom_name if self.env.is_test and (not name in self.env.pre_installed_apps): return False return not shutilwhich.which(name) == None
def is_installed(self, custom_name=None):
name = self.app_name
if not custom_name == None:
name = custom_name
if self.env.is_test and (not name in self.env.pre_installed_apps):
return False
return not shutilwhich.which(name) == None
def init(args, kwargs):
'''Create Gramex scaffolding files.'''
if len(args) > 1:
app_log.error(show_usage('init'))
return
kwargs.setdefault('target', os.getcwd())
app_log.info('Initializing Gramex project at %s', kwargs.target)
data = {
'appname': os.path.basename(kwargs.target),
'author': _check_output('git config user.name', default='Author'),
'email': _check_output('git config user.email',
default='*****@*****.**'),
'date': datetime.datetime.today().strftime('%Y-%m-%d'),
'version': gramex.__version__,
}
# Ensure that appname is a valid Python module name
appname = slug.module(data['appname'])
if appname[0] not in string.ascii_lowercase:
appname = 'app' + appname
data['appname'] = appname
# Create a git repo. But if git fails, do not stop. Continue with the rest.
try:
_run_console('git init')
except OSError:
pass
# Install Git LFS if available. Set git_lfs=None if it fails, so .gitignore ignores assets/**
data['git_lfs'] = which('git-lfs')
if data['git_lfs']:
try:
_run_console('git lfs install')
_run_console('git lfs track "assets/**"')
except OSError:
data['git_lfs'] = None
# Copy all directories & files (as templates)
source_dir = os.path.join(variables['GRAMEXPATH'], 'apps', 'init')
for root, dirs, files in os.walk(source_dir):
for name in dirs + files:
source = os.path.join(root, name)
relpath = os.path.relpath(root, start=source_dir)
target = os.path.join(kwargs.target, relpath,
name.replace('appname', appname))
_copy(source, target, template_data=data)
for empty_dir in ('img', 'data'):
_mkdir(os.path.join(kwargs.target, 'assets', empty_dir))
# Copy error files as-is (not as templates)
error_dir = os.path.join(kwargs.target, 'error')
_mkdir(error_dir)
for source in glob(
os.path.join(variables['GRAMEXPATH'], 'handlers', '?0?.html')):
target = os.path.join(error_dir, os.path.basename(source))
_copy(source, target)
run_setup(kwargs.target)
def check_command(name):
""" Check command is available on the system.
Args:
name (str): Command name.
Returns:
True if command ``name`` was found on the system.
"""
return which(name) is not None
def flog_gatherer(reactor, temp_dir, flog_binary, request):
out_protocol = _CollectOutputProtocol()
gather_dir = join(temp_dir, 'flog_gather')
process = reactor.spawnProcess(
out_protocol,
flog_binary,
(
'flogtool', 'create-gatherer',
'--location', 'tcp:localhost:3117',
'--port', '3117',
gather_dir,
)
)
pytest.blockon(out_protocol.done)
twistd_protocol = _MagicTextProtocol("Gatherer waiting at")
twistd_process = reactor.spawnProcess(
twistd_protocol,
which('twistd'),
(
'twistd', '--nodaemon', '--python',
join(gather_dir, 'gatherer.tac'),
),
path=gather_dir,
)
pytest.blockon(twistd_protocol.magic_seen)
def cleanup():
try:
twistd_process.signalProcess('TERM')
pytest.blockon(twistd_protocol.exited)
except ProcessExitedAlready:
pass
flog_file = mktemp('.flog_dump')
flog_protocol = _DumpOutputProtocol(open(flog_file, 'w'))
flog_dir = join(temp_dir, 'flog_gather')
flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')]
print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file))
reactor.spawnProcess(
flog_protocol,
flog_binary,
(
'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0])
),
)
pytest.blockon(flog_protocol.done)
request.addfinalizer(cleanup)
with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f:
furl = f.read().strip()
return furl
def add_yarn_or_bower(args, fullpath):
if args.yarn:
print("Adding yarn dependencies...")
yarn = args.yarn.split(',')
yarn_exe = which('yarn')
if yarn_exe:
os.chdir(os.path.join(fullpath, 'project', 'client', 'static'))
output, error = subprocess.Popen([yarn_exe, 'init', '-y'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=fullpath).communicate()
if error:
print("An error occurred at init Yarn, please check the "
"package.json file.")
print(error.decode('ascii'))
for dependency in yarn:
output, error = subprocess.Popen([yarn_exe, 'add', dependency],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=fullpath).communicate()
if error:
print("An error occurred with Yarn")
print(error.decode('ascii'))
else:
print("Could not find yarn. Ignoring.")
elif args.bower:
print("Adding bower dependencies...")
bower = args.bower.split(',')
bower_exe = which('bower')
if bower_exe:
os.chdir(os.path.join(fullpath, 'project', 'client', 'static'))
for dependency in bower:
output, error = subprocess.Popen(
[bower_exe, 'install', dependency],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if error:
print("An error occurred with Bower")
print(error)
else:
print("Could not find bower. Ignoring.")
def find_program(env_variable, names):
"""Find an executable in the env_variable environment variable or in PATH in
with one of the names in the argument names."""
global search_paths
for name in chain([os.environ.get(env_variable, "")], names):
path = which(name, path=search_paths)
if path:
return path
raise RuntimeError("Couldn't find " + env_variable +
" or any of the following executables in PATH: " +
" ".join(names))
def add_virtualenv(fullpath):
"""Create a python virtual environment named flask."""
if args.virtualenv == "2.7" or args.virtualenv == "3.3":
virtualenv_exe = which(''.join(['virtualenv-', args.virtualenv]))
else:
virtualenv_exe = which(''.join(['pyvenv-', args.virtualenv]))
if virtualenv_exe:
output, error = Popen(
[virtualenv_exe, os.path.join(fullpath, 'flask')],
stdout=PIPE,
stderr=PIPE
).communicate()
if error:
with open('virtualenv_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
print('An error occurred with virtualenv!')
sys.exit(2)
venv_bin = os.path.join(fullpath, 'flask/bin')
print('Adding application requirements...')
output, error = Popen(
[
os.path.join(venv_bin, 'pip'),
'install',
'-r',
os.path.join(fullpath, 'requirements.txt')
],
stdout=PIPE,
stderr=PIPE
).communicate()
if error:
with open('pip_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
sys.exit(2)
else:
print('Could not find a valid virtualenv executable. '
'Ignoring the request for installing a '
'virtual environment...')
def js(self, code=None, path=None, **kwargs):
if self.conn is None:
try:
self.conn = yield websocket_connect(
self.url, connect_timeout=self.timeout)
except OSError as exc:
import errno
if exc.errno != errno.ECONNREFUSED:
raise
# TODO: node_path
self.proc = yield daemon(
[which('node'), self._path,
'--port=%s' % self.port],
first_line=re.compile('pynode: 1.\d+.\d+ port: %s' %
self.port),
cwd=self.cwd,
)
self.conn = yield websocket_connect(
self.url, connect_timeout=self.timeout)
# code= takes preference over path=
if code is not None:
kwargs['code'] = code
elif path is not None:
kwargs['path'] = path
# Send the commands. If node has died, clear the connection.
try:
yield self.conn.write_message(json.dumps(kwargs))
except WebSocketClosedError:
self.conn = None
raise
# Receive the response.
# Note: read_message() cannot be called again while a request is running.
# (Yes, that's odd. Maybe Anand is missing something.)
# So wait until the read_future is cleared.
while self.conn.read_future is not None:
yield sleep(self._delay)
msg = yield self.conn.read_message()
# If node has died, clear the connection to restart it.
if msg is None:
self.conn = None
raise WebSocketClosedError()
# Parse the result as JSON. Log errors if any
result = json.loads(msg)
if result['error']:
app_log.error(result['error']['stack'])
raise Return(result)
def check_conversion_tools(do_il=False):
# Check that the conversion tools are available
if do_il:
input_files = six.itervalues(INPUT_FILES)
else:
input_files = (f for f in six.itervalues(INPUT_FILES) if f['type'] != 'il')
for input_file in input_files:
tool = input_file['conversion_tool']
if shutilwhich.which(tool) is None:
error_message = "Failed to find conversion tool: {}".format(tool)
logging.error(error_message)
raise OasisException(error_message)
return True
def __init__(self, *args, **kwargs):
super(Smartmon, self).__init__(*args)
if "DeviceList" in kwargs:
self.devices = kwargs.get("DeviceList")().devices
else:
if which("smartctl") is None:
sys.exit("smartctl needs to be installed")
self.devices = []
for device in DeviceList().devices:
if not device.supports_smart:
LOG.warning("Skipping %s. SMART is not enabled.", device.path)
else:
self.devices.append(device)
if len(self.devices) == 0:
sys.exit("No devices detected. Check permissions. Hint run: 'smartctl --scan'")
def add_bower(bower, skeleton, fullpath):
"""Install the selected bower dependencies."""
bower_exe = which('bower')
if bower_exe:
os.chdir(os.path.join(fullpath, skeletons[skeleton], 'static'))
for dependency in bower:
output, error = Popen(
[bower_exe, 'install', dependency],
stdout=PIPE,
stderr=PIPE
).communicate()
if error:
print('An error occurred with Bower!')
print(error)
else:
print('Could not find bower. '
'Ignoring the request for bower '
'dependencies installation...')
def scan_python_versions():
"""Scan for the python versions existing in the system.
It will limit the choices of python virtual environments to install.
It is assumed that the python binayries are in /usr/bin although some
systems have them in /usr/local/bin, or in any other place,
so we shall get the directory by using `which` to locate
those existing in the system.
"""
# start defining a list of python versions we accept
# and then find those existing in the system
acceptable_versions = ['python2.7', 'python3.3', 'python3.4', 'python3.5']
py_versions_found = []
for py_version in acceptable_versions:
py_version = which(py_version)
if py_version:
# we just need the number version, so we strip the rest
py_version = ntpath.basename(py_version)[6:]
py_versions_found.append(py_version)
return py_versions_found
def run_setup(target):
'''
Install any setup file in target directory. Target directory can be:
- An absolute path
- A relative path to current directory
- A relative path to the Gramex apps/ folder
Returns the absolute path of the final target path.
This supports:
- ``make`` (if Makefile exists)
- ``powershell -File setup.ps1``
- ``bash setup.sh``
- ``pip install -r requirements.txt``
- ``python setup.py``
- ``yarn install`` else ``npm install``
- ``bower --allow-root install``
'''
if not os.path.exists(target):
app_target = os.path.join(variables['GRAMEXPATH'], 'apps', target)
if not os.path.exists(app_target):
raise OSError('No directory %s' % target)
target = app_target
target = os.path.abspath(target)
app_log.info('Setting up %s', target)
for file, runners in setup_paths.items():
setup_file = os.path.join(target, file)
if not os.path.exists(setup_file):
continue
for exe, cmd in runners.items():
exe_path = which(exe)
if exe_path is not None:
cmd = cmd.format(FILE=setup_file, EXE=exe_path)
app_log.info('Running %s', cmd)
_run_console(cmd, cwd=target)
break
else:
app_log.warning('Skipping %s. No %s found', setup_file, exe)
def get_mysql_version(cls):
'''
Return a tuple: (major, minor).
Example, for MySQL 5.7.15, return (5,7).
Return (None,None) if version number not found.
'''
# Where is mysql client program?
mysql_path = which('mysql')
# Get version string, which looks like this:
# 'Distrib 5.7.15, for osx10.11 (x86_64) using EditLine wrapper\n'
version_str = subprocess.check_output([mysql_path, '--version'])
# Isolate the major and minor version numbers (e.g. '5', and '7')
pat = re.compile(r'Distrib ([0-9]*)[.]([0-9]*)[.]')
match_obj = pat.search(version_str)
if match_obj is None:
return (None,None)
(major, minor) = match_obj.groups()
return (int(major), int(minor))
def test_setup(self):
subprocess.call([sys.executable, '-m', 'pip', 'uninstall', '-y', '-r', self.req_path])
install(['setup'], AttrDict(url=self.install_path))
result = set()
for root, dirs, files in os.walk(self.install_path):
for filename in files:
path = os.path.join(root, filename)
result.add(os.path.relpath(path, self.install_path))
# See http://go.microsoft.com/fwlink/?LinkID=135170
# Requires: Set-ExecutionPolicy -ExecutionPolicy RemoteSigned
if which('powershell'):
result.add('powershell-setup.txt')
if which('make'):
result.add('makefile-setup.txt')
if which('bash'):
result.add('bash-setup.txt')
if which('python'):
result.add('python-setup.txt')
if which('yarn'):
result.add('yarn.lock')
result.add('node_modules/.yarn-integrity')
result.add('node_modules/gramex-npm-package/package.json')
result.add('node_modules/gramex-npm-package/npm-setup.js')
elif which('npm'):
# package-lock.json needs node 8.x -- which is required for CaptureHandler anyway
result.add('package-lock.json')
if which('bower'):
result.add('bower_components/gramex-bower-package/bower.json')
result.add('bower_components/gramex-bower-package/bower-setup.txt')
result.add('bower_components/gramex-bower-package/.bower.json')
if which('pip'):
import dicttoxml # noqa
self.check_files('setup', result)
self.check_uninstall('setup')
def check_conversion_tools(il=False):
"""
Check that the conversion tools are available
:param il: Flag whether to check insured loss tools
:type il: bool
:return: True if all required tools are present, False otherwise
:rtype: bool
"""
if il:
input_files = INPUT_FILES.values()
else:
input_files = (f for f in INPUT_FILES.values() if f['type'] != 'il')
for input_file in input_files:
tool = input_file['conversion_tool']
if shutilwhich.which(tool) is None:
error_message = "Failed to find conversion tool: {}".format(tool)
logging.error(error_message)
raise OasisException(error_message)
return True
def test_init(self):
if os.path.exists(self.app_dir):
shutil.rmtree(self.app_dir, onerror=_ensure_remove)
os.makedirs(self.app_dir)
os.chdir(self.app_dir)
init([], AttrDict())
# Ensure files are present
source = os.path.join(variables['GRAMEXPATH'], 'apps', 'init')
for path in os.listdir(source):
path = path.replace('appname', self.appname.replace('-', '_'))
ok_(os.path.exists(os.path.join(self.app_dir, path)),
path + ' in init')
# Ensure templates work
with io.open(os.path.join(self.app_dir, 'gramex.yaml'),
encoding='utf-8') as handle:
line = handle.readline().strip()
ok_('don\'t delete this line' in line)
ok_(re.match(r'# Generated by gramex init 1\.\d+', line))
# If Git LFS is present, ensure that it's set up to track assets/**
if which('git-lfs'):
path = os.path.join(self.app_dir, '.gitattributes')
ok_(os.path.exists(path), 'Git LFS worked')
with open(path, encoding='utf-8') as handle:
ok_('assets/**' in handle.read(), 'Git LFS tracks assets/**')
path = os.path.join(self.app_dir, '.gitignore')
with open(path, encoding='utf-8') as handle:
ok_('assets/**' not in handle.read(),
'.gitignore allows assets/**')
# Else, check that .gitignore does not commit assets/**
else:
path = os.path.join(self.app_dir, '.gitignore')
with open(path, encoding='utf-8') as handle:
ok_('assets/**' in handle.read(),
'.gitignore allows assets/**')
def add_git(fullpath):
"""Create a git repository at the project's root directory."""
git_exe = which('git')
if git_exe:
output, error = Popen(
['git', 'init', fullpath],
stdout=PIPE,
stderr=PIPE
).communicate()
if error:
with open('git_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
print('Error with git init!')
sys.exit(2)
print('Adding a `.gitignore` file...')
copyfile(
os.path.join(script_dir, 'templates', '.gitignore'),
os.path.join(fullpath, '.gitignore')
)
else:
print('Could not find git. '
'Ignoring the request for initializing '
'a `.git` repository...')
def skip_if_no_vagrant_executable():
"""Skip the test if you cannot find the vagrant executable"""
if not shutilwhich.which('vagrant'):
pytest.skip("No vagrant executable found")
def flog_binary():
return which('flogtool')
def main(args):
print("\nScaffolding...")
# Variables #
appname = args.appname
fullpath = os.path.join(cwd, appname)
skeleton_dir = args.skeleton
# Tasks #
# Copy files and folders
print("Copying files and folders...")
shutil.copytree(os.path.join(script_dir, skeleton_dir), fullpath)
# Create config.py
print("Creating the config...")
secret_key = codecs.encode(os.urandom(32), 'hex').decode('utf-8')
template = template_env.get_template('config.jinja2')
template_var = {
'secret_key': secret_key,
}
dir_path = os.path.join(fullpath, 'project')
mkdir_p(dir_path)
with open(os.path.join(dir_path, 'config.py'), 'w') as fd:
fd.write(template.render(template_var))
# Add bower dependencies
# NOTE: make sure bower installed
# npm install -g bower
if args.bower:
print("Adding bower dependencies...")
bower = args.bower.split(',')
bower_exe = which('bower')
if bower_exe:
dir_path = os.path.join(fullpath, 'project', 'client', 'static')
mkdir_p(dir_path)
os.chdir(dir_path)
for dependency in bower:
output, error = subprocess.Popen(
[bower_exe, 'install', dependency],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if error:
print("An error occurred with Bower")
print(error)
else:
print("Could not find bower. Ignoring.")
# Add a virtualenv
virtualenv = args.virtualenv
if virtualenv:
print("Adding a virtualenv...")
# NOTE: We should already be in an activated venv from the skeleton
python_exe = "{}".format(which('python'))
if python_exe:
output, error = subprocess.Popen(
[python_exe, "-m", "venv",
os.path.join(fullpath, 'env')],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if error:
with open('virtualenv_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
print("An error occurred with virtualenv")
sys.exit(2)
venv_bin = os.path.join(fullpath, 'env/bin')
output, error = subprocess.Popen(
[
os.path.join(venv_bin, 'pip'), 'install', '-r',
os.path.join(fullpath, 'requirements.txt')
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if error:
with open('pip_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
sys.exit(2)
else:
print("Could not find virtualenv executable. Ignoring")
# Git init
if args.git:
print("Initializing Git...")
output, error = subprocess.Popen(['git', 'init', fullpath],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if error:
with open('git_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
print("Error with git init")
sys.exit(2)
shutil.copyfile(os.path.join(script_dir, 'templates', '.gitignore'),
os.path.join(fullpath, '.gitignore'))
def _initialize(self):
# Count number of plots to create:
num_plots = 0
for config in self._config.itervalues():
num_plots += len(config)
# Set default grid of plot positions:
if not self._rows * self._cols == num_plots:
self._cols = int(np.ceil(np.sqrt(num_plots)))
self._rows = int(np.ceil(num_plots / float(self._cols)))
self.f, self.axarr = plt.subplots(self._rows,
self._cols,
figsize=self._figsize)
# Remove unused subplots:
for i in xrange(num_plots, self._rows * self._cols):
plt.delaxes(self.axarr[np.unravel_index(i,
(self._rows, self._cols))])
cnt = 0
self.handles = []
self.types = []
keywds = ['handle', 'ydata', 'fmt', 'type', 'ids', 'shape', 'norm']
# TODO: Irregular grid in U will make the plot better
U, V = np.mgrid[0:np.pi / 2:complex(0, 60), 0:2 * np.pi:complex(0, 60)]
X = np.cos(V) * np.sin(U)
Y = np.sin(V) * np.sin(U)
Z = np.cos(U)
self._dome_pos_flat = (X.flatten(), Y.flatten(), Z.flatten())
self._dome_pos = (X, Y, Z)
self._dome_arr_shape = X.shape
if not isinstance(self.axarr, np.ndarray):
self.axarr = np.asarray([self.axarr])
for LPU, configs in self._config.iteritems():
for plt_id, config in enumerate(configs):
ind = np.unravel_index(cnt, self.axarr.shape)
cnt += 1
# Some plot types require specific numbers of
# neuron ID arrays:
if 'type' in config:
if config['type'] == 'quiver':
assert len(config['ids']) == 2
config['type'] = 0
elif config['type'] == 'hsv':
assert len(config['ids']) == 2
config['type'] = 1
elif config['type'] == 'image':
assert len(config['ids']) == 1
config['type'] = 2
elif config['type'] == 'waveform':
config['type'] = 3
elif config['type'] == 'raster':
config['type'] = 4
elif config['type'] == 'rate':
config['type'] = 5
elif config['type'] == 'dome':
config['type'] = 6
else:
raise ValueError('Plot type not supported')
else:
if str(LPU).startswith(
'input') and not self._graph[LPU].node[str(
config['ids'][0][0])]['spiking']:
config['type'] = 2
else:
config['type'] = 4
if config['type'] < 3:
if not 'shape' in config:
# XXX This can cause problems when the number
# of neurons is not equal to
# np.prod(config['shape'])
num_neurons = len(config['ids'][0])
config['shape'] = [int(np.ceil(np.sqrt(num_neurons)))]
config['shape'].append(
int(
np.ceil(num_neurons /
float(config['shape'][0]))))
if config['type'] == 0:
config['handle'] = self.axarr[ind].quiver(\
np.reshape(self._data[LPU][config['ids'][0],0],config['shape']),\
np.reshape(self._data[LPU][config['ids'][1],0],config['shape']))
elif config['type'] == 1:
X = np.reshape(self._data[LPU][config['ids'][0], 0],
config['shape'])
Y = np.reshape(self._data[LPU][config['ids'][1], 0],
config['shape'])
V = (X**2 + Y**2)**0.5
H = (np.arctan2(X, Y) + np.pi) / (2 * np.pi)
S = np.ones_like(V)
HSV = np.dstack((H, S, V))
RGB = hsv_to_rgb(HSV)
config['handle'] = self.axarr[ind].imshow(RGB)
elif config['type'] == 2:
if 'trans' in config:
if config['trans'] is True:
to_transpose = True
else:
to_transpose = False
else:
to_transpose = False
config['trans'] = False
if to_transpose:
temp = self.axarr[ind].imshow(np.transpose(np.reshape(\
self._data[LPU][config['ids'][0],0], config['shape'])))
else:
temp = self.axarr[ind].imshow(np.reshape(\
self._data[LPU][config['ids'][0],0], config['shape']))
temp.set_clim(self._imlim)
temp.set_cmap(plt.cm.gist_gray)
config['handle'] = temp
elif config['type'] == 3:
fmt = config['fmt'] if 'fmt' in config else ''
self.axarr[ind].set_xlim(self._xlim)
self.axarr[ind].set_ylim(self._ylim)
if len(config['ids'][0]) == 1:
config['handle'] = self.axarr[ind].plot([0], \
[self._data[LPU][config['ids'][0][0],0]], fmt)[0]
config['ydata'] = [
self._data[LPU][config['ids'][0][0], 0]
]
else:
config['handle'] = self.axarr[ind].plot(
self._data[LPU][config['ids'][0], 0])[0]
elif config['type'] == 4:
config['handle'] = self.axarr[ind]
config['handle'].vlines(0, 0, 0.01)
config['handle'].set_ylim([.5, len(config['ids'][0]) + .5])
config['handle'].set_ylabel('Neurons',
fontsize=self._fontsize - 1,
weight='bold')
config['handle'].set_xlabel('Time (s)',
fontsize=self._fontsize - 1,
weight='bold')
min_id = min(self._id_to_data_idx[LPU].keys())
min_idx = self._id_to_data_idx[LPU][min_id]
config['handle'].set_xlim(
[0, len(self._data[LPU][min_idx, :]) * self._dt])
config['handle'].axes.set_yticks([])
config['handle'].axes.set_xticks([])
elif config['type'] == 6:
self.axarr[ind].axes.set_yticks([])
self.axarr[ind].axes.set_xticks([])
self.axarr[ind] = self.f.add_subplot(self._rows,
self._cols,
cnt,
projection='3d')
config['handle'] = self.axarr[ind]
config['handle'].axes.set_yticks([])
config['handle'].axes.set_xticks([])
config['handle'].xaxis.set_ticks([])
config['handle'].yaxis.set_ticks([])
config['handle'].zaxis.set_ticks([])
if 'norm' not in config.keys():
config['norm'] = Normalize(vmin=-70, vmax=0, clip=True)
elif config['norm'] == 'auto':
if self._data[LPU].shape[1] > 100:
config['norm'] = Normalize(
vmin=np.min(self._data[LPU][config['ids'][0],
100:]),
vmax=np.max(self._data[LPU][config['ids'][0],
100:]),
clip=True)
else:
config['norm'] = Normalize(
vmin=np.min(
self._data[LPU][config['ids'][0], :]),
vmax=np.max(
self._data[LPU][config['ids'][0], :]),
clip=True)
node_dict = self._graph[LPU].node
if str(LPU).startswith('input'):
latpositions = np.asarray([ node_dict[str(nid)]['lat'] \
for nid in range(len(node_dict)) \
if node_dict[str(nid)]['extern'] ])
longpositions = np.asarray([ node_dict[str(nid)]['long'] \
for nid in range(len(node_dict)) \
if node_dict[str(nid)]['extern'] ])
else:
latpositions = np.asarray([
node_dict[str(nid)]['lat']
for nid in config['ids'][0]
])
longpositions = np.asarray([
node_dict[str(nid)]['long']
for nid in config['ids'][0]
])
xx = np.cos(longpositions) * np.sin(latpositions)
yy = np.sin(longpositions) * np.sin(latpositions)
zz = np.cos(latpositions)
config['positions'] = (xx, yy, zz)
colors = griddata(config['positions'],
self._data[LPU][config['ids'][0],
0], self._dome_pos_flat,
'nearest').reshape(self._dome_arr_shape)
colors = config['norm'](colors).data
colors = np.tile(
np.reshape(colors, [
self._dome_arr_shape[0], self._dome_arr_shape[1], 1
]), [1, 1, 4])
colors[:, :, 3] = 1.0
config['handle'].plot_surface(self._dome_pos[0],
self._dome_pos[1],
self._dome_pos[2],
rstride=1,
cstride=1,
facecolors=colors,
antialiased=False,
shade=False)
for key in config.iterkeys():
if key not in keywds:
try:
self._set_wrapper(self.axarr[ind], key,
config[key])
except:
pass
try:
self._set_wrapper(config['handle'], key,
config[key])
except:
pass
if config['type'] < 3:
config['handle'].axes.set_xticks([])
config['handle'].axes.set_yticks([])
if self.suptitle is not None:
self.f.suptitle(self._title,
fontsize=self._fontsize + 1,
x=0.5,
y=0.03,
weight='bold')
plt.tight_layout()
if self.out_filename:
if self.FFMpeg is None:
if which(matplotlib.rcParams['animation.ffmpeg_path']):
self.writer = FFMpegFileWriter(fps=self.fps,
codec=self.codec)
elif which(matplotlib.rcParams['animation.avconv_path']):
self.writer = AVConvFileWriter(fps=self.fps,
codec=self.codec)
else:
raise RuntimeError('cannot find ffmpeg or avconv')
elif self.FFMpeg:
if which(matplotlib.rcParams['animation.ffmpeg_path']):
self.writer = FFMpegFileWriter(fps=self.fps,
codec=self.codec)
else:
raise RuntimeError('cannot find ffmpeg')
else:
if which(matplotlib.rcParams['animation.avconv_path']):
self.writer = AVConvFileWriter(fps=self.fps,
codec=self.codec)
else:
raise RuntimeError('cannot find avconv')
# Use the output file to determine the name of the temporary frame
# files so that two concurrently run visualizations don't clobber
# each other's frames:
self.writer.setup(
self.f,
self.out_filename,
dpi=80,
frame_prefix=os.path.splitext(self.out_filename)[0] + '_')
self.writer.frame_format = 'png'
self.writer.grab_frame()
else:
self.f.show()
def find():
if sys.platform == "win32":
return which('7z.exe') or which('7za.exe') or which('bin/7za.exe')
else:
# I don't know if this works.
return which('7z') or which('7za') or which('bin/7za')
def _initialize(self):
# Count number of plots to create:
num_plots = 0
for config in self._config.itervalues():
num_plots += len(config)
# Set default grid of plot positions:
if not self._rows*self._cols == num_plots:
self._cols = int(np.ceil(np.sqrt(num_plots)))
self._rows = int(np.ceil(num_plots/float(self._cols)))
self.f, self.axarr = plt.subplots(self._rows, self._cols,
figsize=self._figsize)
# Remove unused subplots:
for i in xrange(num_plots, self._rows*self._cols):
plt.delaxes(self.axarr[np.unravel_index(i, (self._rows, self._cols))])
cnt = 0
self.handles = []
self.types = []
keywds = ['handle', 'ydata', 'fmt', 'type', 'ids', 'shape', 'norm']
# TODO: Irregular grid in U will make the plot better
U, V = np.mgrid[0:np.pi/2:complex(0, 60),
0:2*np.pi:complex(0, 60)]
X = np.cos(V)*np.sin(U)
Y = np.sin(V)*np.sin(U)
Z = np.cos(U)
self._dome_pos_flat = (X.flatten(), Y.flatten(), Z.flatten())
self._dome_pos = (X, Y, Z)
self._dome_arr_shape = X.shape
if not isinstance(self.axarr, np.ndarray):
self.axarr = np.asarray([self.axarr])
for LPU, configs in self._config.iteritems():
for plt_id, config in enumerate(configs):
ind = np.unravel_index(cnt, self.axarr.shape)
cnt+=1
# Some plot types require specific numbers of
# neuron ID arrays:
if 'type' in config:
if config['type'] == 'quiver':
assert len(config['ids'])==2
config['type'] = 0
elif config['type'] == 'hsv':
assert len(config['ids'])==2
config['type'] = 1
elif config['type'] == 'image':
assert len(config['ids'])==1
config['type'] = 2
elif config['type'] == 'waveform':
config['type'] = 3
elif config['type'] == 'raster':
config['type'] = 4
elif config['type'] == 'rate':
config['type'] = 5
elif config['type'] == 'dome':
config['type'] = 6
else:
raise ValueError('Plot type not supported')
else:
if str(LPU).startswith('input') and not self._graph[LPU].node[str(config['ids'][0][0])]['spiking']:
config['type'] = 2
else:
config['type'] = 4
if config['type'] < 3:
if not 'shape' in config:
# XXX This can cause problems when the number
# of neurons is not equal to
# np.prod(config['shape'])
num_neurons = len(config['ids'][0])
config['shape'] = [int(np.ceil(np.sqrt(num_neurons)))]
config['shape'].append(int(np.ceil(num_neurons/float(config['shape'][0]))))
if config['type'] == 0:
config['handle'] = self.axarr[ind].quiver(\
np.reshape(self._data[LPU][config['ids'][0],0],config['shape']),\
np.reshape(self._data[LPU][config['ids'][1],0],config['shape']))
elif config['type'] == 1:
X = np.reshape(self._data[LPU][config['ids'][0],0],config['shape'])
Y = np.reshape(self._data[LPU][config['ids'][1],0],config['shape'])
V = (X**2 + Y**2)**0.5
H = (np.arctan2(X,Y)+np.pi)/(2*np.pi)
S = np.ones_like(V)
HSV = np.dstack((H,S,V))
RGB = hsv_to_rgb(HSV)
config['handle'] = self.axarr[ind].imshow(RGB)
elif config['type'] == 2:
if 'trans' in config:
if config['trans'] is True:
to_transpose = True
else:
to_transpose = False
else:
to_transpose = False
config['trans'] = False
if to_transpose:
temp = self.axarr[ind].imshow(np.transpose(np.reshape(\
self._data[LPU][config['ids'][0],0], config['shape'])))
else:
temp = self.axarr[ind].imshow(np.reshape(\
self._data[LPU][config['ids'][0],0], config['shape']))
temp.set_clim(self._imlim)
temp.set_cmap(plt.cm.gist_gray)
config['handle'] = temp
elif config['type'] == 3:
fmt = config['fmt'] if 'fmt' in config else ''
self.axarr[ind].set_xlim(self._xlim)
self.axarr[ind].set_ylim(self._ylim)
if len(config['ids'][0])==1:
config['handle'] = self.axarr[ind].plot([0], \
[self._data[LPU][config['ids'][0][0],0]], fmt)[0]
config['ydata'] = [self._data[LPU][config['ids'][0][0],0]]
else:
config['handle'] = self.axarr[ind].plot(self._data[LPU][config['ids'][0],0])[0]
elif config['type'] == 4:
config['handle'] = self.axarr[ind]
config['handle'].vlines(0, 0, 0.01)
config['handle'].set_ylim([.5, len(config['ids'][0]) + .5])
config['handle'].set_ylabel('Neurons',
fontsize=self._fontsize-1, weight='bold')
config['handle'].set_xlabel('Time (s)',fontsize=self._fontsize-1, weight='bold')
min_id = min(self._id_to_data_idx[LPU].keys())
min_idx = self._id_to_data_idx[LPU][min_id]
config['handle'].set_xlim([0,len(self._data[LPU][min_idx,:])*self._dt])
config['handle'].axes.set_yticks([])
config['handle'].axes.set_xticks([])
elif config['type'] == 6:
self.axarr[ind].axes.set_yticks([])
self.axarr[ind].axes.set_xticks([])
self.axarr[ind] = self.f.add_subplot(self._rows,
self._cols,
cnt,
projection='3d')
config['handle' ] = self.axarr[ind]
config['handle'].axes.set_yticks([])
config['handle'].axes.set_xticks([])
config['handle'].xaxis.set_ticks([])
config['handle'].yaxis.set_ticks([])
config['handle'].zaxis.set_ticks([])
if 'norm' not in config.keys():
config['norm'] = Normalize(vmin=-70, vmax=0, clip=True)
elif config['norm'] == 'auto':
if self._data[LPU].shape[1] > 100:
config['norm'] = Normalize(vmin = np.min(self._data[LPU][config['ids'][0],100:]),
vmax = np.max(self._data[LPU][config['ids'][0],100:]),
clip = True)
else:
config['norm'] = Normalize(vmin = np.min(self._data[LPU][config['ids'][0],:]),
vmax = np.max(self._data[LPU][config['ids'][0],:]),
clip = True)
node_dict = self._graph[LPU].node
if str(LPU).startswith('input'):
latpositions = np.asarray([ node_dict[str(nid)]['lat'] \
for nid in range(len(node_dict)) \
if node_dict[str(nid)]['extern'] ])
longpositions = np.asarray([ node_dict[str(nid)]['long'] \
for nid in range(len(node_dict)) \
if node_dict[str(nid)]['extern'] ])
else:
latpositions = np.asarray([ node_dict[str(nid)]['lat']
for nid in config['ids'][0] ])
longpositions = np.asarray([ node_dict[str(nid)]['long']
for nid in config['ids'][0] ])
xx = np.cos(longpositions) * np.sin(latpositions)
yy = np.sin(longpositions) * np.sin(latpositions)
zz = np.cos(latpositions)
config['positions'] = (xx, yy, zz)
colors = griddata(config['positions'], self._data[LPU][config['ids'][0],0],
self._dome_pos_flat, 'nearest').reshape(self._dome_arr_shape)
colors = config['norm'](colors).data
colors = np.tile(np.reshape(colors,
[self._dome_arr_shape[0],self._dome_arr_shape[1],1])
,[1,1,4])
colors[:,:,3] = 1.0
config['handle'].plot_surface(self._dome_pos[0], self._dome_pos[1],
self._dome_pos[2], rstride=1, cstride=1,
facecolors=colors, antialiased=False,
shade=False)
for key in config.iterkeys():
if key not in keywds:
try:
self._set_wrapper(self.axarr[ind],key, config[key])
except:
pass
try:
self._set_wrapper(config['handle'],key, config[key])
except:
pass
if config['type']<3:
config['handle'].axes.set_xticks([])
config['handle'].axes.set_yticks([])
if self.suptitle is not None:
self.f.suptitle(self._title, fontsize=self._fontsize+1, x=0.5,y=0.03, weight='bold')
plt.tight_layout()
if self.out_filename:
if self.FFMpeg is None:
if which(matplotlib.rcParams['animation.ffmpeg_path']):
self.writer = FFMpegFileWriter(fps=self.fps, codec=self.codec)
elif which(matplotlib.rcParams['animation.avconv_path']):
self.writer = AVConvFileWriter(fps=self.fps, codec=self.codec)
else:
raise RuntimeError('cannot find ffmpeg or avconv')
elif self.FFMpeg:
if which(matplotlib.rcParams['animation.ffmpeg_path']):
self.writer = FFMpegFileWriter(fps=self.fps, codec=self.codec)
else:
raise RuntimeError('cannot find ffmpeg')
else:
if which(matplotlib.rcParams['animation.avconv_path']):
self.writer = AVConvFileWriter(fps=self.fps, codec=self.codec)
else:
raise RuntimeError('cannot find avconv')
# Use the output file to determine the name of the temporary frame
# files so that two concurrently run visualizations don't clobber
# each other's frames:
self.writer.setup(self.f, self.out_filename, dpi=80,
frame_prefix=os.path.splitext(self.out_filename)[0]+'_')
self.writer.frame_format = 'png'
self.writer.grab_frame()
else:
self.f.show()
from pdfminer.layout import LAParams
from pdfminer.pdfpage import PDFPage
from pdfminer.pdfparser import PDFParser
from pdfminer.pdfdocument import PDFDocument
from pdfminer.converter import PDFPageAggregator
from pdfminer.pdfinterp import PDFResourceManager
from pdfminer.pdfinterp import PDFPageInterpreter
from pdfminer.high_level import extract_text_to_fp
from shutilwhich import which
from tornado.web import create_signed_value
import gramex
from gramex.handlers import Capture
from . import TestGramex, server
_captures = {}
paths = {'phantomjs': which('phantomjs'), 'node': which('node')}
def get_capture(name, **kwargs):
'''Return a cached Capture() object constructed with kwargs'''
if name in _captures:
return _captures[name]
capture = _captures[name] = Capture(**kwargs)
end_time, delay = time.time() + 10, 0.05
logging.info('Waiting for capture.js...')
while not capture.started:
if time.time() > end_time:
raise RuntimeError('capture.js took too long to start')
time.sleep(delay)
return capture
def setup_workdir(workdir=".", overwrite_dir=False, backup_config=True):
"""
Setup execution environment and dump default configuration file
Parameters
----------
workdir: str
The directory to store the created dirs and files.
require_empty: bool
when set True, raise RuntimeError if the workdir is not empty
backup_config: bool
when not require_empty, this controls whether the files are
get backed up when an overwriting is about to occur.
"""
logger = logging.getLogger(name="setup")
logdir = 'logs' # stores logs for the apus pipelines
externdir = 'extern' # stores external dependencies
tmpdir = 'tmp' # stores temporary files
configfile = DEFAULT_CONFIG_FILE # main configuration file
# validate the workdir
workdir_content = os.listdir(workdir)
if workdir_content:
if not overwrite_dir:
# skip setup when all files are present
if all(d in workdir_content
for d in [logdir, externdir, tmpdir, configfile]):
raise RuntimeError(
"it seems that the workdir has been setup already. "
"Re-run with -f to proceed anyway")
else:
raise RuntimeError(
"the workdir is not empty, re-run with -f to proceed "
"anyway")
else:
logger.warning(
"the workdir is not empty but a forced setup is requested")
logger.info("setup workdir {}".format(workdir))
# external dependencies
externdir = os.path.join(workdir, externdir)
which_path = os.path.abspath(externdir) + ':' + os.environ['PATH']
if os.path.isdir(externdir):
logger.warning("use existing extern dir {}".format(externdir))
else:
os.makedirs(externdir)
logger.info("create extern dir {}".format(externdir))
logger.info("check external dependencies")
astromatic_prefix = []
for name, cmds, datadir in [("SExtractor", ("sex",
'ldactoasc'), "sextractor"),
("SCAMP", ("scamp", ), "scamp"),
("SWarp", ("swarp", ), "swarp")]:
cmds = [which(cmd, path=which_path) for cmd in cmds]
if any(c is None for c in cmds):
raise RuntimeError("not able to locate {}".format(name))
prefix = os.path.normpath(os.path.join(os.path.dirname(cmds[0]),
'../'))
datadir = os.path.join(prefix, "share", datadir)
if not os.path.exists(datadir):
raise RuntimeError(
"not able to locate data files for {0}. It is likely that {0} "
"is compiled within the source directory but without proper "
"installation. To resolve the issue, either run `make install`"
" in the source directory, or manually link the data "
"directory to {1}.".format(name, datadir))
logger.info("{0:10s} ... OK".format(name))
astromatic_prefix.append(prefix)
if len(set(astromatic_prefix)) > 1:
raise RuntimeError(
"it seems that the SExtractor, SCAMP and SWarp are not installed "
"into the same prefix. {app_name} for now does not deal with this "
"situation. Try to re-configure SExtractor, SCAMP and SWarp with "
"--prefix=<prefixpath>".format(app_name=APP_NAME))
astromatic_prefix = os.path.normpath(astromatic_prefix[0])
logger.info("use shared astromatic prefix {}".format(astromatic_prefix))
stilts_cmd = which("stilts", path=which_path)
if stilts_cmd is None:
logger.warning("not able to find stilts. Get from internet")
# retrieve stilts
from astropy.utils.data import download_file
stilts_jar_tmp = download_file(
"http://www.star.bris.ac.uk/%7Embt/stilts/stilts.jar", cache=True)
stilts_jar = os.path.join(externdir, 'stilts.jar')
shutil.copyfile(stilts_jar_tmp, stilts_jar)
stilts_cmd = os.path.join(externdir, 'stilts')
with open(stilts_cmd, 'w') as fo:
fo.write("""#!/bin/sh
java -Xmx4000M -classpath "{0}:$CLASSPATH" uk.ac.starlink.ttools.Stilts "$@"
""".format(os.path.abspath(stilts_jar)))
os.chmod(stilts_cmd, os.stat(stilts_cmd).st_mode | stat.S_IEXEC)
logger.info("{0:10s} ... OK".format("stilts"))
logger.info("use stilts {}".format(stilts_cmd))
funpack_cmd = which("funpack", path=which_path)
if funpack_cmd is None:
logger.warning("not able to find funpack. Get from internet")
# retrieve stilts
from astropy.utils.data import download_file
funpack_tmp = download_file(
"http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/"
"cfitsio_latest.tar.gz",
cache=True)
funpack_src = os.path.join(externdir, 'cfitsio')
import tarfile
with tarfile.open(funpack_tmp) as tar:
tar.extractall(path=externdir)
logger.warning("try compiling funpack")
import subprocess
try:
for command in [
'./configure', 'make', 'make fpack', 'make funpack'
]:
subprocess.check_call(command.split(), cwd=funpack_src)
except subprocess.CalledProcessError:
raise RuntimeError("unable to compile funpack")
funpack_cmd = os.path.join(externdir, 'funpack')
shutil.copy(os.path.join(externdir, 'cfitsio', 'funpack'), funpack_cmd)
logger.info("{0:10s} ... OK".format("funpack"))
logger.info("use funpack {}".format(funpack_cmd))
# create logging directory
logdir = os.path.join(workdir, logdir)
if os.path.isdir(logdir):
logger.warning("use existing log dir {}".format(logdir))
else:
os.makedirs(logdir)
logger.info("create log dir {}".format(logdir))
# setup scratch space
tmpdir = os.path.join(workdir, tmpdir)
# def freespace_GiB(path):
# stat = os.statvfs(path)
# return stat.f_bfree * stat.f_frsize / 1024 ** 3
# logger.info("{:.2f} GiB free in {}".format(
# freespace_GiB(scratch_dir), scratch_dir))
if os.path.isdir(tmpdir):
logger.warning("use existing tmp dir {}".format(tmpdir))
else:
os.makedirs(tmpdir)
logger.info("create tmp dir {}".format(tmpdir))
# dump default config
time_fmt = "%b-%d-%Y_%H-%M-%S"
base_fmt = ("{obsid}_{object}_{instru}_{band}")
config = """## config file of {app_name:s}
## {time:s}
# calib setting
phot_model_flags: 'color,chip,expo'
# qa inputs
qa_headers:
odi: [
'OBSID', 'CRVAL1', 'CRVAL2', 'OBJECT', 'EXPMEAS', 'AIRMASS',
'SEEING', 'SKY_MEDI', 'SKY_STD',
'FILTER', 'INSTRUME', 'MJD-OBS'
]
decam: [
'OBSID', 'CRVAL1', 'CRVAL2', 'OBJECT', 'EXPTIME', 'AIRMASS',
'FWHM', 'AVSKY', 'SKYSIGMA',
'FILTER', 'INSTRUME', 'MJD-OBS'
]
# naming
reg_arch: '{reg_arch:s}' # regex to parse the filenames from data archive
fmt_orig: '{{imflag}}{base_fmt}.{{ext}}'
reg_orig: '{reg_orig:s}' # regex to parse the filenames in job-in dir
reg_inputs: '{reg_inputs:s}' # regex to parse the filenames in jobdir
fmt_inputs: 'orig_{base_fmt}.{{ext}}' # format string for image in jobdir
sel_inputs: ['orig_*_?.fits', 'orig_*_?.fits.fz']
fmt_masked: 'masked_{base_fmt:s}.fits' # format string of masked images
sel_masked: 'masked_*_?.fits' # selection string of masked images
fmt_selected: '{{ppflag}}{{imflag}}_{base_fmt:s}.fits'
sel_fcomb: 'fcomb[0-9]*_masked_*.fits'
fmt_objcat: '{{ppflag}}objcat_{base_fmt:s}.cat' # format of object catalog
fmt_objmask: '{{ppflag}}objmask_{base_fmt:s}.fits' # format of object masks
sel_objmask: 'fcomb[0-9]*_objmask_*.fits' # selection string of object masks
fmt_sky: '{{ppflag}}sky_{base_fmt:s}.fits' # format string of sky images
fmt_fcomb: '{{ppflag}}combined.fits'
reg_fcomb: '{reg_fcomb:s}' # regex to parse grouped master
fmt_fsmooth: '{{ppflag}}smoothed.fits'
sel_fsub: 'fsub[0-9]*_masked_*.fits'
fmt_fsub_fsmooth: 'fcomb{{grpid}}_smoothed.fits' # fmt for subtract fcomb
fmt_fsub: 'fsub_{base_fmt:s}.fits'
sel_fsubed: 'fsub_*.fits'
sel_phot: 'phot[0-9]*_*_*.fits'
fmt_photcat: '{{ppflag}}{{imflag}}_{base_fmt:s}.cat'
fmt_photcat_cleaned: '{{ppflag}}{{imflag}}_{base_fmt:s}.cln.cat'
fmt_photcat_matched: '{{ppflag}}{{imflag}}_{base_fmt:s}.zp.cat'
fmt_phot: '{{ppflag}}{{instru}}_{{band}}.cat'
reg_phot: '{reg_phot:s}' # regex to parse grouped phot master
phot_hdr_suffix: 'hdr_phot'
phot_hdr_glob: 'phot[0-9]*_*{{obsid}}*{{instru}}_{{band}}.hdr_phot'
sel_mosaic: 'mosaic[0-9]*_*_*.fits'
fmt_mosaic_orig: 'swarp{{grpid}}_{{imflag}}_{{instru}}_{{band}}.fits'
fmt_mosaic_hdr: 'coadd{{grpid}}.mosaic'
fmt_mosaic: 'coadd{{grpid}}_{{imflag}}_{{instru}}_{{band}}.fits'
fmt_mosaic_wht: 'coadd{{grpid}}_{{imflag}}_{{instru}}_{{band}}.wht.fits'
reg_mosaic: {reg_mosaic:s}
reg_mosaic_fits: {reg_mosaic_fits:s}
fmt_msccat_matched: 'coadd{{grpid}}{{imflag}}_{{instru}}_{{band}}.zp.cat'
reg_grp: '{reg_grp:s}' # regex to parse grouped images
fmt_grp: '{{ppflag}}{{grpid}}_{{imflag}}_{base_fmt:s}.fits'
# environ
workdir: {workdir}
tmpdir: {tmpdir}
logdir: {logdir}
astromatic_prefix: {astromatic_prefix}
stilts_cmd: {stilts_cmd}
funpack_cmd: {funpack_cmd}
""".format(
app_name=APP_NAME,
time=datetime.now().strftime(time_fmt),
version="0.0",
reg_arch=r'(?P<imflag>[^_/]+_)?'
r'(?P<obsid>[^_/]*20\d{6}[Tt]\d{6}(?:\.\d)?)'
r'_(?P<object>.+?)'
r'_(?P<instru>odi|decam)_(?P<band>[ugrizY])(?:_.+)?\.'
r'(?P<ext>fits|fits\.fz)$',
reg_orig=r'(?P<ppflag>[^_/]+_)?(?P<imflag>[^_/]+)_'
r'(?P<obsid>[^_/]*20\d{6}[Tt]\d{6}(?:\.\d)?)'
r'_(?P<object>.+?)'
r'_(?P<instru>odi|decam)_(?P<band>[ugrizY])'
r'\.(?P<ext>fits|fits\.fz)$',
reg_inputs=r'(?P<ppflag>[^_/]+_)?(?P<imflag>[^_/]+)_'
r'(?P<obsid>[^_/]*20\d{6}[Tt]\d{6}(?:\.\d)?)'
r'_(?P<object>.+?)'
r'_(?P<instru>odi|decam)_(?P<band>[ugrizY])'
r'\.(?P<ext>[^/]+)$',
reg_fcomb=r'(?P<ppflag>[^_/]+_)(?P<imflag>[^/]+)\.fits',
reg_phot=r'(?P<ppflag>[^_/]+)'
r'_(?P<instru>odi|decam)_(?P<band>[ugrizY])'
r'\.(?P<ext>[^/]+)',
reg_mosaic=r'(?P<ppflag>[a-z]+)(?P<grpid>\d+)_(?P<imflag>[^_/]+)'
r'_(?P<instru>odi|decam)_(?P<band>[ugrizY])'
r'\.(?P<ext>[^/]+)$',
reg_mosaic_fits=r'(?P<ppflag>[a-z]+)(?P<grpid>\d+)'
r'_(?P<imflag>[^_/]+)'
r'_(?P<instru>odi|decam)_(?P<band>[ugrizY])'
r'\.fits$',
reg_grp=r'(?P<ppflag>[a-z]+)(?P<grpid>\d+)_(?P<imflag>[^_/]+)_'
r'(?P<obsid>[^_/]*20\d{6}[Tt]\d{6}(?:\.\d)?)'
r'_(?P<object>.+?)'
r'_(?P<instru>odi|decam)_(?P<band>[ugrizY])'
r'\.(?P<ext>[^/]+)$',
# reg_inputs=r'(?P<ppflag>[^_/]+_)?(?P<imflag>[^_/]+)_'
# r'(?P<obsid>20\d{6}T\d{6}\.\d)_(?P<object>.+?)'
# r'_odi_(?P<band>[ugriz])'
# r'_(?P<featgrp>\d+)_(?P<photgrp>\d+)_(?P<mscgrp>\d+)'
# r'\.(?P<ext>fits|fits\.fz)$',
**locals())
configfile = os.path.join(workdir, configfile)
if os.path.exists(configfile):
if backup_config:
timestamp = datetime.fromtimestamp(
os.path.getmtime(configfile)).strftime(time_fmt)
bakfile = os.path.join(
workdir, "{1}_{0}{2}".format(
timestamp,
*os.path.splitext(os.path.basename(configfile))))
logger.warning("backup existing config file to {}".format(bakfile))
os.rename(configfile, bakfile)
else:
logger.warning(
"overwrite existing config file {}".format(configfile))
with open(configfile, 'w') as fo:
fo.write(config)
def is_available(self):
return bool(which(self.pdflatex))
def is_installed(self):
return not shutilwhich.which('subl') == None
def main(args):
print("\nScaffolding...")
# Variables #
appname = args.appname
fullpath = os.path.join(cwd, appname)
skeleton_dir = args.skeleton
# Tasks #
# Copy files and folders
print("Copying files and folders...")
shutil.copytree(os.path.join(script_dir, skeleton_dir), fullpath)
# Create config.py
print("Creating the config...")
secret_key = codecs.encode(os.urandom(32), 'hex').decode('utf-8')
template = template_env.get_template('config.jinja2')
template_var = {
'secret_key': secret_key,
}
with open(os.path.join(fullpath, 'project', 'config.py'), 'w') as fd:
fd.write(template.render(template_var))
# Add bower dependencies
if args.bower:
print("Adding bower dependencies...")
bower = args.bower.split(',')
bower_exe = which('bower')
if bower_exe:
os.chdir(os.path.join(fullpath, 'project', 'client', 'static'))
for dependency in bower:
output, error = subprocess.Popen(
[bower_exe, 'install', dependency],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate()
if error:
print("An error occurred with Bower")
print(error)
else:
print("Could not find bower. Ignoring.")
# Add a virtualenv
virtualenv = args.virtualenv
if virtualenv:
print("Adding a virtualenv...")
virtualenv_exe = which('pyvenv')
if virtualenv_exe:
output, error = subprocess.Popen(
[virtualenv_exe, os.path.join(fullpath, 'env')],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate()
if error:
with open('virtualenv_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
print("An error occurred with virtualenv")
sys.exit(2)
venv_bin = os.path.join(fullpath, 'env/bin')
output, error = subprocess.Popen(
[
os.path.join(venv_bin, 'pip'),
'install',
'-r',
os.path.join(fullpath, 'requirements.txt')
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate()
if error:
with open('pip_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
sys.exit(2)
else:
print("Could not find virtualenv executable. Ignoring")
# Git init
if args.git:
print("Initializing Git...")
output, error = subprocess.Popen(
['git', 'init', fullpath],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate()
if error:
with open('git_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
print("Error with git init")
sys.exit(2)
shutil.copyfile(
os.path.join(script_dir, 'templates', '.gitignore'),
os.path.join(fullpath, '.gitignore')
)
def main(argv):
# Arguments #
parser = argparse.ArgumentParser(description='Scaffold a Flask Skeleton.')
parser.add_argument('appname', help='The application name')
parser.add_argument('-s', '--skeleton', help='The skeleton folder to use.')
parser.add_argument('-b', '--bower', help='Install dependencies via bower')
parser.add_argument('-v', '--virtualenv', action='store_true')
parser.add_argument('-g', '--git', action='store_true')
args = parser.parse_args()
# Variables #
appname = args.appname
fullpath = os.path.join(cwd, appname)
skeleton_dir = args.skeleton
# Tasks #
# Copy files and folders
shutil.copytree(os.path.join(script_dir, skeleton_dir), fullpath)
# Create config.py
template = template_env.get_template('config.jinja2')
template_var = {
'secret_key': secret_key,
}
with open(os.path.join(fullpath, 'project', 'config.py'), 'w') as fd:
fd.write(template.render(template_var))
# Add bower dependencies
if args.bower:
bower = args.bower.split(',')
bower_exe = shutil.which('bower')
if bower_exe:
os.chdir(os.path.join(fullpath, 'project', 'static'))
for dependency in bower:
output, error = subprocess.Popen(
[bower_exe, 'install', dependency],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate()
# print(output)
if error:
print("An error occurred with Bower")
print(error)
else:
print("Could not find bower. Ignoring.")
# Add a virtualenv
virtualenv = args.virtualenv
if virtualenv:
virtualenv_exe = which('virtualenv')
if virtualenv_exe:
output, error = subprocess.Popen(
[virtualenv_exe, os.path.join(fullpath, 'env')],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate()
if error:
with open('virtualenv_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
print("An error occurred with virtualenv")
sys.exit(2)
venv_bin = os.path.join(fullpath, 'env/bin')
output, error = subprocess.Popen(
[
os.path.join(venv_bin, 'pip'),
'install',
'-r',
os.path.join(fullpath, 'requirements.txt')
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate()
if error:
with open('pip_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
sys.exit(2)
else:
print("Could not find virtualenv executable. Ignoring")
# Git init
if args.git:
output, error = subprocess.Popen(
['git', 'init', fullpath],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
).communicate()
if error:
with open('git_error.log', 'w') as fd:
fd.write(error.decode('utf-8'))
print("Error with git init")
sys.exit(2)
shutil.copyfile(
os.path.join(script_dir, 'templates', '.gitignore'),
os.path.join(fullpath, '.gitignore')
)
def is_installed(self): return not shutilwhich.which(self.app_name) == None
def is_installed(b):
''' Returns true if an executable named b exists in the current path.
b may also be a list of binaries.
'''
blist = b if isinstance(b, list) else [b,]
return all([which(b) for b in blist])
