def test_cron():
fileobj = open('/home/robouser/cron_test.dat','w')
t = datetime.utcnow()
fileobj.write( t.strftime("%Y-%m-%dT%H:%M:%S") + '\n' )
fileobj.write('Completed imports\n')
fileobj.flush()
config = { 'k2_footprint_data': \
'/home/robouser/Software/robonet_site/data/k2-footprint.json',
'xsuperstamp_target_data': \
'/home/robouser/Software/robonet_site/data/xsuperstamp_targets.json',
'ddt_target_data': \
'/home/robouser/Software/robonet_site/data/c9-ddt-targets-preliminary.csv',
'tmp_location': \
'/science/robonet/rob/Operations/ExoFOP', \
'k2_campaign': 9, \
'k2_year': 2016, \
'log_directory': '/science/robonet/rob/Operations/ExoFOP', \
'log_root_name': 'test_cron'
}
fileobj.write('Set up config\n')
log = log_utilities.start_day_log( config, __name__, console=False )
fileobj.write('Started logging\n')
log.info('Started logging')
environ['DISPLAY'] = ':99'
environ['PWD'] = '/science/robonet/rob/Operations/ExoFOP/'
chdir(environ['PWD'])
log.info(str(environ.items()))
fileobj.write( str(environ.items())+ '\n')
#pkg_path = '/opt/anaconda.2.5.0/bin/K2onSilicon'
#chdir('/science/robonet/rob/Operations/ExoFOP')
#target_file = 'target.csv'
#output_file = '/science/robonet/rob/Operations/ExoFOP/targets_siliconFlag.cav'
comm = '/opt/anaconda.2.5.0/bin/K2onSilicon /science/robonet/rob/Operations/ExoFOP/target.csv 9'
#( iexec, coutput ) = getstatusoutput( pkg_path + ' ' + target_file + \
# ' ' + str(config['k2_campaign']) )
( iexec, coutput ) = getstatusoutput( comm )
log.info(coutput + '\n')
log.info('Loaded K2 data')
fileobj.write(coutput + '\n')
fileobj.write('Loaded K2 Campaign data\n')
fileobj.flush()
fileobj.close()
log_utilities.end_day_log( log )
def setbg(imghash):
img = join(IMG_PATH, imghash+'.'+CONFIG['IMG_EXT'])
env = dict(environ.items() + {'DISPLAY': CONFIG['DISPLAY']}.items())
if isfile(img):
call(['feh', '--bg-max', img], env=env)
else:
logging.warn('Cannot find image: %s' % img)
def feed_init(self, *args, **kwargs):
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Fetch Script Specific Arguments
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
feedid = kwargs.get('feedid')
filename = kwargs.get('filename')
# Fetch/Load Feed Script Configuration
script_config = dict([(FEED_OPTS_RE.match(k).group(1), v.strip()) \
for (k, v) in environ.items() if FEED_OPTS_RE.match(k)])
if self.vvdebug:
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Print Global Script Varables to help debugging process
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
for k, v in script_config.items():
self.logger.vvdebug('%s%s=%s' % (FEED_ENVIRO_ID, k, v))
# Merge Script Configuration With System Config
self.system = dict(script_config.items() + self.system.items())
# self.feedid
# This is the Feed Identifier passed in from NZBGet
if feedid is None:
self.feedid = environ.get(
'%sFEEDID' % FEED_ENVIRO_ID,
)
else:
self.feedid = feedid
# self.filename
# This is the Feed Filename passed in from NZBGet
if filename is None:
self.filename = environ.get(
'%sFILENAME' % FEED_ENVIRO_ID,
)
else:
self.filename = filename
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Error Handling
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
try:
self.feedid = int(self.feedid)
self.logger.info('Feed ID assigned: %d' % self.feedid)
except (ValueError, TypeError):
# Default is 0
self.feedid = 0
self.logger.warning('No Feed ID was assigned')
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Enforce system/global variables for script processing
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
self.system['FEEDID'] = self.feedid
if isinstance(self.feedid, int) and self.feedid > 0:
environ['%sFEEDID' % FEED_ENVIRO_ID] = str(self.feedid)
self.system['FILENAME'] = self.filename
if self.filename:
environ['%sFILENAME' % FEED_ENVIRO_ID] = self.filename
def load_config(self):
"""
For now just load all uppercase options from the
environment.
"""
for option, value in environ.items():
if option.isupper():
setattr(self, option, value)
def get_env(self):
include_dirs = [
"-I{}/{}".format(
self.ctx.include_dir,
d.format(arch=self))
for d in self.ctx.include_dirs]
env = {}
ccache = sh.which('ccache')
cc = sh.xcrun("-find", "-sdk", self.sdk, "clang").strip()
if ccache:
ccache = ccache.strip()
use_ccache = environ.get("USE_CCACHE", "1")
if use_ccache != '1':
env["CC"] = cc
else:
if not self._ccsh:
self._ccsh = ccsh = sh.mktemp().strip()
with open(ccsh, 'w') as f:
f.write('#!/bin/sh\n')
f.write(ccache + ' ' + cc + ' "$@"\n')
sh.chmod('+x', ccsh)
else:
ccsh = self._ccsh
env["USE_CCACHE"] = '1'
env["CCACHE"] = ccache
env["CC"] = ccsh
env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')})
env.setdefault('CCACHE_MAXSIZE', '10G')
env.setdefault('CCACHE_HARDLINK', 'true')
env.setdefault('CCACHE_SLOPPINESS', ('file_macro,time_macros,'
'include_file_mtime,include_file_ctime,file_stat_matches'))
else:
env["CC"] = cc
env["AR"] = sh.xcrun("-find", "-sdk", self.sdk, "ar").strip()
env["LD"] = sh.xcrun("-find", "-sdk", self.sdk, "ld").strip()
env["OTHER_CFLAGS"] = " ".join(include_dirs)
env["OTHER_LDFLAGS"] = " ".join([
"-L{}/{}".format(self.ctx.dist_dir, "lib"),
])
env["CFLAGS"] = " ".join([
"-arch", self.arch,
"-pipe", "-no-cpp-precomp",
"--sysroot", self.sysroot,
#"-I{}/common".format(self.ctx.include_dir),
#"-I{}/{}".format(self.ctx.include_dir, self.arch),
"-O3",
self.version_min
] + include_dirs)
env["LDFLAGS"] = " ".join([
"-arch", self.arch,
"--sysroot", self.sysroot,
"-L{}/{}".format(self.ctx.dist_dir, "lib"),
"-lsqlite3",
self.version_min
])
return env
def checkOSEnviron(self,handler):
empty = {}; setup_testing_defaults(empty)
env = handler.environ
from os import environ
for k,v in environ.items():
if not empty.has_key(k):
self.assertEqual(env[k],v)
for k,v in empty.items():
self.failUnless(env.has_key(k))
def checkOSEnviron(self,handler):
empty = {}; setup_testing_defaults(empty)
env = handler.environ
from os import environ
for k,v in environ.items():
if k not in empty:
self.assertEqual(env[k],v)
for k,v in empty.items():
self.assertIn(k, env)
def get_db_url():
"""
Attempt to find any possible database configuration URL
Datica: DATABASE_1_URL
Heroku: DATABASE_URL
"""
candidate_db_envvars = (
value for name, value in environ.items()
if 'DATABASE' in name and value
)
# Return None if no candidates found
return next(candidate_db_envvars, None)
def format_message(type, value, traceback):
message = StringIO()
out = lambda m: message.write(u'{}\n'.format(m))
out(ctime())
out('== Traceback ==')
out(''.join(format_exception(type, value, traceback)))
out('\n== Command line ==')
out(' '.join(sys.argv))
out('\n== Environment ==')
for key, value in environ.items():
out('{} = {}'.format(key, value))
return message.getvalue()
def __exit(exit_code):
from os import environ
__env__new__items = environ.items()
global __env__old__items
k_v = [(k, v)
for (k, v) in __env__new__items if (k, v) not in __env__old__items]
__env__file = open(environ[ENV_FILE_NAME], 'w')
for k, v in k_v:
__env__file.write('%s=%s\n' % (k, json.dumps(v)))
__env__file.close()
del __env__old__items
del __env__new__items
exit(exit_code)
def submit_job():
opts = [
["--{}".format(key[4:].replace("_", "-")), value]
for key, value in environ.items()
if key.startswith("TBV_") and key != "TBV_CLASS"
]
command = [
"spark-submit",
"--master", "yarn",
"--deploy-mode", "client",
"--class", environ["TBV_CLASS"],
artifact_file,
] + [v for opt in opts for v in opt if v]
call_exit_errors(command)
def index():
response.content_type = 'text/plain; charset=utf-8'
ret = 'Hello world, I\'m %s!\n\n' % os.getpid()
ret += 'Request vars:\n'
for k, v in request.environ.items():
if 'bottle.' in k:
continue
ret += '%s=%s\n' % (k, v)
ret += '\n'
ret += 'Environment vars:\n'
for k, v in env.items():
if 'bottle.' in k:
continue
ret += '%s=%s\n' % (k, v)
return ret
def get_command_output(*command_line):
"""Execute a command line, and return its output.
Raises an exception if return value is nonzero.
:param *command_line: Words for the command line. No shell expansions
are performed.
:type *command_line: Sequence of basestring.
:return: Output from the command.
:rtype: List of basestring, one per line.
"""
env = {
variable: value
for variable, value in environ.items()
if not variable.startswith('LC_')}
env.update({
'LC_ALL': 'C',
'LANG': 'en_US.UTF-8',
})
return check_output(command_line, env=env).splitlines()
def dump_environment(to_var=False):
"""
Dump the shell's env vars to stdout as JSON document.
Args:
to_var (boolean): also dump the env vars to /var?
"""
print(
json.dumps({k: os.environ[k] for k in os.environ.keys()},
indent=4,
sort_keys=True))
if to_var:
try:
with open('env.sh', 'w') as env_log:
env_log.write("# {}\n".format(time.strftime("%c")))
for key, value in environ.items():
env_log.write('export {}="{}"\n'.format(key, value))
except IOError as e:
log_and_stdout(str(e))
log_and_stdout("Failed to create env.sh in cookbook dir...")
def create_app(config):
from os import environ
from flask import Flask, render_template
app = Flask(__name__)
app.config.from_pyfile(config)
app.config.update({
key: val
for key, val in environ.items()
if key.startswith("STAFF") and key.isupper()
})
db.init_app(app)
from .ext import admin, cache, security
# init admin
admin.init_app(app)
# init cache
cache.init_app(app)
# init security
from .account.models import User, Role
from flask.ext.security import SQLAlchemyUserDatastore, login_required
datastore = SQLAlchemyUserDatastore(db, User, Role)
security.init_app(app, datastore)
from . import account, wechat, gitlab
account.init_app(app)
wechat.init_app(app)
gitlab.init_app(app)
account.init_admin()
wechat.init_admin()
gitlab.init_admin()
@app.route("/")
def index():
return render_template("index.html")
return app
def get_options(self):
"""Returns dictionary with application variables from system environment.
Application variables start with {app_} prefix,
but in returned dictionary is set without this prefix.
#!ini
poor_LogLevel = warn # Poor WSGI variable
app_db_server = localhost # application variable db_server
app_templates = app/templ # application variable templates
This method works like Request.get_options, but work with
os.environ, so it works only with wsgi servers, which set not only
request environ, but os.environ too. Apaches mod_wsgi don't do that,
uWsgi and PoorHTTP do that.
"""
options = {}
for key, val in environ.items():
key = key.strip()
if key[:4].lower() == 'app_':
options[key[4:].lower()] = val.strip()
return options
def get_feed(self, feedid=None):
"""Returns a dictionary of feed details identified by the id
specified. If no id is specified, then the current feed is
detected and returned.
"""
if feedid is None:
# assume default
feedid = self.feedid
if not isinstance(feedid, int):
try:
feedid = int(feedid)
except (ValueError, TypeError):
# can't be typecasted to an integer
return {}
if feedid <= 0:
# No feed id defined
return {}
# Precompile Regulare Expression for Speed
feed_re = re.compile('^%s%s%d_([A-Z0-9_]+)$' % (
FEED_ENVIRO_ID,
FEEDID_ENVIRO_ID,
feedid,
))
self.logger.debug('Looking for %s%s%d_([A-Z0-9_]+)$' % (
FEED_ENVIRO_ID,
FEEDID_ENVIRO_ID,
feedid,
))
# Fetch Feed related content
return dict([(feed_re.match(k).group(1), v.strip()) \
for (k, v) in environ.items() if feed_re.match(k)])
def cgi_app_runner(app):
""" Runs "app" in a CGI environment.
CGI example script
==================
>>> import cgitb; cgitb.enable()
>>> import sys
>>> from enkel.wansgli.cgigateway import cgi_app_runner
>>> from enkel.wansgli.demo_apps import simple_app
>>> cgi_app_runner(simple_app)
Put the above in a executable python script, and run it using
a cgi server.
@note: This code is almost a copy of the CGI gateway example in
PEP 333. It will be replaced by a more powerfull class
that supports logging ++ in the future.
@param app: A WSGI app.
"""
env = dict(environ.items())
env['wsgi.input'] = sys.stdin
env['wsgi.errors'] = sys.stderr
env['wsgi.version'] = (1,0)
env['wsgi.multithread'] = False
env['wsgi.multiprocess'] = True
env['wsgi.run_once'] = True
if env.get('HTTPS','off') in ('on','1'):
env['wsgi.url_scheme'] = 'https'
else:
env['wsgi.url_scheme'] = 'http'
req = Response(sys.stdout, env)
run_app(app, req)
def parse_from_env(self) -> Dict[int, str]:
self.tokens = dict((c + 1, t) for c, (_, t) in enumerate(
filter(lambda n: n[0].startswith("MULTI_TOKEN"),
sorted(environ.items()))))
return self.tokens
def get_env(self, with_flags_in_cc=True, clang=False):
env = {}
cflags = [
'-DANDROID',
'-fomit-frame-pointer',
'-D__ANDROID_API__={}'.format(self.ctx.ndk_api)]
if not clang:
cflags.append('-mandroid')
else:
cflags.append('-target ' + self.target)
toolchain = '{android_host}-{toolchain_version}'.format(
android_host=self.ctx.toolchain_prefix,
toolchain_version=self.ctx.toolchain_version)
toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain,
'prebuilt', build_platform)
cflags.append('-gcc-toolchain {}'.format(toolchain))
env['CFLAGS'] = ' '.join(cflags)
# Link the extra global link paths first before anything else
# (such that overriding system libraries with them is possible)
env['LDFLAGS'] = ' ' + " ".join([
"-L'" + l.replace("'", "'\"'\"'") + "'" # no shlex.quote in py2
for l in self.extra_global_link_paths
]) + ' '
sysroot = join(self.ctx._ndk_dir, 'sysroot')
if exists(sysroot):
# post-15 NDK per
# https://android.googlesource.com/platform/ndk/+/ndk-r15-release/docs/UnifiedHeaders.md
env['CFLAGS'] += ' -isystem {}/sysroot/usr/include/{}'.format(
self.ctx.ndk_dir, self.ctx.toolchain_prefix)
env['CFLAGS'] += ' -I{}/sysroot/usr/include/{}'.format(
self.ctx.ndk_dir, self.command_prefix)
else:
sysroot = self.ctx.ndk_platform
env['CFLAGS'] += ' -I{}'.format(self.ctx.ndk_platform)
env['CFLAGS'] += ' -isysroot {} '.format(sysroot)
env['CFLAGS'] += '-I' + join(self.ctx.get_python_install_dir(),
'include/python{}'.format(
self.ctx.python_recipe.version[0:3])
)
env['LDFLAGS'] += '--sysroot={} '.format(self.ctx.ndk_platform)
env["CXXFLAGS"] = env["CFLAGS"]
env["LDFLAGS"] += " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)])
if self.ctx.ndk == 'crystax':
env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch)
toolchain_prefix = self.ctx.toolchain_prefix
toolchain_version = self.ctx.toolchain_version
command_prefix = self.command_prefix
env['TOOLCHAIN_PREFIX'] = toolchain_prefix
env['TOOLCHAIN_VERSION'] = toolchain_version
ccache = ''
if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))):
# print('ccache found, will optimize builds')
ccache = self.ctx.ccache + ' '
env['USE_CCACHE'] = '1'
env['NDK_CCACHE'] = self.ctx.ccache
env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')})
if clang:
llvm_dirname = split(
glob(join(self.ctx.ndk_dir, 'toolchains', 'llvm*'))[-1])[-1]
clang_path = join(self.ctx.ndk_dir, 'toolchains', llvm_dirname,
'prebuilt', build_platform, 'bin')
environ['PATH'] = '{clang_path}:{path}'.format(
clang_path=clang_path, path=environ['PATH'])
exe = join(clang_path, 'clang')
execxx = join(clang_path, 'clang++')
else:
exe = '{command_prefix}-gcc'.format(command_prefix=command_prefix)
execxx = '{command_prefix}-g++'.format(command_prefix=command_prefix)
cc = find_executable(exe, path=environ['PATH'])
if cc is None:
print('Searching path are: {!r}'.format(environ['PATH']))
raise BuildInterruptingException(
'Couldn\'t find executable for CC. This indicates a '
'problem locating the {} executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.'.format(exe))
if with_flags_in_cc:
env['CC'] = '{ccache}{exe} {cflags}'.format(
exe=exe,
ccache=ccache,
cflags=env['CFLAGS'])
env['CXX'] = '{ccache}{execxx} {cxxflags}'.format(
execxx=execxx,
ccache=ccache,
cxxflags=env['CXXFLAGS'])
else:
env['CC'] = '{ccache}{exe}'.format(
exe=exe,
ccache=ccache)
env['CXX'] = '{ccache}{execxx}'.format(
execxx=execxx,
ccache=ccache)
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
env['LDSHARED'] = env["CC"] + " -pthread -shared " +\
"-Wl,-O1 -Wl,-Bsymbolic-functions "
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
# For crystax python, we can't use the host python headers:
env["CFLAGS"] += ' -I{}/sources/python/{}/include/python/'.\
format(self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3])
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j5'
env['READELF'] = '{}-readelf'.format(command_prefix)
env['NM'] = '{}-nm'.format(command_prefix)
hostpython_recipe = Recipe.get_recipe(
'host' + self.ctx.python_recipe.name, self.ctx)
env['BUILDLIB_PATH'] = join(
hostpython_recipe.get_build_dir(self.arch),
'build', 'lib.{}-{}'.format(
build_platform, self.ctx.python_recipe.major_minor_version_string)
)
env['PATH'] = environ['PATH']
env['ARCH'] = self.arch
env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api))
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version
return env
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Used by L{twisted.test.test_process}.
"""
from sys import stdout
from os import environ
items = environ.items()
stdout.write(chr(0).join([k + chr(0) + v for k, v in items]))
stdout.flush()
#!/usr/bin/python
#coding=utf-8
import logging,os
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
import traceback
from os import environ
import ks_http
ks_http.head()
for key in environ.keys():
print key
print "<br>"
print "-----------<br><br>"
print " sys.stdout.encoding =%s"%(sys.stdout.encoding)
for name ,value in environ.items():
print "%s:%s"%(name,value)
print "<br>"
print "-----------<br><br>"
for param in os.environ.keys():
print "<b>%20s</b>: %s<br>" % (param, os.environ[param])
print sys.getdefaultencoding()
def queue_init(self, *args, **kwargs):
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Fetch Script Specific Arguments
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
directory = kwargs.get('directory')
nzbname = kwargs.get('nzbname')
filename = kwargs.get('filename')
category = kwargs.get('category')
priority = kwargs.get('priority')
top = kwargs.get('top')
paused = kwargs.get('paused')
parse_nzbfile = kwargs.get('parse_nzbfile')
use_database = kwargs.get('use_database')
event = kwargs.get('event')
# Fetch/Load Queue Script Configuration
script_config = dict([(QUEUE_OPTS_RE.match(k).group(1), v.strip()) \
for (k, v) in environ.items() if QUEUE_OPTS_RE.match(k)])
if self.vvdebug:
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Print Global Script Varables to help debugging process
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
for k, v in script_config.items():
self.logger.vvdebug('%s%s=%s' % (QUEUE_ENVIRO_ID, k, v))
# Merge Script Configuration With System Config
self.system = dict(script_config.items() + self.system.items())
# self.directory
# This is the path to the destination directory for downloaded files.
if directory is None:
self.directory = environ.get('%sDIRECTORY' % QUEUE_ENVIRO_ID, )
else:
self.directory = directory
# self.nzbname
# User-friendly name of processed nzb-file as it is displayed by the
# program. The file path and extension are removed. If download was
# renamed, this parameter reflects the new name.
if nzbname is None:
self.nzbname = environ.get('%sNZBNAME' % QUEUE_ENVIRO_ID, )
else:
self.nzbname = nzbname
# self.filename
# Name of file to be processed
if filename is None:
self.filename = environ.get('%sFILENAME' % QUEUE_ENVIRO_ID, )
else:
self.filename = filename
# self.category
# Category assigned to nzb-file (can be empty string).
if category is None:
self.category = environ.get('%sCATEGORY' % QUEUE_ENVIRO_ID, )
else:
self.category = category
# self.priority
# The priority of the nzb file being scanned
if priority is None:
self.priority = environ.get('%sPRIORITY' % QUEUE_ENVIRO_ID, )
else:
self.priority = priority
# self.top
# Flag indicating that the file will be added to the top of queue
if top is None:
self.top = environ.get('%sTOP' % QUEUE_ENVIRO_ID, )
else:
self.top = top
# self.paused
# Flag indicating that the file will be added as paused
if paused is None:
self.paused = environ.get('%sPAUSED' % QUEUE_ENVIRO_ID, )
else:
self.paused = paused
# self.event
# Type of Queue Event
if event is None:
self.event = environ.get('%sEVENT' % QUEUE_ENVIRO_ID, )
else:
self.event = event
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Error Handling
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
if self.filename:
# absolute path names
self.filename = abspath(self.filename)
if parse_nzbfile:
# Initialize information fetched from NZB-File
# We intentionally allow existing nzbheaders to over-ride
# any found in the nzbfile
self.nzbheaders = dict(
self.filename(
self.nzbfilename, check_queued=True)\
.items() + self.pull_dnzb().items(),
)
if self.directory:
# absolute path names
self.directory = abspath(self.directory)
if not (self.directory and isdir(self.directory)):
self.logger.warning('Process directory is missing: %s' % \
self.directory)
else:
try:
chdir(self.directory)
except OSError:
self.logger.warning('Directory is not accessible: %s' % \
self.directory)
# Priority
if not isinstance(self.priority, int):
try:
self.priority = int(self.priority)
except:
self.priority = PRIORITY.NORMAL
if self.priority not in PRIORITIES:
self.priority = PRIORITY.NORMAL
# Top
try:
self.top = bool(int(self.top))
except:
self.top = False
# Paused
try:
self.paused = bool(int(self.paused))
except:
self.paused = False
# Event
if self.event not in QUEUE_EVENTS:
self.event = QueueEvent.UNKNOWN
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Enforce system/global variables for script processing
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
self.system['DIRECTORY'] = self.directory
if self.directory is not None:
environ['%sDIRECTORY' % QUEUE_ENVIRO_ID] = self.directory
self.system['NZBNAME'] = self.nzbname
if self.nzbname is not None:
environ['%sNZBNAME' % QUEUE_ENVIRO_ID] = self.nzbname
self.system['FILENAME'] = self.filename
if self.filename is not None:
environ['%sFILENAME' % QUEUE_ENVIRO_ID] = self.filename
self.system['CATEGORY'] = self.category
if self.category is not None:
environ['%sCATEGORY' % QUEUE_ENVIRO_ID] = self.category
self.system['PRIORITY'] = self.priority
if self.priority is not None:
environ['%sPRIORITY' % QUEUE_ENVIRO_ID] = str(self.priority)
self.system['TOP'] = self.top
if self.top is not None:
environ['%sTOP' % QUEUE_ENVIRO_ID] = str(int(self.top))
self.system['EVENT'] = self.event
if self.event is not None:
environ['%sEVENT' % QUEUE_ENVIRO_ID] = str(self.event).upper()
self.system['PAUSED'] = self.paused
if self.paused is not None:
environ['%sPAUSED' % QUEUE_ENVIRO_ID] = str(int(self.paused))
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Create Database for set() and get() operations
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
if use_database:
# database_key is inherited in the parent class
# future calls of set() and get() will allow access
# to the database now
try:
self.database_key = basename(self.filename)
self.logger.info('Connected to SQLite Database')
except AttributeError:
pass
def get_blockdevice_config():
"""
Get configuration dictionary suitable for use in the instantiation
of an ``IBlockDeviceAPI`` implementation.
:raises: ``InvalidConfig`` if a
``FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_FILE`` was not set and the
default config file could not be read.
:return: A ``dict`` of backend configuration parameters.
"""
flocker_functional_test = environ.get('FLOCKER_FUNCTIONAL_TEST')
if flocker_functional_test is None:
raise SkipTest(
'Please set FLOCKER_FUNCTIONAL_TEST environment variable to '
'run storage backend functional tests.'
)
config_file_path = environ.get('FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_FILE')
if config_file_path is None:
raise SkipTest(
'Supply the path to a backend configuration file '
'using the FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_FILE environment '
'variable.'
)
# ie storage-drivers.rackspace
config_section = environ.get(
'FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_SECTION',
)
if config_section is None:
raise SkipTest(
'Supply the section of the config file '
'containing the configuration for the driver under test '
'with the FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_SECTION '
'environment variable.'
)
with open(config_file_path) as config_file:
config = yaml.safe_load(config_file.read())
section = None
for section in config_section.split('.'):
config = config.get(section)
if config is None:
raise InvalidConfig(
"The requested section "
"was not found in the configuration file. "
"Missing subsection: %s, "
"Requested sections: %s, "
"Configuration file: %s" % (
section, config_section, config_file_path
)
)
# XXX A hack to work around the fact that the sub-sections of
# storage-drivers in acceptance.yml do not all have a ``backend`` key.
if "backend" not in config:
config["backend"] = section
# XXX Allow certain configuration to be overridden from environment
# variables.
# This allows us to dynamically use the zone where the functional test
# instance has been launched.
for key, value in environ.items():
if key.startswith(CONFIG_OVERRIDE_PREFIX):
key = key[len(CONFIG_OVERRIDE_PREFIX):]
if key:
config[key] = value.decode('ascii')
return config
def get_env(self, with_flags_in_cc=True, clang=False):
env = {}
cflags = [
'-DANDROID', '-fomit-frame-pointer',
'-D__ANDROID_API__={}'.format(self.ctx.ndk_api)
]
if not clang:
cflags.append('-mandroid')
else:
cflags.append('-target ' + self.target)
toolchain = '{android_host}-{toolchain_version}'.format(
android_host=self.ctx.toolchain_prefix,
toolchain_version=self.ctx.toolchain_version)
toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain,
'prebuilt', build_platform)
cflags.append('-gcc-toolchain {}'.format(toolchain))
env['CFLAGS'] = ' '.join(cflags)
# Link the extra global link paths first before anything else
# (such that overriding system libraries with them is possible)
env['LDFLAGS'] = ' ' + " ".join([
"-L'" + l.replace("'", "'\"'\"'") + "'" # no shlex.quote in py2
for l in self.extra_global_link_paths
]) + ' '
sysroot = join(self.ctx._ndk_dir, 'sysroot')
if exists(sysroot):
# post-15 NDK per
# https://android.googlesource.com/platform/ndk/+/ndk-r15-release/docs/UnifiedHeaders.md
env['CFLAGS'] += ' -isystem {}/sysroot/usr/include/{}'.format(
self.ctx.ndk_dir, self.ctx.toolchain_prefix)
env['CFLAGS'] += ' -I{}/sysroot/usr/include/{}'.format(
self.ctx.ndk_dir, self.command_prefix)
else:
sysroot = self.ctx.ndk_platform
env['CFLAGS'] += ' -I{}'.format(self.ctx.ndk_platform)
env['CFLAGS'] += ' -isysroot {} '.format(sysroot)
env['CFLAGS'] += '-I' + join(
self.ctx.get_python_install_dir(), 'include/python{}'.format(
self.ctx.python_recipe.version[0:3]))
env['LDFLAGS'] += '--sysroot={} '.format(self.ctx.ndk_platform)
env["CXXFLAGS"] = env["CFLAGS"]
env["LDFLAGS"] += " ".join(
['-lm', '-L' + self.ctx.get_libs_dir(self.arch)])
if self.ctx.ndk == 'crystax':
env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(
self.ctx.ndk_dir, self.arch)
toolchain_prefix = self.ctx.toolchain_prefix
toolchain_version = self.ctx.toolchain_version
command_prefix = self.command_prefix
env['TOOLCHAIN_PREFIX'] = toolchain_prefix
env['TOOLCHAIN_VERSION'] = toolchain_version
ccache = ''
if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))):
# print('ccache found, will optimize builds')
ccache = self.ctx.ccache + ' '
env['USE_CCACHE'] = '1'
env['NDK_CCACHE'] = self.ctx.ccache
env.update(
{k: v
for k, v in environ.items() if k.startswith('CCACHE_')})
if clang:
llvm_dirname = split(
glob(join(self.ctx.ndk_dir, 'toolchains', 'llvm*'))[-1])[-1]
clang_path = join(self.ctx.ndk_dir, 'toolchains', llvm_dirname,
'prebuilt', build_platform, 'bin')
environ['PATH'] = '{clang_path}:{path}'.format(
clang_path=clang_path, path=environ['PATH'])
exe = join(clang_path, 'clang')
execxx = join(clang_path, 'clang++')
else:
exe = '{command_prefix}-gcc'.format(command_prefix=command_prefix)
execxx = '{command_prefix}-g++'.format(
command_prefix=command_prefix)
cc = find_executable(exe, path=environ['PATH'])
if cc is None:
print('Searching path are: {!r}'.format(environ['PATH']))
raise BuildInterruptingException(
'Couldn\'t find executable for CC. This indicates a '
'problem locating the {} executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.'.format(exe))
if with_flags_in_cc:
env['CC'] = '{ccache}{exe} {cflags}'.format(exe=exe,
ccache=ccache,
cflags=env['CFLAGS'])
env['CXX'] = '{ccache}{execxx} {cxxflags}'.format(
execxx=execxx, ccache=ccache, cxxflags=env['CXXFLAGS'])
else:
env['CC'] = '{ccache}{exe}'.format(exe=exe, ccache=ccache)
env['CXX'] = '{ccache}{execxx}'.format(execxx=execxx,
ccache=ccache)
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
env['LDSHARED'] = env["CC"] + " -pthread -shared " +\
"-Wl,-O1 -Wl,-Bsymbolic-functions "
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
# For crystax python, we can't use the host python headers:
env["CFLAGS"] += ' -I{}/sources/python/{}/include/python/'.\
format(self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3])
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j5'
env['READELF'] = '{}-readelf'.format(command_prefix)
env['NM'] = '{}-nm'.format(command_prefix)
hostpython_recipe = Recipe.get_recipe(
'host' + self.ctx.python_recipe.name, self.ctx)
env['BUILDLIB_PATH'] = join(
hostpython_recipe.get_build_dir(self.arch),
'native-build',
'build',
'lib.{}-{}'.format(
build_platform,
self.ctx.python_recipe.major_minor_version_string,
),
)
env['PATH'] = environ['PATH']
env['ARCH'] = self.arch
env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api))
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version
return env
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Used by L{twisted.test.test_process}.
"""
from sys import stdout
from os import environ
items = environ.items()
stdout.write(chr(0).join([k + chr(0) + v for k, v in items]))
stdout.flush()
def main(unused_argv):
"""Main function."""
del unused_argv
logging.set_verbosity(logging.INFO)
# Prepare
proc_in_run = {} # hash: str -> (id_gpu: int, popen: Popen}
proc_to_run = {} # hash: str -> cmd: str
free_gpus_set = set(range(FLAGS.nb_gpu))
last_state_changed = True
# Main loop
while True:
# logging.info('Loop tick')
# logging.info('proc_in_run %s', proc_in_run)
# logging.info('proc_to_run %s', proc_to_run)
# logging.info('free_gpus_set %s', free_gpus_set)
state_changed = False
# print and eject finished popen object
hash_keys = list(proc_in_run.keys())
for hash_ in hash_keys:
id_gpu, popen = proc_in_run[hash_]
poll_result = popen.poll()
if poll_result is not None:
logging.info(
'command done (return code = %d): "%s"',
popen.returncode,
popen.args,
)
free_gpus_set.add(id_gpu)
add_to_finished_hash(hash_)
del proc_in_run[hash_]
state_changed = True
# Add job to queue if is not seen before
finished_hashs_set = get_finished_hashs_set()
cmds = load_cmds()
for cmd in cmds:
hash_ = hash_cmd(cmd)
if (hash_ not in finished_hashs_set) and (hash_ not in proc_in_run) and (
hash_ not in proc_to_run):
proc_to_run[hash_] = cmd
logging.info('Add to queue: "%s"', shlex.split(cmd))
state_changed = True
# Launch if there is slot
hash_keys = list(proc_to_run.keys())
for hash_ in hash_keys:
cmd = proc_to_run[hash_]
if free_gpus_set:
logging.info('Launch: "%s"', shlex.split(cmd))
id_gpu = free_gpus_set.pop()
env = {k: v for k, v in environ.items()}
env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # match the order from nvidia-smi
env['CUDA_VISIBLE_DEVICES'] = '%d' % id_gpu
popen = subprocess.Popen(
args=shlex.split(cmd),
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
env=env,
)
proc_in_run[hash_] = (id_gpu, popen)
del proc_to_run[hash_]
state_changed = True
if not state_changed:
if last_state_changed: # only print once when entering the state-not-changed
logging.info('State not changed.')
last_state_changed = state_changed
# Sleep
time.sleep(FLAGS.loop_interval)
def setenv(self, shell, where=stdout):
if self.recursive:
return
ref = where
shell_data = self.shell[shell]
sep = shell_data['SEP']
udata = {}
data = []
index = 0
if not self._unsetenv:
env_prefix = self.env_backup_prefix
env = self._runtime()
for d in env['variables']:
for var, val in data.items():
udata[var] = 1
data.append({var: val[0]})
for var in env['path']:
if '_SRTOPT_' in var:
continue
udata[var] = 1
benv = '%s%s%s' % (env_prefix, var, self.backup_type[var])
val = self._fixpathvar(var, sep)
if benv in environ:
val = environ[benv] + val
data.append({var: val})
if shell == 'RTBOURNE':
data.append({'SCRAM_RTBOURNE_SET': environ['SCRAMRT_SET']})
for var, val in env['xenv'].items():
udata[var] = 1
data.append({var: val})
for var, val in environ.items():
if var not in udata:
data.insert(0, {var: val})
udata[var] = 1
oenv = self.OENV
unset = ""
unset_vars = ""
for v in oenv:
if v in udata:
continue
if v in environ:
del environ[v]
if shell == 'RTBOURNE':
continue
unset += " %s" % v
if not v.startswith('SCRAMRT_') and \
not v.endswith('_SCRAMRT') and \
not v.endswith('_SCRAMRTDEL'):
unset_vars += " %s\n" % v
if unset:
if unset_vars and not self._unsetenv:
SCRAM.printerror(
"**** Following environment variables are going to be unset.\n%s"
% unset_vars)
print("%s %s;" % (shell_data['UNEXPORT'], unset), file=ref)
for d in data:
for var, val in d.items():
if shell == 'RTBOURNE':
if var.startswith('SCRAMRT_') or \
var.endswith('_SCRAMRT') or \
var.endswith('_SCRAMRTDEL'):
if var != 'SCRAMRT_SET':
del environ[var]
continue
environ[var] = val
continue
if var != 'PATH' and var in oenv:
if val == oenv[var]:
continue
environ[var] = val
print('%s %s%s\"%s\";' %
(shell_data['EXPORT'], var, shell_data['EQUALS'], val),
file=stdout)
return True
XX = """
def _patch_env(*filtered_keys, **kw):
env = {k: v for k, v in environ.items() if k not in filtered_keys}
env.update(kw)
return env
def get_env(self, with_flags_in_cc=True):
env = {}
env["CFLAGS"] = " ".join([
"-DANDROID", "-mandroid", "-fomit-frame-pointer",
"--sysroot", self.ctx.ndk_platform])
env["CXXFLAGS"] = env["CFLAGS"]
env["LDFLAGS"] = " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)])
if self.ctx.ndk == 'crystax':
env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch)
py_platform = sys.platform
if py_platform in ['linux2', 'linux3']:
py_platform = 'linux'
toolchain_prefix = self.ctx.toolchain_prefix
toolchain_version = self.ctx.toolchain_version
command_prefix = self.command_prefix
env['TOOLCHAIN_PREFIX'] = toolchain_prefix
env['TOOLCHAIN_VERSION'] = toolchain_version
ccache = ''
if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))):
# print('ccache found, will optimize builds')
ccache = self.ctx.ccache + ' '
env['USE_CCACHE'] = '1'
env['NDK_CCACHE'] = self.ctx.ccache
env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')})
cc = find_executable('{command_prefix}-gcc'.format(
command_prefix=command_prefix), path=environ['PATH'])
if cc is None:
print('Searching path are: {!r}'.format(environ['PATH']))
warning('Couldn\'t find executable for CC. This indicates a '
'problem locating the {} executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.')
exit(1)
if with_flags_in_cc:
env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cflags=env['CFLAGS'])
env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cxxflags=env['CXXFLAGS'])
else:
env['CC'] = '{ccache}{command_prefix}-gcc'.format(
command_prefix=command_prefix,
ccache=ccache)
env['CXX'] = '{ccache}{command_prefix}-g++'.format(
command_prefix=command_prefix,
ccache=ccache)
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
# env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink')
# env['LDSHARED'] = env['LD']
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j5'
env['READELF'] = '{}-readelf'.format(command_prefix)
env['NM'] = '{}-nm'.format(command_prefix)
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
# AND: This hardcodes python version 2.7, needs fixing
env['BUILDLIB_PATH'] = join(
hostpython_recipe.get_build_dir(self.arch),
'build', 'lib.linux-{}-2.7'.format(uname()[-1]))
env['PATH'] = environ['PATH']
env['ARCH'] = self.arch
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version
return env
def sanic_config_manager(app: Sanic, prefix: str = "SANIC_"):
for variable, value in environ.items():
if variable.startswith(prefix):
_, key = variable.split(prefix, 1)
app.config[key] = value
# -*- coding: utf-8 -*-
import argparse
import subprocess
from os import environ
from shlex import split as shlex_split
from textwrap import dedent
from unittest import mock
from functools import partial
import pytest
from bumpversion import main, DESCRIPTION, bumpversion
from bumpversion.bumpversion import split_args_in_optional_and_positional, RawConfigParser
from bumpversion.exceptions import WorkingDirectoryIsDirtyException
SUBPROCESS_ENV = dict(list(environ.items()) + [(b'HGENCODING', b'utf-8')])
call = partial(subprocess.call, env=SUBPROCESS_ENV)
check_call = partial(subprocess.check_call, env=SUBPROCESS_ENV)
check_output = partial(subprocess.check_output, env=SUBPROCESS_ENV)
xfail_if_no_git = pytest.mark.xfail(call(["git", "help"]) != 0,
reason="git is not installed")
@pytest.fixture(params=['.bumpv.cfg', 'setup.cfg'])
def configfile(request):
return request.param
try:
import sys
import logging
import mock
import argparse
import subprocess
from os import curdir, makedirs, chdir, environ
from os.path import join, curdir, dirname
from shlex import split as shlex_split
from textwrap import dedent
from functools import partial
from bumpversion import main, DESCRIPTION
SUBPROCESS_ENV = dict(
list(environ.items()) + [('HGENCODING', 'utf-8')]
)
call = partial(subprocess.call, env=SUBPROCESS_ENV)
check_call = partial(subprocess.check_call, env=SUBPROCESS_ENV)
check_output = partial(subprocess.check_output, env=SUBPROCESS_ENV)
xfail_if_no_git = pytest.mark.xfail(
call(["git", "--help"], shell=True) != 1,
reason="git is not installed"
)
xfail_if_no_hg = pytest.mark.xfail(
call(["hg", "--help"], shell=True) != 0,
reason="hg is not installed"
)
import sys
import logging
import mock
import argparse
import subprocess
from os import curdir, makedirs, chdir, environ
from os.path import join, curdir, dirname
from shlex import split as shlex_split
from textwrap import dedent
from functools import partial
from bumpversion import main, DESCRIPTION
SUBPROCESS_ENV = dict(
list(environ.items()) + [(str('HGENCODING'), str('utf-8'))]
)
call = partial(subprocess.call, env=SUBPROCESS_ENV)
check_call = partial(subprocess.check_call, env=SUBPROCESS_ENV)
check_output = partial(subprocess.check_output, env=SUBPROCESS_ENV)
xfail_if_no_git = pytest.mark.xfail(
call(["git", "--help"], shell=True) != 1,
reason="git is not installed"
)
xfail_if_no_hg = pytest.mark.xfail(
call(["hg", "--help"], shell=True) != 0,
reason="hg is not installed"
)
from requests import post
from requests_oauthlib import OAuth2Session
from git import (
Getter, is_authenticated, repo_exists, split_branch_path, get_circle_artifacts,
select_path, _LONGTIME, get_branch_info, ERR_TESTS_PENDING, ERR_TESTS_FAILED,
skip_webhook_payload, get_webhook_commit_info, post_github_status
)
from href import needs_redirect, get_redirect, absolute_url
from util import errors_logged, nice_relative_time, parse_webhook_config, err_codes
from git import github_client_id, github_client_secret, FAKE_TOKEN, GithubDisallowed
from git import ERR_NO_REPOSITORY, ERR_TESTS_PENDING, ERR_TESTS_FAILED, ERR_NO_REF_STATUS
flask_secret_key = environ.get('FLASK_SECRET') or 'poop'
webhook_config = parse_webhook_config(*[val for (key, val) in environ.items()
if key.startswith('WEBHOOK_CONFIG_')
or key == 'WEBHOOK_CONFIG'])
app = Flask(__name__)
if sys.argv[0] != 'test.py':
getLogger('precog').info('SSLifying')
SSLify(app)
app.secret_key = flask_secret_key
app.jinja_env.filters['nice_relative_time'] = nice_relative_time
app.config['HOOK_SECRETS_TOKENS'] = webhook_config
@app.before_first_request
def adjust_log_level():
getLogger('precog').setLevel(DEBUG if app.debug else INFO)
A copy of this file may be found in `telemetry-airflow/bin`
This file is used as an argument in the SparkPythonTask in the Databricks
api.[0] The library is assumed to be installed on all of the machines in the
cluster. Arguments are passed to the script through `MOZETL_`-prefixed
environment variables.
This script is deployed to `s3://telemetry-airflow/steps/mozetl_runner.py`.[1]
[0]: https://docs.databricks.com/api/latest/jobs.html#sparkpythontask
[1]: https://bugzilla.mozilla.org/show_bug.cgi?id=1484331
"""
from os import environ
from pprint import pformat
from mozetl import cli
print(
pformat({
k: v for k, v in environ.items()
if k.startswith("MOZETL")
})
)
try:
cli.entry_point(auto_envvar_prefix="MOZETL")
except SystemExit:
# avoid calling sys.exit() in databricks
# http://click.palletsprojects.com/en/7.x/api/?highlight=auto_envvar_prefix#click.BaseCommand.main
pass
from flask import Flask
from models import initialize_database
from os import environ
from resources import initialize_resources
from schema.schema_file import Schema
from sqlalchemy import create_engine
# Starting Flask application
application = Flask(__name__)
# Loading environment variables from .env file only in development
if environ.get('EBS_ENVIRONMENT') in ['local', None]:
load_dotenv('./environments/local.env')
# Loading environment variables into Flask application
for item in environ.items():
application.config[item[0]] = item[1]
# Starting database configuration
initialize_database(application)
# Starting RESTful endpoints
initialize_resources(application)
@application.before_first_request
def startup():
print("Initializing migration DB")
Schema.migration()
Schema.prepare_db()
# Now, activate log file
Logger.logfile_activated = bool(Config.getint('kivy', 'log_enable'))
# If no configuration exist, write the default one.
if ((not exists(kivy_config_fn) or need_save)
and 'KIVY_NO_CONFIG' not in environ):
try:
Config.filename = kivy_config_fn
Config.write()
except Exception as e:
Logger.exception('Core: Error while saving default config file')
# Load configuration from env
if environ.get('KIVY_NO_ENV_CONFIG', '0') != '1':
for key, value in environ.items():
if not key.startswith("KCFG_"):
continue
try:
_, section, name = key.split("_", 2)
except ValueError:
Logger.warning(("Config: Environ `{}` invalid format, "
"must be KCFG_section_name").format(key))
continue
# extract and check section
section = section.lower()
if not Config.has_section(section):
Logger.warning(
"Config: Environ `{}`: unknown section `{}`".format(
key, section))
def get_env(self, with_flags_in_cc=True):
env = {}
env["CFLAGS"] = " ".join([
"-DANDROID", "-mandroid", "-fomit-frame-pointer",
"--sysroot", self.ctx.ndk_platform])
env["CXXFLAGS"] = env["CFLAGS"]
env["LDFLAGS"] = " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)])
if self.ctx.ndk == 'crystax':
env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch)
py_platform = sys.platform
if py_platform in ['linux2', 'linux3']:
py_platform = 'linux'
toolchain_prefix = self.ctx.toolchain_prefix
toolchain_version = self.ctx.toolchain_version
command_prefix = self.command_prefix
env['TOOLCHAIN_PREFIX'] = toolchain_prefix
env['TOOLCHAIN_VERSION'] = toolchain_version
ccache = ''
if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))):
print('ccache found, will optimize builds')
ccache = self.ctx.ccache + ' '
env['USE_CCACHE'] = '1'
env['NDK_CCACHE'] = self.ctx.ccache
env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')})
print('path is', environ['PATH'])
cc = find_executable('{command_prefix}-gcc'.format(
command_prefix=command_prefix), path=environ['PATH'])
if cc is None:
warning('Couldn\'t find executable for CC. This indicates a '
'problem locating the {} executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.')
exit(1)
if with_flags_in_cc:
env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cflags=env['CFLAGS'])
env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cxxflags=env['CXXFLAGS'])
else:
env['CC'] = '{ccache}{command_prefix}-gcc'.format(
command_prefix=command_prefix,
ccache=ccache)
env['CXX'] = '{ccache}{command_prefix}-g++'.format(
command_prefix=command_prefix,
ccache=ccache)
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
# env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink')
# env['LDSHARED'] = env['LD']
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j5'
env['READELF'] = '{}-readelf'.format(command_prefix)
env['NM'] = '{}-nm'.format(command_prefix)
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
# AND: This hardcodes python version 2.7, needs fixing
env['BUILDLIB_PATH'] = join(
hostpython_recipe.get_build_dir(self.arch),
'build', 'lib.linux-{}-2.7'.format(uname()[-1]))
env['PATH'] = environ['PATH']
env['ARCH'] = self.arch
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version
return env
def get_env(self):
include_dirs = [
"-I{}/{}".format(self.ctx.include_dir, d.format(arch=self))
for d in self.ctx.include_dirs
]
env = {}
ccache = sh.which('ccache')
cc = sh.xcrun("-find", "-sdk", self.sdk, "clang").strip()
if ccache:
ccache = ccache.strip()
use_ccache = environ.get("USE_CCACHE", "1")
if use_ccache != '1':
env["CC"] = cc
else:
if not self._ccsh:
self._ccsh = ccsh = sh.mktemp().strip()
with open(ccsh, 'w') as f:
f.write('#!/bin/sh\n')
f.write(ccache + ' ' + cc + ' "$@"\n')
sh.chmod('+x', ccsh)
else:
ccsh = self._ccsh
env["USE_CCACHE"] = '1'
env["CCACHE"] = ccache
env["CC"] = ccsh
env.update({
k: v
for k, v in environ.items() if k.startswith('CCACHE_')
})
env.setdefault('CCACHE_MAXSIZE', '10G')
env.setdefault('CCACHE_HARDLINK', 'true')
env.setdefault('CCACHE_SLOPPINESS', (
'file_macro,time_macros,'
'include_file_mtime,include_file_ctime,file_stat_matches'))
else:
env["CC"] = cc
env["AR"] = sh.xcrun("-find", "-sdk", self.sdk, "ar").strip()
env["LD"] = sh.xcrun("-find", "-sdk", self.sdk, "ld").strip()
env["OTHER_CFLAGS"] = " ".join(include_dirs)
env["OTHER_LDFLAGS"] = " ".join([
"-L{}/{}".format(self.ctx.dist_dir, "lib"),
])
env["CFLAGS"] = " ".join([
"-arch",
self.arch,
"-pipe",
"-no-cpp-precomp",
"--sysroot",
self.sysroot,
#"-I{}/common".format(self.ctx.include_dir),
#"-I{}/{}".format(self.ctx.include_dir, self.arch),
"-O3",
self.version_min
] + include_dirs)
env["LDFLAGS"] = " ".join([
"-arch", self.arch, "--sysroot", self.sysroot,
"-L{}/{}".format(self.ctx.dist_dir,
"lib"), "-lsqlite3", self.version_min
])
return env
def get_env(self, with_flags_in_cc=True):
env = {}
env['CFLAGS'] = ' '.join([
'-DANDROID', '-mandroid', '-fomit-frame-pointer'
' -D__ANDROID_API__={}'.format(self.ctx.ndk_api),
])
env['LDFLAGS'] = ' '
sysroot = join(self.ctx._ndk_dir, 'sysroot')
if exists(sysroot):
# post-15 NDK per
# https://android.googlesource.com/platform/ndk/+/ndk-r15-release/docs/UnifiedHeaders.md
env['CFLAGS'] += ' -isystem {}/sysroot/usr/include/{}'.format(
self.ctx.ndk_dir, self.ctx.toolchain_prefix)
else:
sysroot = self.ctx.ndk_platform
env['CFLAGS'] += ' -I{}'.format(self.ctx.ndk_platform)
env['CFLAGS'] += ' -isysroot {} '.format(sysroot)
env['CFLAGS'] += '-I' + join(self.ctx.get_python_install_dir(),
'include/python{}'.format(
self.ctx.python_recipe.version[0:3])
)
env['LDFLAGS'] += '--sysroot {} '.format(self.ctx.ndk_platform)
env["CXXFLAGS"] = env["CFLAGS"]
env["LDFLAGS"] += " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)])
if self.ctx.ndk == 'crystax':
env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch)
py_platform = sys.platform
if py_platform in ['linux2', 'linux3']:
py_platform = 'linux'
toolchain_prefix = self.ctx.toolchain_prefix
toolchain_version = self.ctx.toolchain_version
command_prefix = self.command_prefix
env['TOOLCHAIN_PREFIX'] = toolchain_prefix
env['TOOLCHAIN_VERSION'] = toolchain_version
ccache = ''
if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))):
# print('ccache found, will optimize builds')
ccache = self.ctx.ccache + ' '
env['USE_CCACHE'] = '1'
env['NDK_CCACHE'] = self.ctx.ccache
env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')})
cc = find_executable('{command_prefix}-gcc'.format(
command_prefix=command_prefix), path=environ['PATH'])
if cc is None:
print('Searching path are: {!r}'.format(environ['PATH']))
warning('Couldn\'t find executable for CC. This indicates a '
'problem locating the {} executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.')
exit(1)
if with_flags_in_cc:
env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cflags=env['CFLAGS'])
env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cxxflags=env['CXXFLAGS'])
else:
env['CC'] = '{ccache}{command_prefix}-gcc'.format(
command_prefix=command_prefix,
ccache=ccache)
env['CXX'] = '{ccache}{command_prefix}-g++'.format(
command_prefix=command_prefix,
ccache=ccache)
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
env['LDSHARED'] = env["CC"] + " -pthread -shared " +\
"-Wl,-O1 -Wl,-Bsymbolic-functions "
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
# For crystax python, we can't use the host python headers:
env["CFLAGS"] += ' -I{}/sources/python/{}/include/python/'.\
format(self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3])
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j5'
env['READELF'] = '{}-readelf'.format(command_prefix)
env['NM'] = '{}-nm'.format(command_prefix)
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
# This hardcodes python version 2.7, needs fixing
env['BUILDLIB_PATH'] = join(
hostpython_recipe.get_build_dir(self.arch),
'build', 'lib.linux-{}-2.7'.format(uname()[-1]))
env['PATH'] = environ['PATH']
env['ARCH'] = self.arch
env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api))
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version
return env
def scan_init(self, *args, **kwargs):
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Fetch Script Specific Arguments
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
directory = kwargs.get('directory')
nzbname = kwargs.get('nzbname')
filename = kwargs.get('filename')
category = kwargs.get('category')
priority = kwargs.get('priority')
top = kwargs.get('top')
paused = kwargs.get('paused')
parse_nzbfile = kwargs.get('parse_nzbfile')
use_database = kwargs.get('use_database')
# Fetch/Load Scan Script Configuration
script_config = dict([(SCAN_OPTS_RE.match(k).group(1), v.strip()) \
for (k, v) in environ.items() if SCAN_OPTS_RE.match(k)])
if self.vvdebug:
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Print Global Script Varables to help debugging process
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
for k, v in script_config.items():
self.logger.vvdebug('%s%s=%s' % (SCAN_ENVIRO_ID, k, v))
# Merge Script Configuration With System Config
self.system = dict(script_config.items() + self.system.items())
# self.directory
# This is the path to the destination directory for downloaded files.
if directory is None:
self.directory = environ.get(
'%sDIRECTORY' % SCAN_ENVIRO_ID,
)
else:
self.directory = directory
# self.nzbname
# User-friendly name of processed nzb-file as it is displayed by the
# program. The file path and extension are removed. If download was
# renamed, this parameter reflects the new name.
if nzbname is None:
self.nzbname = environ.get(
'%sNZBNAME' % SCAN_ENVIRO_ID,
)
else:
self.nzbname = nzbname
# self.filename
# Name of file to be processed
if filename is None:
self.filename = environ.get(
'%sFILENAME' % SCAN_ENVIRO_ID,
)
else:
self.filename = filename
# self.category
# Category assigned to nzb-file (can be empty string).
if category is None:
self.category = environ.get(
'%sCATEGORY' % SCAN_ENVIRO_ID,
)
else:
self.category = category
# self.priority
# The priority of the nzb file being scanned
if priority is None:
self.priority = environ.get(
'%sPRIORITY' % SCAN_ENVIRO_ID,
)
else:
self.priority = priority
# self.top
# Flag indicating that the file will be added to the top of queue
if top is None:
self.top = environ.get(
'%sTOP' % SCAN_ENVIRO_ID,
)
else:
self.top = top
# self.paused
# Flag indicating that the file will be added as paused
if paused is None:
self.paused = environ.get(
'%sPAUSED' % SCAN_ENVIRO_ID,
)
else:
self.paused = paused
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Error Handling
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
if self.filename:
# absolute path names
self.filename = abspath(self.filename)
if parse_nzbfile:
# Initialize information fetched from NZB-File
# We intentionally allow existing nzbheaders to over-ride
# any found in the nzbfile
self.nzbheaders = dict(
self.parse_nzbfile(
self.filename, check_queued=True)\
.items() + self.pull_dnzb().items(),
)
if self.directory:
# absolute path names
self.directory = abspath(self.directory)
if not (self.directory and isdir(self.directory)):
self.logger.debug('Process directory is missing: %s' % \
self.directory)
else:
try:
chdir(self.directory)
except OSError:
self.logger.debug('Process directory is not accessible: %s' % \
self.directory)
# Priority
if not isinstance(self.priority, int):
try:
self.priority = int(self.priority)
except:
self.priority = PRIORITY.NORMAL
if self.priority not in PRIORITIES:
self.priority = PRIORITY.NORMAL
# Top
try:
self.top = bool(int(self.top))
except:
self.top = False
# Paused
try:
self.paused = bool(int(self.paused))
except:
self.paused = False
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Enforce system/global variables for script processing
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
self.system['DIRECTORY'] = self.directory
if self.directory is not None:
environ['%sDIRECTORY' % SCAN_ENVIRO_ID] = self.directory
self.system['NZBNAME'] = self.nzbname
if self.nzbname is not None:
environ['%sNZBNAME' % SCAN_ENVIRO_ID] = self.nzbname
self.system['FILENAME'] = self.filename
if self.filename is not None:
environ['%sFILENAME' % SCAN_ENVIRO_ID] = self.filename
self.system['CATEGORY'] = self.category
if self.category is not None:
environ['%sCATEGORY' % SCAN_ENVIRO_ID] = self.category
self.system['PRIORITY'] = self.priority
if self.priority is not None:
environ['%sPRIORITY' % SCAN_ENVIRO_ID] = str(self.priority)
self.system['TOP'] = self.top
if self.top is not None:
environ['%sTOP' % SCAN_ENVIRO_ID] = str(int(self.top))
self.system['PAUSED'] = self.paused
if self.paused is not None:
environ['%sPAUSED' % SCAN_ENVIRO_ID] = str(int(self.paused))
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# Create Database for set() and get() operations
# =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
if use_database:
# database_key is inherited in the parent class
# future calls of set() and get() will allow access
# to the database now
try:
self.database_key = basename(self.filename)
self.logger.info('Connected to SQLite Database')
except AttributeError:
pass
def _insert_kube_env(self):
return {
key: value
for (key, value) in environ.items() if "KUBERNETES" in key
}
from requests import post
from requests_oauthlib import OAuth2Session
from git import (
Getter, is_authenticated, repo_exists, split_branch_path, get_circle_artifacts,
select_path, _LONGTIME, get_branch_info, ERR_TESTS_PENDING, ERR_TESTS_FAILED,
skip_webhook_payload, get_webhook_commit_info, post_github_status
)
from href import needs_redirect, get_redirect, absolute_url
from util import errors_logged, nice_relative_time, parse_webhook_config, err_codes
from git import github_client_id, github_client_secret, FAKE_TOKEN, GithubDisallowed
from git import ERR_NO_REPOSITORY, ERR_TESTS_PENDING, ERR_TESTS_FAILED, ERR_NO_REF_STATUS
flask_secret_key = environ.get('FLASK_SECRET') or 'poop'
webhook_config = parse_webhook_config(*[val for (key, val) in environ.items()
if key.startswith('WEBHOOK_CONFIG_')
or key == 'WEBHOOK_CONFIG'])
app = Flask(__name__)
# if sys.argv[0] != 'test.py':
# getLogger('precog').info('SSLifying')
# SSLify(app)
app.secret_key = flask_secret_key
app.jinja_env.filters['nice_relative_time'] = nice_relative_time
app.config['HOOK_SECRETS_TOKENS'] = webhook_config
@app.before_first_request
def adjust_log_level():
getLogger('precog').setLevel(DEBUG if app.debug else INFO)
def create_global_comm(nprocs, dir=None):
""" Figures out global communicator through external mpi call. """
from sys import executable
from tempfile import NamedTemporaryFile
from subprocess import PIPE
from os.path import exists
from os import environ
from os import remove, getcwd
from re import finditer
from .. import mpirun_exe, modify_global_comm, do_multiple_mpi_programs, \
figure_out_machines as script, launch_program as launch
from ..misc import local_path
from ..error import ConfigError
import pylada
from ..process import logger
logger.debug('process/mpi: create_global_comm: entry')
logger.debug('process/mpi: create_global_comm: nprocs: %s' % nprocs)
logger.debug('process/mpi: create_global_comm: dir: \"%s\"' % dir)
logger.debug('process/mpi: create_global_comm: script: \"%s\"' % script)
if not do_multiple_mpi_programs:
return
if nprocs <= 0:
raise MPISizeError(nprocs)
if dir is None:
dir = getcwd()
# each proc prints its name to the standard output.
local_path(dir).ensure(dir=True)
filename = None
try:
with NamedTemporaryFile(delete=False, dir=dir) as file:
file.write(str(script).encode('utf-8'))
filename = file.name
formatter = Communicator(n=nprocs).copy()
formatter['program'] = executable + ' ' + filename
logger.debug("process.mpi: create_global_comm: formatter: %s" % formatter)
logger.debug("process.mpi: filename: \"%s\"" % filename)
logger.debug("===content:===\n%s===end===" % open(filename).read())
logger.debug("process.mpi: create_global_comm: mpirun_exe: %s " % mpirun_exe)
logger.debug("process.mpi: *** launch ***")
process = launch(mpirun_exe, stdout=PIPE, formatter=formatter,
stderr=PIPE, env=environ)
logger.debug("process.mpi: create_global_comm: process: %s" % process)
logger.debug("process.mpi: *** start process.communicate ***")
stdout, stderr = process.communicate()
logger.debug("process.mpi: === start stdout ===\n%s\n=== end ===" % stdout)
logger.debug("process.mpi: === start stderr ===\n%s\n=== end ===" % stderr)
logger.debug("process.mpi: *** start process.communicate ***")
finally:
if filename is not None and exists(filename):
try:
remove(filename)
except:
pass
# we use that to deduce the number of machines and processes per machine.
processes = [line.group(1) for line
in finditer('PYLADA MACHINE HOSTNAME:\s*(\S*)', stdout)]
logger.debug(" process.mpi: create_global_comm: processes: %s" % processes)
logger.debug("process.mpi: nprocs: create_global_comm: %s" % nprocs)
# sanity check.
if nprocs != len(processes):
envstring = ''
for key, value in environ.items():
envstring += ' {0} = {1!r}\n'.format(key, value)
raise ConfigError('Pylada could not determine host machines.\n'
'Standard output reads as follows:\n{0}\n'
'Standard error reads as follows:\n{1}\n'
'environment variables were set as follows:\n{2}\n'
'following script was run:\n{3}\n'
.format(stdout, stderr, envstring, script))
# now set up the default communicator.
machines = set(processes)
pylada.default_comm = Communicator(pylada.default_comm)
pylada.default_comm['n'] = nprocs
pylada.default_comm.machines = {}
for machine in machines:
pylada.default_comm.machines[machine] = processes.count(machine)
# called when we need to change the machine names on some supercomputers.
modify_global_comm(pylada.default_comm)
def get_env(self, with_flags_in_cc=True):
env = {}
env['CFLAGS'] = ' '.join([
'-DANDROID', '-mandroid', '-fomit-frame-pointer'
' -D__ANDROID_API__={}'.format(self.ctx.ndk_api),
])
env['LDFLAGS'] = ' '
sysroot = join(self.ctx._ndk_dir, 'sysroot')
if exists(sysroot):
# post-15 NDK per
# https://android.googlesource.com/platform/ndk/+/ndk-r15-release/docs/UnifiedHeaders.md
env['CFLAGS'] += ' -isystem {}/sysroot/usr/include/{}'.format(
self.ctx.ndk_dir, self.ctx.toolchain_prefix)
else:
sysroot = self.ctx.ndk_platform
env['CFLAGS'] += ' -I{}'.format(self.ctx.ndk_platform)
env['CFLAGS'] += ' -isysroot {} '.format(sysroot)
env['CFLAGS'] += '-I' + join(self.ctx.get_python_install_dir(),
'include/python{}'.format(
self.ctx.python_recipe.version[0:3])
)
env['LDFLAGS'] += '--sysroot {} '.format(self.ctx.ndk_platform)
env["CXXFLAGS"] = env["CFLAGS"]
env["LDFLAGS"] += " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)])
if self.ctx.ndk == 'crystax':
env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch)
py_platform = sys.platform
if py_platform in ['linux2', 'linux3']:
py_platform = 'linux'
toolchain_prefix = self.ctx.toolchain_prefix
toolchain_version = self.ctx.toolchain_version
command_prefix = self.command_prefix
env['TOOLCHAIN_PREFIX'] = toolchain_prefix
env['TOOLCHAIN_VERSION'] = toolchain_version
ccache = ''
if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))):
# print('ccache found, will optimize builds')
ccache = self.ctx.ccache + ' '
env['USE_CCACHE'] = '1'
env['NDK_CCACHE'] = self.ctx.ccache
env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')})
cc = find_executable('{command_prefix}-gcc'.format(
command_prefix=command_prefix), path=environ['PATH'])
if cc is None:
print('Searching path are: {!r}'.format(environ['PATH']))
warning('Couldn\'t find executable for CC. This indicates a '
'problem locating the {} executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.')
exit(1)
if with_flags_in_cc:
env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cflags=env['CFLAGS'])
env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cxxflags=env['CXXFLAGS'])
else:
env['CC'] = '{ccache}{command_prefix}-gcc'.format(
command_prefix=command_prefix,
ccache=ccache)
env['CXX'] = '{ccache}{command_prefix}-g++'.format(
command_prefix=command_prefix,
ccache=ccache)
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
env['LDSHARED'] = env["CC"] + " -pthread -shared " +\
"-Wl,-O1 -Wl,-Bsymbolic-functions "
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
# For crystax python, we can't use the host python headers:
env["CFLAGS"] += ' -I{}/sources/python/{}/include/python/'.\
format(self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3])
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j5'
env['READELF'] = '{}-readelf'.format(command_prefix)
env['NM'] = '{}-nm'.format(command_prefix)
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
# This hardcodes python version 2.7, needs fixing
env['BUILDLIB_PATH'] = join(
hostpython_recipe.get_build_dir(self.arch),
'build', 'lib.linux-{}-2.7'.format(uname()[-1]))
env['PATH'] = environ['PATH']
env['ARCH'] = self.arch
env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api))
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version
return env
'''
Created on Jan 19, 2019
@author: dad
'''
if __name__ == '__main__':
pass
import os
# get environment
from os import environ
env_items = environ.items()
env_values = {(k, v) for (k, v) in environ.items()}
import io
from sys import argv
with open('somefile.txt', 'rb') as inf:
txtdata = inf.read()
if txtdata.startswith(b'\xff\xd8'):
text = u'This is a Text file (%d bytes long)\n'
else:
text = u'This is a random file (%d bytes long)\n'
with io.open('summary.txt', 'w', encoding='utf-8') as outf:
outf.write(text % len(txtdata))
env_filtered = [
env_item for env_item in environ.items()
def test_main(self) -> None:
""" Tests that main will be called """
with self.assertRaises(SystemExit) as e, patch(
"sys.stdout", new_callable=StringIO) as out, patch(
"sys.stderr", new_callable=StringIO) as err, patch(
"sys.argv", [sys.executable]):
self.assertIsNone(main())
help_text, _usage, engine_help_text = err.getvalue().rpartition(
"usage")
engine_help_text = _usage + engine_help_text
engines = help_text[(lambda k: help_text.index(k) + len(k) + 2
)("--engine"):help_text.index("\n") - 2]
self.assertEqual(
engine_help_text,
"usage: python -m ml_params [-h] [--version] [--engine {{{engines}}}]\n"
"python -m ml_params: error: --engine must be provided,"
" and from installed ml-params-* options\n".format(
engines=engines),
)
self.assertEqual(*map(
lambda s: s.replace(" ", "").replace("\n", ""),
((
help_text,
"usage: python -m ml_params [-h] [--version] [--engine {{{engines}}}]\n\n"
"Consistent CLI for every popular ML framework.\n\n"
"{option_help}:\n"
" -h, --help show this help message and exit\n"
" --version show program's version number and exit\n"
" --engine {{{engines}}}"
' ML engine, e.g., "TensorFlow", "JAX", "pytorch"\n'
.format(
option_help="options" if sys.version_info[:2] >
(3, 9) else "optional arguments",
engines=engines,
),
)),
))
self.assertEqual(e.exception.code, SystemExit(2).code)
self.assertEqual(out.getvalue(), "")
# With engine set
mod = "ml-params-tensorflow"
if mod not in sys.modules:
sys.modules[mod] = ml_params # TODO: Some fake test module
if not TF_SUPPORTED:
return
env_backup = dict(environ.items())
environ["ML_PARAMS_ENGINE"] = "tensorflow"
_argv = ["--engine", "tensorflow", "-h"]
with self.assertRaises(SystemExit) as e, patch(
"sys.stdout", new_callable=StringIO) as out, patch(
"sys.stderr", new_callable=StringIO) as err, patch(
"sys.argv", [sys.executable] + _argv), patch(
"os.environ", environ):
self.assertIsNone(main(_argv))
self.assertEqual(e.exception.code, SystemExit(0).code)
self.assertEqual(err.getvalue(), "")
self.assertEqual(
out.getvalue(),
"Adding CLI parser: load_data_parser ;\n"
"Adding CLI parser: load_model_parser ;\n"
"Adding CLI parser: train_parser ;\n"
"usage: python -m ml_params [-h] [--version] [--engine {{{engines}}}]"
"\n{rest}".format(
engines=engines,
rest=
" {load_data,load_model,train} ...\n\n"
"Consistent CLI for every popular ML framework.\n\n"
"positional arguments:\n"
" {load_data,load_model,train}\n"
" subcommand to run. Hacked to be chainable.\n\n"
"optional arguments:\n"
" -h, --help show this help message and exit\n"
" --version show program's version number and exit\n"
" " + "--engine {{{engines}}}\n"
' ML engine, e.g., "TensorFlow", "JAX", "pytorch"\n'
.format(engines=engines),
),
)
del environ["ML_PARAMS_ENGINE"]
environ.update(env_backup)
