def test_cron(): fileobj = open('/home/robouser/cron_test.dat','w') t = datetime.utcnow() fileobj.write( t.strftime("%Y-%m-%dT%H:%M:%S") + '\n' ) fileobj.write('Completed imports\n') fileobj.flush() config = { 'k2_footprint_data': \ '/home/robouser/Software/robonet_site/data/k2-footprint.json', 'xsuperstamp_target_data': \ '/home/robouser/Software/robonet_site/data/xsuperstamp_targets.json', 'ddt_target_data': \ '/home/robouser/Software/robonet_site/data/c9-ddt-targets-preliminary.csv', 'tmp_location': \ '/science/robonet/rob/Operations/ExoFOP', \ 'k2_campaign': 9, \ 'k2_year': 2016, \ 'log_directory': '/science/robonet/rob/Operations/ExoFOP', \ 'log_root_name': 'test_cron' } fileobj.write('Set up config\n') log = log_utilities.start_day_log( config, __name__, console=False ) fileobj.write('Started logging\n') log.info('Started logging') environ['DISPLAY'] = ':99' environ['PWD'] = '/science/robonet/rob/Operations/ExoFOP/' chdir(environ['PWD']) log.info(str(environ.items())) fileobj.write( str(environ.items())+ '\n') #pkg_path = '/opt/anaconda.2.5.0/bin/K2onSilicon' #chdir('/science/robonet/rob/Operations/ExoFOP') #target_file = 'target.csv' #output_file = '/science/robonet/rob/Operations/ExoFOP/targets_siliconFlag.cav' comm = '/opt/anaconda.2.5.0/bin/K2onSilicon /science/robonet/rob/Operations/ExoFOP/target.csv 9' #( iexec, coutput ) = getstatusoutput( pkg_path + ' ' + target_file + \ # ' ' + str(config['k2_campaign']) ) ( iexec, coutput ) = getstatusoutput( comm ) log.info(coutput + '\n') log.info('Loaded K2 data') fileobj.write(coutput + '\n') fileobj.write('Loaded K2 Campaign data\n') fileobj.flush() fileobj.close() log_utilities.end_day_log( log )
def setbg(imghash): img = join(IMG_PATH, imghash+'.'+CONFIG['IMG_EXT']) env = dict(environ.items() + {'DISPLAY': CONFIG['DISPLAY']}.items()) if isfile(img): call(['feh', '--bg-max', img], env=env) else: logging.warn('Cannot find image: %s' % img)
def feed_init(self, *args, **kwargs): # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Fetch Script Specific Arguments # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- feedid = kwargs.get('feedid') filename = kwargs.get('filename') # Fetch/Load Feed Script Configuration script_config = dict([(FEED_OPTS_RE.match(k).group(1), v.strip()) \ for (k, v) in environ.items() if FEED_OPTS_RE.match(k)]) if self.vvdebug: # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Print Global Script Varables to help debugging process # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- for k, v in script_config.items(): self.logger.vvdebug('%s%s=%s' % (FEED_ENVIRO_ID, k, v)) # Merge Script Configuration With System Config self.system = dict(script_config.items() + self.system.items()) # self.feedid # This is the Feed Identifier passed in from NZBGet if feedid is None: self.feedid = environ.get( '%sFEEDID' % FEED_ENVIRO_ID, ) else: self.feedid = feedid # self.filename # This is the Feed Filename passed in from NZBGet if filename is None: self.filename = environ.get( '%sFILENAME' % FEED_ENVIRO_ID, ) else: self.filename = filename # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Error Handling # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- try: self.feedid = int(self.feedid) self.logger.info('Feed ID assigned: %d' % self.feedid) except (ValueError, TypeError): # Default is 0 self.feedid = 0 self.logger.warning('No Feed ID was assigned') # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Enforce system/global variables for script processing # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- self.system['FEEDID'] = self.feedid if isinstance(self.feedid, int) and self.feedid > 0: environ['%sFEEDID' % FEED_ENVIRO_ID] = str(self.feedid) self.system['FILENAME'] = self.filename if self.filename: environ['%sFILENAME' % FEED_ENVIRO_ID] = self.filename
def load_config(self): """ For now just load all uppercase options from the environment. """ for option, value in environ.items(): if option.isupper(): setattr(self, option, value)
def get_env(self): include_dirs = [ "-I{}/{}".format( self.ctx.include_dir, d.format(arch=self)) for d in self.ctx.include_dirs] env = {} ccache = sh.which('ccache') cc = sh.xcrun("-find", "-sdk", self.sdk, "clang").strip() if ccache: ccache = ccache.strip() use_ccache = environ.get("USE_CCACHE", "1") if use_ccache != '1': env["CC"] = cc else: if not self._ccsh: self._ccsh = ccsh = sh.mktemp().strip() with open(ccsh, 'w') as f: f.write('#!/bin/sh\n') f.write(ccache + ' ' + cc + ' "$@"\n') sh.chmod('+x', ccsh) else: ccsh = self._ccsh env["USE_CCACHE"] = '1' env["CCACHE"] = ccache env["CC"] = ccsh env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')}) env.setdefault('CCACHE_MAXSIZE', '10G') env.setdefault('CCACHE_HARDLINK', 'true') env.setdefault('CCACHE_SLOPPINESS', ('file_macro,time_macros,' 'include_file_mtime,include_file_ctime,file_stat_matches')) else: env["CC"] = cc env["AR"] = sh.xcrun("-find", "-sdk", self.sdk, "ar").strip() env["LD"] = sh.xcrun("-find", "-sdk", self.sdk, "ld").strip() env["OTHER_CFLAGS"] = " ".join(include_dirs) env["OTHER_LDFLAGS"] = " ".join([ "-L{}/{}".format(self.ctx.dist_dir, "lib"), ]) env["CFLAGS"] = " ".join([ "-arch", self.arch, "-pipe", "-no-cpp-precomp", "--sysroot", self.sysroot, #"-I{}/common".format(self.ctx.include_dir), #"-I{}/{}".format(self.ctx.include_dir, self.arch), "-O3", self.version_min ] + include_dirs) env["LDFLAGS"] = " ".join([ "-arch", self.arch, "--sysroot", self.sysroot, "-L{}/{}".format(self.ctx.dist_dir, "lib"), "-lsqlite3", self.version_min ]) return env
def checkOSEnviron(self,handler): empty = {}; setup_testing_defaults(empty) env = handler.environ from os import environ for k,v in environ.items(): if not empty.has_key(k): self.assertEqual(env[k],v) for k,v in empty.items(): self.failUnless(env.has_key(k))
def checkOSEnviron(self,handler): empty = {}; setup_testing_defaults(empty) env = handler.environ from os import environ for k,v in environ.items(): if k not in empty: self.assertEqual(env[k],v) for k,v in empty.items(): self.assertIn(k, env)
def get_db_url(): """ Attempt to find any possible database configuration URL Datica: DATABASE_1_URL Heroku: DATABASE_URL """ candidate_db_envvars = ( value for name, value in environ.items() if 'DATABASE' in name and value ) # Return None if no candidates found return next(candidate_db_envvars, None)
def format_message(type, value, traceback): message = StringIO() out = lambda m: message.write(u'{}\n'.format(m)) out(ctime()) out('== Traceback ==') out(''.join(format_exception(type, value, traceback))) out('\n== Command line ==') out(' '.join(sys.argv)) out('\n== Environment ==') for key, value in environ.items(): out('{} = {}'.format(key, value)) return message.getvalue()
def __exit(exit_code): from os import environ __env__new__items = environ.items() global __env__old__items k_v = [(k, v) for (k, v) in __env__new__items if (k, v) not in __env__old__items] __env__file = open(environ[ENV_FILE_NAME], 'w') for k, v in k_v: __env__file.write('%s=%s\n' % (k, json.dumps(v))) __env__file.close() del __env__old__items del __env__new__items exit(exit_code)
def submit_job(): opts = [ ["--{}".format(key[4:].replace("_", "-")), value] for key, value in environ.items() if key.startswith("TBV_") and key != "TBV_CLASS" ] command = [ "spark-submit", "--master", "yarn", "--deploy-mode", "client", "--class", environ["TBV_CLASS"], artifact_file, ] + [v for opt in opts for v in opt if v] call_exit_errors(command)
def index(): response.content_type = 'text/plain; charset=utf-8' ret = 'Hello world, I\'m %s!\n\n' % os.getpid() ret += 'Request vars:\n' for k, v in request.environ.items(): if 'bottle.' in k: continue ret += '%s=%s\n' % (k, v) ret += '\n' ret += 'Environment vars:\n' for k, v in env.items(): if 'bottle.' in k: continue ret += '%s=%s\n' % (k, v) return ret
def get_command_output(*command_line): """Execute a command line, and return its output. Raises an exception if return value is nonzero. :param *command_line: Words for the command line. No shell expansions are performed. :type *command_line: Sequence of basestring. :return: Output from the command. :rtype: List of basestring, one per line. """ env = { variable: value for variable, value in environ.items() if not variable.startswith('LC_')} env.update({ 'LC_ALL': 'C', 'LANG': 'en_US.UTF-8', }) return check_output(command_line, env=env).splitlines()
def dump_environment(to_var=False): """ Dump the shell's env vars to stdout as JSON document. Args: to_var (boolean): also dump the env vars to /var? """ print( json.dumps({k: os.environ[k] for k in os.environ.keys()}, indent=4, sort_keys=True)) if to_var: try: with open('env.sh', 'w') as env_log: env_log.write("# {}\n".format(time.strftime("%c"))) for key, value in environ.items(): env_log.write('export {}="{}"\n'.format(key, value)) except IOError as e: log_and_stdout(str(e)) log_and_stdout("Failed to create env.sh in cookbook dir...")
def create_app(config): from os import environ from flask import Flask, render_template app = Flask(__name__) app.config.from_pyfile(config) app.config.update({ key: val for key, val in environ.items() if key.startswith("STAFF") and key.isupper() }) db.init_app(app) from .ext import admin, cache, security # init admin admin.init_app(app) # init cache cache.init_app(app) # init security from .account.models import User, Role from flask.ext.security import SQLAlchemyUserDatastore, login_required datastore = SQLAlchemyUserDatastore(db, User, Role) security.init_app(app, datastore) from . import account, wechat, gitlab account.init_app(app) wechat.init_app(app) gitlab.init_app(app) account.init_admin() wechat.init_admin() gitlab.init_admin() @app.route("/") def index(): return render_template("index.html") return app
def get_options(self): """Returns dictionary with application variables from system environment. Application variables start with {app_} prefix, but in returned dictionary is set without this prefix. #!ini poor_LogLevel = warn # Poor WSGI variable app_db_server = localhost # application variable db_server app_templates = app/templ # application variable templates This method works like Request.get_options, but work with os.environ, so it works only with wsgi servers, which set not only request environ, but os.environ too. Apaches mod_wsgi don't do that, uWsgi and PoorHTTP do that. """ options = {} for key, val in environ.items(): key = key.strip() if key[:4].lower() == 'app_': options[key[4:].lower()] = val.strip() return options
def get_feed(self, feedid=None): """Returns a dictionary of feed details identified by the id specified. If no id is specified, then the current feed is detected and returned. """ if feedid is None: # assume default feedid = self.feedid if not isinstance(feedid, int): try: feedid = int(feedid) except (ValueError, TypeError): # can't be typecasted to an integer return {} if feedid <= 0: # No feed id defined return {} # Precompile Regulare Expression for Speed feed_re = re.compile('^%s%s%d_([A-Z0-9_]+)$' % ( FEED_ENVIRO_ID, FEEDID_ENVIRO_ID, feedid, )) self.logger.debug('Looking for %s%s%d_([A-Z0-9_]+)$' % ( FEED_ENVIRO_ID, FEEDID_ENVIRO_ID, feedid, )) # Fetch Feed related content return dict([(feed_re.match(k).group(1), v.strip()) \ for (k, v) in environ.items() if feed_re.match(k)])
def cgi_app_runner(app): """ Runs "app" in a CGI environment. CGI example script ================== >>> import cgitb; cgitb.enable() >>> import sys >>> from enkel.wansgli.cgigateway import cgi_app_runner >>> from enkel.wansgli.demo_apps import simple_app >>> cgi_app_runner(simple_app) Put the above in a executable python script, and run it using a cgi server. @note: This code is almost a copy of the CGI gateway example in PEP 333. It will be replaced by a more powerfull class that supports logging ++ in the future. @param app: A WSGI app. """ env = dict(environ.items()) env['wsgi.input'] = sys.stdin env['wsgi.errors'] = sys.stderr env['wsgi.version'] = (1,0) env['wsgi.multithread'] = False env['wsgi.multiprocess'] = True env['wsgi.run_once'] = True if env.get('HTTPS','off') in ('on','1'): env['wsgi.url_scheme'] = 'https' else: env['wsgi.url_scheme'] = 'http' req = Response(sys.stdout, env) run_app(app, req)
def parse_from_env(self) -> Dict[int, str]: self.tokens = dict((c + 1, t) for c, (_, t) in enumerate( filter(lambda n: n[0].startswith("MULTI_TOKEN"), sorted(environ.items())))) return self.tokens
def get_env(self, with_flags_in_cc=True, clang=False): env = {} cflags = [ '-DANDROID', '-fomit-frame-pointer', '-D__ANDROID_API__={}'.format(self.ctx.ndk_api)] if not clang: cflags.append('-mandroid') else: cflags.append('-target ' + self.target) toolchain = '{android_host}-{toolchain_version}'.format( android_host=self.ctx.toolchain_prefix, toolchain_version=self.ctx.toolchain_version) toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain, 'prebuilt', build_platform) cflags.append('-gcc-toolchain {}'.format(toolchain)) env['CFLAGS'] = ' '.join(cflags) # Link the extra global link paths first before anything else # (such that overriding system libraries with them is possible) env['LDFLAGS'] = ' ' + " ".join([ "-L'" + l.replace("'", "'\"'\"'") + "'" # no shlex.quote in py2 for l in self.extra_global_link_paths ]) + ' ' sysroot = join(self.ctx._ndk_dir, 'sysroot') if exists(sysroot): # post-15 NDK per # https://android.googlesource.com/platform/ndk/+/ndk-r15-release/docs/UnifiedHeaders.md env['CFLAGS'] += ' -isystem {}/sysroot/usr/include/{}'.format( self.ctx.ndk_dir, self.ctx.toolchain_prefix) env['CFLAGS'] += ' -I{}/sysroot/usr/include/{}'.format( self.ctx.ndk_dir, self.command_prefix) else: sysroot = self.ctx.ndk_platform env['CFLAGS'] += ' -I{}'.format(self.ctx.ndk_platform) env['CFLAGS'] += ' -isysroot {} '.format(sysroot) env['CFLAGS'] += '-I' + join(self.ctx.get_python_install_dir(), 'include/python{}'.format( self.ctx.python_recipe.version[0:3]) ) env['LDFLAGS'] += '--sysroot={} '.format(self.ctx.ndk_platform) env["CXXFLAGS"] = env["CFLAGS"] env["LDFLAGS"] += " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)]) if self.ctx.ndk == 'crystax': env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch) toolchain_prefix = self.ctx.toolchain_prefix toolchain_version = self.ctx.toolchain_version command_prefix = self.command_prefix env['TOOLCHAIN_PREFIX'] = toolchain_prefix env['TOOLCHAIN_VERSION'] = toolchain_version ccache = '' if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))): # print('ccache found, will optimize builds') ccache = self.ctx.ccache + ' ' env['USE_CCACHE'] = '1' env['NDK_CCACHE'] = self.ctx.ccache env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')}) if clang: llvm_dirname = split( glob(join(self.ctx.ndk_dir, 'toolchains', 'llvm*'))[-1])[-1] clang_path = join(self.ctx.ndk_dir, 'toolchains', llvm_dirname, 'prebuilt', build_platform, 'bin') environ['PATH'] = '{clang_path}:{path}'.format( clang_path=clang_path, path=environ['PATH']) exe = join(clang_path, 'clang') execxx = join(clang_path, 'clang++') else: exe = '{command_prefix}-gcc'.format(command_prefix=command_prefix) execxx = '{command_prefix}-g++'.format(command_prefix=command_prefix) cc = find_executable(exe, path=environ['PATH']) if cc is None: print('Searching path are: {!r}'.format(environ['PATH'])) raise BuildInterruptingException( 'Couldn\'t find executable for CC. This indicates a ' 'problem locating the {} executable in the Android ' 'NDK, not that you don\'t have a normal compiler ' 'installed. Exiting.'.format(exe)) if with_flags_in_cc: env['CC'] = '{ccache}{exe} {cflags}'.format( exe=exe, ccache=ccache, cflags=env['CFLAGS']) env['CXX'] = '{ccache}{execxx} {cxxflags}'.format( execxx=execxx, ccache=ccache, cxxflags=env['CXXFLAGS']) else: env['CC'] = '{ccache}{exe}'.format( exe=exe, ccache=ccache) env['CXX'] = '{ccache}{execxx}'.format( execxx=execxx, ccache=ccache) env['AR'] = '{}-ar'.format(command_prefix) env['RANLIB'] = '{}-ranlib'.format(command_prefix) env['LD'] = '{}-ld'.format(command_prefix) env['LDSHARED'] = env["CC"] + " -pthread -shared " +\ "-Wl,-O1 -Wl,-Bsymbolic-functions " if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: # For crystax python, we can't use the host python headers: env["CFLAGS"] += ' -I{}/sources/python/{}/include/python/'.\ format(self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3]) env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix) env['MAKE'] = 'make -j5' env['READELF'] = '{}-readelf'.format(command_prefix) env['NM'] = '{}-nm'.format(command_prefix) hostpython_recipe = Recipe.get_recipe( 'host' + self.ctx.python_recipe.name, self.ctx) env['BUILDLIB_PATH'] = join( hostpython_recipe.get_build_dir(self.arch), 'build', 'lib.{}-{}'.format( build_platform, self.ctx.python_recipe.major_minor_version_string) ) env['PATH'] = environ['PATH'] env['ARCH'] = self.arch env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api)) if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version return env
# Copyright (c) Twisted Matrix Laboratories. # See LICENSE for details. """ Used by L{twisted.test.test_process}. """ from sys import stdout from os import environ items = environ.items() stdout.write(chr(0).join([k + chr(0) + v for k, v in items])) stdout.flush()
#!/usr/bin/python #coding=utf-8 import logging,os import sys reload(sys) sys.setdefaultencoding('utf-8') import traceback from os import environ import ks_http ks_http.head() for key in environ.keys(): print key print "<br>" print "-----------<br><br>" print " sys.stdout.encoding =%s"%(sys.stdout.encoding) for name ,value in environ.items(): print "%s:%s"%(name,value) print "<br>" print "-----------<br><br>" for param in os.environ.keys(): print "<b>%20s</b>: %s<br>" % (param, os.environ[param]) print sys.getdefaultencoding()
def queue_init(self, *args, **kwargs): # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Fetch Script Specific Arguments # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- directory = kwargs.get('directory') nzbname = kwargs.get('nzbname') filename = kwargs.get('filename') category = kwargs.get('category') priority = kwargs.get('priority') top = kwargs.get('top') paused = kwargs.get('paused') parse_nzbfile = kwargs.get('parse_nzbfile') use_database = kwargs.get('use_database') event = kwargs.get('event') # Fetch/Load Queue Script Configuration script_config = dict([(QUEUE_OPTS_RE.match(k).group(1), v.strip()) \ for (k, v) in environ.items() if QUEUE_OPTS_RE.match(k)]) if self.vvdebug: # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Print Global Script Varables to help debugging process # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- for k, v in script_config.items(): self.logger.vvdebug('%s%s=%s' % (QUEUE_ENVIRO_ID, k, v)) # Merge Script Configuration With System Config self.system = dict(script_config.items() + self.system.items()) # self.directory # This is the path to the destination directory for downloaded files. if directory is None: self.directory = environ.get('%sDIRECTORY' % QUEUE_ENVIRO_ID, ) else: self.directory = directory # self.nzbname # User-friendly name of processed nzb-file as it is displayed by the # program. The file path and extension are removed. If download was # renamed, this parameter reflects the new name. if nzbname is None: self.nzbname = environ.get('%sNZBNAME' % QUEUE_ENVIRO_ID, ) else: self.nzbname = nzbname # self.filename # Name of file to be processed if filename is None: self.filename = environ.get('%sFILENAME' % QUEUE_ENVIRO_ID, ) else: self.filename = filename # self.category # Category assigned to nzb-file (can be empty string). if category is None: self.category = environ.get('%sCATEGORY' % QUEUE_ENVIRO_ID, ) else: self.category = category # self.priority # The priority of the nzb file being scanned if priority is None: self.priority = environ.get('%sPRIORITY' % QUEUE_ENVIRO_ID, ) else: self.priority = priority # self.top # Flag indicating that the file will be added to the top of queue if top is None: self.top = environ.get('%sTOP' % QUEUE_ENVIRO_ID, ) else: self.top = top # self.paused # Flag indicating that the file will be added as paused if paused is None: self.paused = environ.get('%sPAUSED' % QUEUE_ENVIRO_ID, ) else: self.paused = paused # self.event # Type of Queue Event if event is None: self.event = environ.get('%sEVENT' % QUEUE_ENVIRO_ID, ) else: self.event = event # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Error Handling # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- if self.filename: # absolute path names self.filename = abspath(self.filename) if parse_nzbfile: # Initialize information fetched from NZB-File # We intentionally allow existing nzbheaders to over-ride # any found in the nzbfile self.nzbheaders = dict( self.filename( self.nzbfilename, check_queued=True)\ .items() + self.pull_dnzb().items(), ) if self.directory: # absolute path names self.directory = abspath(self.directory) if not (self.directory and isdir(self.directory)): self.logger.warning('Process directory is missing: %s' % \ self.directory) else: try: chdir(self.directory) except OSError: self.logger.warning('Directory is not accessible: %s' % \ self.directory) # Priority if not isinstance(self.priority, int): try: self.priority = int(self.priority) except: self.priority = PRIORITY.NORMAL if self.priority not in PRIORITIES: self.priority = PRIORITY.NORMAL # Top try: self.top = bool(int(self.top)) except: self.top = False # Paused try: self.paused = bool(int(self.paused)) except: self.paused = False # Event if self.event not in QUEUE_EVENTS: self.event = QueueEvent.UNKNOWN # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Enforce system/global variables for script processing # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- self.system['DIRECTORY'] = self.directory if self.directory is not None: environ['%sDIRECTORY' % QUEUE_ENVIRO_ID] = self.directory self.system['NZBNAME'] = self.nzbname if self.nzbname is not None: environ['%sNZBNAME' % QUEUE_ENVIRO_ID] = self.nzbname self.system['FILENAME'] = self.filename if self.filename is not None: environ['%sFILENAME' % QUEUE_ENVIRO_ID] = self.filename self.system['CATEGORY'] = self.category if self.category is not None: environ['%sCATEGORY' % QUEUE_ENVIRO_ID] = self.category self.system['PRIORITY'] = self.priority if self.priority is not None: environ['%sPRIORITY' % QUEUE_ENVIRO_ID] = str(self.priority) self.system['TOP'] = self.top if self.top is not None: environ['%sTOP' % QUEUE_ENVIRO_ID] = str(int(self.top)) self.system['EVENT'] = self.event if self.event is not None: environ['%sEVENT' % QUEUE_ENVIRO_ID] = str(self.event).upper() self.system['PAUSED'] = self.paused if self.paused is not None: environ['%sPAUSED' % QUEUE_ENVIRO_ID] = str(int(self.paused)) # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Create Database for set() and get() operations # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- if use_database: # database_key is inherited in the parent class # future calls of set() and get() will allow access # to the database now try: self.database_key = basename(self.filename) self.logger.info('Connected to SQLite Database') except AttributeError: pass
def get_blockdevice_config(): """ Get configuration dictionary suitable for use in the instantiation of an ``IBlockDeviceAPI`` implementation. :raises: ``InvalidConfig`` if a ``FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_FILE`` was not set and the default config file could not be read. :return: A ``dict`` of backend configuration parameters. """ flocker_functional_test = environ.get('FLOCKER_FUNCTIONAL_TEST') if flocker_functional_test is None: raise SkipTest( 'Please set FLOCKER_FUNCTIONAL_TEST environment variable to ' 'run storage backend functional tests.' ) config_file_path = environ.get('FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_FILE') if config_file_path is None: raise SkipTest( 'Supply the path to a backend configuration file ' 'using the FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_FILE environment ' 'variable.' ) # ie storage-drivers.rackspace config_section = environ.get( 'FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_SECTION', ) if config_section is None: raise SkipTest( 'Supply the section of the config file ' 'containing the configuration for the driver under test ' 'with the FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_SECTION ' 'environment variable.' ) with open(config_file_path) as config_file: config = yaml.safe_load(config_file.read()) section = None for section in config_section.split('.'): config = config.get(section) if config is None: raise InvalidConfig( "The requested section " "was not found in the configuration file. " "Missing subsection: %s, " "Requested sections: %s, " "Configuration file: %s" % ( section, config_section, config_file_path ) ) # XXX A hack to work around the fact that the sub-sections of # storage-drivers in acceptance.yml do not all have a ``backend`` key. if "backend" not in config: config["backend"] = section # XXX Allow certain configuration to be overridden from environment # variables. # This allows us to dynamically use the zone where the functional test # instance has been launched. for key, value in environ.items(): if key.startswith(CONFIG_OVERRIDE_PREFIX): key = key[len(CONFIG_OVERRIDE_PREFIX):] if key: config[key] = value.decode('ascii') return config
def get_env(self, with_flags_in_cc=True, clang=False): env = {} cflags = [ '-DANDROID', '-fomit-frame-pointer', '-D__ANDROID_API__={}'.format(self.ctx.ndk_api) ] if not clang: cflags.append('-mandroid') else: cflags.append('-target ' + self.target) toolchain = '{android_host}-{toolchain_version}'.format( android_host=self.ctx.toolchain_prefix, toolchain_version=self.ctx.toolchain_version) toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain, 'prebuilt', build_platform) cflags.append('-gcc-toolchain {}'.format(toolchain)) env['CFLAGS'] = ' '.join(cflags) # Link the extra global link paths first before anything else # (such that overriding system libraries with them is possible) env['LDFLAGS'] = ' ' + " ".join([ "-L'" + l.replace("'", "'\"'\"'") + "'" # no shlex.quote in py2 for l in self.extra_global_link_paths ]) + ' ' sysroot = join(self.ctx._ndk_dir, 'sysroot') if exists(sysroot): # post-15 NDK per # https://android.googlesource.com/platform/ndk/+/ndk-r15-release/docs/UnifiedHeaders.md env['CFLAGS'] += ' -isystem {}/sysroot/usr/include/{}'.format( self.ctx.ndk_dir, self.ctx.toolchain_prefix) env['CFLAGS'] += ' -I{}/sysroot/usr/include/{}'.format( self.ctx.ndk_dir, self.command_prefix) else: sysroot = self.ctx.ndk_platform env['CFLAGS'] += ' -I{}'.format(self.ctx.ndk_platform) env['CFLAGS'] += ' -isysroot {} '.format(sysroot) env['CFLAGS'] += '-I' + join( self.ctx.get_python_install_dir(), 'include/python{}'.format( self.ctx.python_recipe.version[0:3])) env['LDFLAGS'] += '--sysroot={} '.format(self.ctx.ndk_platform) env["CXXFLAGS"] = env["CFLAGS"] env["LDFLAGS"] += " ".join( ['-lm', '-L' + self.ctx.get_libs_dir(self.arch)]) if self.ctx.ndk == 'crystax': env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format( self.ctx.ndk_dir, self.arch) toolchain_prefix = self.ctx.toolchain_prefix toolchain_version = self.ctx.toolchain_version command_prefix = self.command_prefix env['TOOLCHAIN_PREFIX'] = toolchain_prefix env['TOOLCHAIN_VERSION'] = toolchain_version ccache = '' if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))): # print('ccache found, will optimize builds') ccache = self.ctx.ccache + ' ' env['USE_CCACHE'] = '1' env['NDK_CCACHE'] = self.ctx.ccache env.update( {k: v for k, v in environ.items() if k.startswith('CCACHE_')}) if clang: llvm_dirname = split( glob(join(self.ctx.ndk_dir, 'toolchains', 'llvm*'))[-1])[-1] clang_path = join(self.ctx.ndk_dir, 'toolchains', llvm_dirname, 'prebuilt', build_platform, 'bin') environ['PATH'] = '{clang_path}:{path}'.format( clang_path=clang_path, path=environ['PATH']) exe = join(clang_path, 'clang') execxx = join(clang_path, 'clang++') else: exe = '{command_prefix}-gcc'.format(command_prefix=command_prefix) execxx = '{command_prefix}-g++'.format( command_prefix=command_prefix) cc = find_executable(exe, path=environ['PATH']) if cc is None: print('Searching path are: {!r}'.format(environ['PATH'])) raise BuildInterruptingException( 'Couldn\'t find executable for CC. This indicates a ' 'problem locating the {} executable in the Android ' 'NDK, not that you don\'t have a normal compiler ' 'installed. Exiting.'.format(exe)) if with_flags_in_cc: env['CC'] = '{ccache}{exe} {cflags}'.format(exe=exe, ccache=ccache, cflags=env['CFLAGS']) env['CXX'] = '{ccache}{execxx} {cxxflags}'.format( execxx=execxx, ccache=ccache, cxxflags=env['CXXFLAGS']) else: env['CC'] = '{ccache}{exe}'.format(exe=exe, ccache=ccache) env['CXX'] = '{ccache}{execxx}'.format(execxx=execxx, ccache=ccache) env['AR'] = '{}-ar'.format(command_prefix) env['RANLIB'] = '{}-ranlib'.format(command_prefix) env['LD'] = '{}-ld'.format(command_prefix) env['LDSHARED'] = env["CC"] + " -pthread -shared " +\ "-Wl,-O1 -Wl,-Bsymbolic-functions " if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: # For crystax python, we can't use the host python headers: env["CFLAGS"] += ' -I{}/sources/python/{}/include/python/'.\ format(self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3]) env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix) env['MAKE'] = 'make -j5' env['READELF'] = '{}-readelf'.format(command_prefix) env['NM'] = '{}-nm'.format(command_prefix) hostpython_recipe = Recipe.get_recipe( 'host' + self.ctx.python_recipe.name, self.ctx) env['BUILDLIB_PATH'] = join( hostpython_recipe.get_build_dir(self.arch), 'native-build', 'build', 'lib.{}-{}'.format( build_platform, self.ctx.python_recipe.major_minor_version_string, ), ) env['PATH'] = environ['PATH'] env['ARCH'] = self.arch env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api)) if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version return env
def main(unused_argv): """Main function.""" del unused_argv logging.set_verbosity(logging.INFO) # Prepare proc_in_run = {} # hash: str -> (id_gpu: int, popen: Popen} proc_to_run = {} # hash: str -> cmd: str free_gpus_set = set(range(FLAGS.nb_gpu)) last_state_changed = True # Main loop while True: # logging.info('Loop tick') # logging.info('proc_in_run %s', proc_in_run) # logging.info('proc_to_run %s', proc_to_run) # logging.info('free_gpus_set %s', free_gpus_set) state_changed = False # print and eject finished popen object hash_keys = list(proc_in_run.keys()) for hash_ in hash_keys: id_gpu, popen = proc_in_run[hash_] poll_result = popen.poll() if poll_result is not None: logging.info( 'command done (return code = %d): "%s"', popen.returncode, popen.args, ) free_gpus_set.add(id_gpu) add_to_finished_hash(hash_) del proc_in_run[hash_] state_changed = True # Add job to queue if is not seen before finished_hashs_set = get_finished_hashs_set() cmds = load_cmds() for cmd in cmds: hash_ = hash_cmd(cmd) if (hash_ not in finished_hashs_set) and (hash_ not in proc_in_run) and ( hash_ not in proc_to_run): proc_to_run[hash_] = cmd logging.info('Add to queue: "%s"', shlex.split(cmd)) state_changed = True # Launch if there is slot hash_keys = list(proc_to_run.keys()) for hash_ in hash_keys: cmd = proc_to_run[hash_] if free_gpus_set: logging.info('Launch: "%s"', shlex.split(cmd)) id_gpu = free_gpus_set.pop() env = {k: v for k, v in environ.items()} env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # match the order from nvidia-smi env['CUDA_VISIBLE_DEVICES'] = '%d' % id_gpu popen = subprocess.Popen( args=shlex.split(cmd), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env, ) proc_in_run[hash_] = (id_gpu, popen) del proc_to_run[hash_] state_changed = True if not state_changed: if last_state_changed: # only print once when entering the state-not-changed logging.info('State not changed.') last_state_changed = state_changed # Sleep time.sleep(FLAGS.loop_interval)
def setenv(self, shell, where=stdout): if self.recursive: return ref = where shell_data = self.shell[shell] sep = shell_data['SEP'] udata = {} data = [] index = 0 if not self._unsetenv: env_prefix = self.env_backup_prefix env = self._runtime() for d in env['variables']: for var, val in data.items(): udata[var] = 1 data.append({var: val[0]}) for var in env['path']: if '_SRTOPT_' in var: continue udata[var] = 1 benv = '%s%s%s' % (env_prefix, var, self.backup_type[var]) val = self._fixpathvar(var, sep) if benv in environ: val = environ[benv] + val data.append({var: val}) if shell == 'RTBOURNE': data.append({'SCRAM_RTBOURNE_SET': environ['SCRAMRT_SET']}) for var, val in env['xenv'].items(): udata[var] = 1 data.append({var: val}) for var, val in environ.items(): if var not in udata: data.insert(0, {var: val}) udata[var] = 1 oenv = self.OENV unset = "" unset_vars = "" for v in oenv: if v in udata: continue if v in environ: del environ[v] if shell == 'RTBOURNE': continue unset += " %s" % v if not v.startswith('SCRAMRT_') and \ not v.endswith('_SCRAMRT') and \ not v.endswith('_SCRAMRTDEL'): unset_vars += " %s\n" % v if unset: if unset_vars and not self._unsetenv: SCRAM.printerror( "**** Following environment variables are going to be unset.\n%s" % unset_vars) print("%s %s;" % (shell_data['UNEXPORT'], unset), file=ref) for d in data: for var, val in d.items(): if shell == 'RTBOURNE': if var.startswith('SCRAMRT_') or \ var.endswith('_SCRAMRT') or \ var.endswith('_SCRAMRTDEL'): if var != 'SCRAMRT_SET': del environ[var] continue environ[var] = val continue if var != 'PATH' and var in oenv: if val == oenv[var]: continue environ[var] = val print('%s %s%s\"%s\";' % (shell_data['EXPORT'], var, shell_data['EQUALS'], val), file=stdout) return True XX = """
def _patch_env(*filtered_keys, **kw): env = {k: v for k, v in environ.items() if k not in filtered_keys} env.update(kw) return env
def get_env(self, with_flags_in_cc=True): env = {} env["CFLAGS"] = " ".join([ "-DANDROID", "-mandroid", "-fomit-frame-pointer", "--sysroot", self.ctx.ndk_platform]) env["CXXFLAGS"] = env["CFLAGS"] env["LDFLAGS"] = " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)]) if self.ctx.ndk == 'crystax': env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch) py_platform = sys.platform if py_platform in ['linux2', 'linux3']: py_platform = 'linux' toolchain_prefix = self.ctx.toolchain_prefix toolchain_version = self.ctx.toolchain_version command_prefix = self.command_prefix env['TOOLCHAIN_PREFIX'] = toolchain_prefix env['TOOLCHAIN_VERSION'] = toolchain_version ccache = '' if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))): # print('ccache found, will optimize builds') ccache = self.ctx.ccache + ' ' env['USE_CCACHE'] = '1' env['NDK_CCACHE'] = self.ctx.ccache env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')}) cc = find_executable('{command_prefix}-gcc'.format( command_prefix=command_prefix), path=environ['PATH']) if cc is None: print('Searching path are: {!r}'.format(environ['PATH'])) warning('Couldn\'t find executable for CC. This indicates a ' 'problem locating the {} executable in the Android ' 'NDK, not that you don\'t have a normal compiler ' 'installed. Exiting.') exit(1) if with_flags_in_cc: env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format( command_prefix=command_prefix, ccache=ccache, cflags=env['CFLAGS']) env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format( command_prefix=command_prefix, ccache=ccache, cxxflags=env['CXXFLAGS']) else: env['CC'] = '{ccache}{command_prefix}-gcc'.format( command_prefix=command_prefix, ccache=ccache) env['CXX'] = '{ccache}{command_prefix}-g++'.format( command_prefix=command_prefix, ccache=ccache) env['AR'] = '{}-ar'.format(command_prefix) env['RANLIB'] = '{}-ranlib'.format(command_prefix) env['LD'] = '{}-ld'.format(command_prefix) # env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink') # env['LDSHARED'] = env['LD'] env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix) env['MAKE'] = 'make -j5' env['READELF'] = '{}-readelf'.format(command_prefix) env['NM'] = '{}-nm'.format(command_prefix) hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx) # AND: This hardcodes python version 2.7, needs fixing env['BUILDLIB_PATH'] = join( hostpython_recipe.get_build_dir(self.arch), 'build', 'lib.linux-{}-2.7'.format(uname()[-1])) env['PATH'] = environ['PATH'] env['ARCH'] = self.arch if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version return env
def sanic_config_manager(app: Sanic, prefix: str = "SANIC_"): for variable, value in environ.items(): if variable.startswith(prefix): _, key = variable.split(prefix, 1) app.config[key] = value
# -*- coding: utf-8 -*- import argparse import subprocess from os import environ from shlex import split as shlex_split from textwrap import dedent from unittest import mock from functools import partial import pytest from bumpversion import main, DESCRIPTION, bumpversion from bumpversion.bumpversion import split_args_in_optional_and_positional, RawConfigParser from bumpversion.exceptions import WorkingDirectoryIsDirtyException SUBPROCESS_ENV = dict(list(environ.items()) + [(b'HGENCODING', b'utf-8')]) call = partial(subprocess.call, env=SUBPROCESS_ENV) check_call = partial(subprocess.check_call, env=SUBPROCESS_ENV) check_output = partial(subprocess.check_output, env=SUBPROCESS_ENV) xfail_if_no_git = pytest.mark.xfail(call(["git", "help"]) != 0, reason="git is not installed") @pytest.fixture(params=['.bumpv.cfg', 'setup.cfg']) def configfile(request): return request.param try:
import sys import logging import mock import argparse import subprocess from os import curdir, makedirs, chdir, environ from os.path import join, curdir, dirname from shlex import split as shlex_split from textwrap import dedent from functools import partial from bumpversion import main, DESCRIPTION SUBPROCESS_ENV = dict( list(environ.items()) + [('HGENCODING', 'utf-8')] ) call = partial(subprocess.call, env=SUBPROCESS_ENV) check_call = partial(subprocess.check_call, env=SUBPROCESS_ENV) check_output = partial(subprocess.check_output, env=SUBPROCESS_ENV) xfail_if_no_git = pytest.mark.xfail( call(["git", "--help"], shell=True) != 1, reason="git is not installed" ) xfail_if_no_hg = pytest.mark.xfail( call(["hg", "--help"], shell=True) != 0, reason="hg is not installed" )
import sys import logging import mock import argparse import subprocess from os import curdir, makedirs, chdir, environ from os.path import join, curdir, dirname from shlex import split as shlex_split from textwrap import dedent from functools import partial from bumpversion import main, DESCRIPTION SUBPROCESS_ENV = dict( list(environ.items()) + [(str('HGENCODING'), str('utf-8'))] ) call = partial(subprocess.call, env=SUBPROCESS_ENV) check_call = partial(subprocess.check_call, env=SUBPROCESS_ENV) check_output = partial(subprocess.check_output, env=SUBPROCESS_ENV) xfail_if_no_git = pytest.mark.xfail( call(["git", "--help"], shell=True) != 1, reason="git is not installed" ) xfail_if_no_hg = pytest.mark.xfail( call(["hg", "--help"], shell=True) != 0, reason="hg is not installed" )
from requests import post from requests_oauthlib import OAuth2Session from git import ( Getter, is_authenticated, repo_exists, split_branch_path, get_circle_artifacts, select_path, _LONGTIME, get_branch_info, ERR_TESTS_PENDING, ERR_TESTS_FAILED, skip_webhook_payload, get_webhook_commit_info, post_github_status ) from href import needs_redirect, get_redirect, absolute_url from util import errors_logged, nice_relative_time, parse_webhook_config, err_codes from git import github_client_id, github_client_secret, FAKE_TOKEN, GithubDisallowed from git import ERR_NO_REPOSITORY, ERR_TESTS_PENDING, ERR_TESTS_FAILED, ERR_NO_REF_STATUS flask_secret_key = environ.get('FLASK_SECRET') or 'poop' webhook_config = parse_webhook_config(*[val for (key, val) in environ.items() if key.startswith('WEBHOOK_CONFIG_') or key == 'WEBHOOK_CONFIG']) app = Flask(__name__) if sys.argv[0] != 'test.py': getLogger('precog').info('SSLifying') SSLify(app) app.secret_key = flask_secret_key app.jinja_env.filters['nice_relative_time'] = nice_relative_time app.config['HOOK_SECRETS_TOKENS'] = webhook_config @app.before_first_request def adjust_log_level(): getLogger('precog').setLevel(DEBUG if app.debug else INFO)
A copy of this file may be found in `telemetry-airflow/bin` This file is used as an argument in the SparkPythonTask in the Databricks api.[0] The library is assumed to be installed on all of the machines in the cluster. Arguments are passed to the script through `MOZETL_`-prefixed environment variables. This script is deployed to `s3://telemetry-airflow/steps/mozetl_runner.py`.[1] [0]: https://docs.databricks.com/api/latest/jobs.html#sparkpythontask [1]: https://bugzilla.mozilla.org/show_bug.cgi?id=1484331 """ from os import environ from pprint import pformat from mozetl import cli print( pformat({ k: v for k, v in environ.items() if k.startswith("MOZETL") }) ) try: cli.entry_point(auto_envvar_prefix="MOZETL") except SystemExit: # avoid calling sys.exit() in databricks # http://click.palletsprojects.com/en/7.x/api/?highlight=auto_envvar_prefix#click.BaseCommand.main pass
from flask import Flask from models import initialize_database from os import environ from resources import initialize_resources from schema.schema_file import Schema from sqlalchemy import create_engine # Starting Flask application application = Flask(__name__) # Loading environment variables from .env file only in development if environ.get('EBS_ENVIRONMENT') in ['local', None]: load_dotenv('./environments/local.env') # Loading environment variables into Flask application for item in environ.items(): application.config[item[0]] = item[1] # Starting database configuration initialize_database(application) # Starting RESTful endpoints initialize_resources(application) @application.before_first_request def startup(): print("Initializing migration DB") Schema.migration() Schema.prepare_db()
# Now, activate log file Logger.logfile_activated = bool(Config.getint('kivy', 'log_enable')) # If no configuration exist, write the default one. if ((not exists(kivy_config_fn) or need_save) and 'KIVY_NO_CONFIG' not in environ): try: Config.filename = kivy_config_fn Config.write() except Exception as e: Logger.exception('Core: Error while saving default config file') # Load configuration from env if environ.get('KIVY_NO_ENV_CONFIG', '0') != '1': for key, value in environ.items(): if not key.startswith("KCFG_"): continue try: _, section, name = key.split("_", 2) except ValueError: Logger.warning(("Config: Environ `{}` invalid format, " "must be KCFG_section_name").format(key)) continue # extract and check section section = section.lower() if not Config.has_section(section): Logger.warning( "Config: Environ `{}`: unknown section `{}`".format( key, section))
def get_env(self, with_flags_in_cc=True): env = {} env["CFLAGS"] = " ".join([ "-DANDROID", "-mandroid", "-fomit-frame-pointer", "--sysroot", self.ctx.ndk_platform]) env["CXXFLAGS"] = env["CFLAGS"] env["LDFLAGS"] = " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)]) if self.ctx.ndk == 'crystax': env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch) py_platform = sys.platform if py_platform in ['linux2', 'linux3']: py_platform = 'linux' toolchain_prefix = self.ctx.toolchain_prefix toolchain_version = self.ctx.toolchain_version command_prefix = self.command_prefix env['TOOLCHAIN_PREFIX'] = toolchain_prefix env['TOOLCHAIN_VERSION'] = toolchain_version ccache = '' if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))): print('ccache found, will optimize builds') ccache = self.ctx.ccache + ' ' env['USE_CCACHE'] = '1' env['NDK_CCACHE'] = self.ctx.ccache env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')}) print('path is', environ['PATH']) cc = find_executable('{command_prefix}-gcc'.format( command_prefix=command_prefix), path=environ['PATH']) if cc is None: warning('Couldn\'t find executable for CC. This indicates a ' 'problem locating the {} executable in the Android ' 'NDK, not that you don\'t have a normal compiler ' 'installed. Exiting.') exit(1) if with_flags_in_cc: env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format( command_prefix=command_prefix, ccache=ccache, cflags=env['CFLAGS']) env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format( command_prefix=command_prefix, ccache=ccache, cxxflags=env['CXXFLAGS']) else: env['CC'] = '{ccache}{command_prefix}-gcc'.format( command_prefix=command_prefix, ccache=ccache) env['CXX'] = '{ccache}{command_prefix}-g++'.format( command_prefix=command_prefix, ccache=ccache) env['AR'] = '{}-ar'.format(command_prefix) env['RANLIB'] = '{}-ranlib'.format(command_prefix) env['LD'] = '{}-ld'.format(command_prefix) # env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink') # env['LDSHARED'] = env['LD'] env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix) env['MAKE'] = 'make -j5' env['READELF'] = '{}-readelf'.format(command_prefix) env['NM'] = '{}-nm'.format(command_prefix) hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx) # AND: This hardcodes python version 2.7, needs fixing env['BUILDLIB_PATH'] = join( hostpython_recipe.get_build_dir(self.arch), 'build', 'lib.linux-{}-2.7'.format(uname()[-1])) env['PATH'] = environ['PATH'] env['ARCH'] = self.arch if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version return env
def get_env(self): include_dirs = [ "-I{}/{}".format(self.ctx.include_dir, d.format(arch=self)) for d in self.ctx.include_dirs ] env = {} ccache = sh.which('ccache') cc = sh.xcrun("-find", "-sdk", self.sdk, "clang").strip() if ccache: ccache = ccache.strip() use_ccache = environ.get("USE_CCACHE", "1") if use_ccache != '1': env["CC"] = cc else: if not self._ccsh: self._ccsh = ccsh = sh.mktemp().strip() with open(ccsh, 'w') as f: f.write('#!/bin/sh\n') f.write(ccache + ' ' + cc + ' "$@"\n') sh.chmod('+x', ccsh) else: ccsh = self._ccsh env["USE_CCACHE"] = '1' env["CCACHE"] = ccache env["CC"] = ccsh env.update({ k: v for k, v in environ.items() if k.startswith('CCACHE_') }) env.setdefault('CCACHE_MAXSIZE', '10G') env.setdefault('CCACHE_HARDLINK', 'true') env.setdefault('CCACHE_SLOPPINESS', ( 'file_macro,time_macros,' 'include_file_mtime,include_file_ctime,file_stat_matches')) else: env["CC"] = cc env["AR"] = sh.xcrun("-find", "-sdk", self.sdk, "ar").strip() env["LD"] = sh.xcrun("-find", "-sdk", self.sdk, "ld").strip() env["OTHER_CFLAGS"] = " ".join(include_dirs) env["OTHER_LDFLAGS"] = " ".join([ "-L{}/{}".format(self.ctx.dist_dir, "lib"), ]) env["CFLAGS"] = " ".join([ "-arch", self.arch, "-pipe", "-no-cpp-precomp", "--sysroot", self.sysroot, #"-I{}/common".format(self.ctx.include_dir), #"-I{}/{}".format(self.ctx.include_dir, self.arch), "-O3", self.version_min ] + include_dirs) env["LDFLAGS"] = " ".join([ "-arch", self.arch, "--sysroot", self.sysroot, "-L{}/{}".format(self.ctx.dist_dir, "lib"), "-lsqlite3", self.version_min ]) return env
def get_env(self, with_flags_in_cc=True): env = {} env['CFLAGS'] = ' '.join([ '-DANDROID', '-mandroid', '-fomit-frame-pointer' ' -D__ANDROID_API__={}'.format(self.ctx.ndk_api), ]) env['LDFLAGS'] = ' ' sysroot = join(self.ctx._ndk_dir, 'sysroot') if exists(sysroot): # post-15 NDK per # https://android.googlesource.com/platform/ndk/+/ndk-r15-release/docs/UnifiedHeaders.md env['CFLAGS'] += ' -isystem {}/sysroot/usr/include/{}'.format( self.ctx.ndk_dir, self.ctx.toolchain_prefix) else: sysroot = self.ctx.ndk_platform env['CFLAGS'] += ' -I{}'.format(self.ctx.ndk_platform) env['CFLAGS'] += ' -isysroot {} '.format(sysroot) env['CFLAGS'] += '-I' + join(self.ctx.get_python_install_dir(), 'include/python{}'.format( self.ctx.python_recipe.version[0:3]) ) env['LDFLAGS'] += '--sysroot {} '.format(self.ctx.ndk_platform) env["CXXFLAGS"] = env["CFLAGS"] env["LDFLAGS"] += " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)]) if self.ctx.ndk == 'crystax': env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch) py_platform = sys.platform if py_platform in ['linux2', 'linux3']: py_platform = 'linux' toolchain_prefix = self.ctx.toolchain_prefix toolchain_version = self.ctx.toolchain_version command_prefix = self.command_prefix env['TOOLCHAIN_PREFIX'] = toolchain_prefix env['TOOLCHAIN_VERSION'] = toolchain_version ccache = '' if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))): # print('ccache found, will optimize builds') ccache = self.ctx.ccache + ' ' env['USE_CCACHE'] = '1' env['NDK_CCACHE'] = self.ctx.ccache env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')}) cc = find_executable('{command_prefix}-gcc'.format( command_prefix=command_prefix), path=environ['PATH']) if cc is None: print('Searching path are: {!r}'.format(environ['PATH'])) warning('Couldn\'t find executable for CC. This indicates a ' 'problem locating the {} executable in the Android ' 'NDK, not that you don\'t have a normal compiler ' 'installed. Exiting.') exit(1) if with_flags_in_cc: env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format( command_prefix=command_prefix, ccache=ccache, cflags=env['CFLAGS']) env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format( command_prefix=command_prefix, ccache=ccache, cxxflags=env['CXXFLAGS']) else: env['CC'] = '{ccache}{command_prefix}-gcc'.format( command_prefix=command_prefix, ccache=ccache) env['CXX'] = '{ccache}{command_prefix}-g++'.format( command_prefix=command_prefix, ccache=ccache) env['AR'] = '{}-ar'.format(command_prefix) env['RANLIB'] = '{}-ranlib'.format(command_prefix) env['LD'] = '{}-ld'.format(command_prefix) env['LDSHARED'] = env["CC"] + " -pthread -shared " +\ "-Wl,-O1 -Wl,-Bsymbolic-functions " if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: # For crystax python, we can't use the host python headers: env["CFLAGS"] += ' -I{}/sources/python/{}/include/python/'.\ format(self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3]) env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix) env['MAKE'] = 'make -j5' env['READELF'] = '{}-readelf'.format(command_prefix) env['NM'] = '{}-nm'.format(command_prefix) hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx) # This hardcodes python version 2.7, needs fixing env['BUILDLIB_PATH'] = join( hostpython_recipe.get_build_dir(self.arch), 'build', 'lib.linux-{}-2.7'.format(uname()[-1])) env['PATH'] = environ['PATH'] env['ARCH'] = self.arch env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api)) if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax: env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version return env
def scan_init(self, *args, **kwargs): # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Fetch Script Specific Arguments # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- directory = kwargs.get('directory') nzbname = kwargs.get('nzbname') filename = kwargs.get('filename') category = kwargs.get('category') priority = kwargs.get('priority') top = kwargs.get('top') paused = kwargs.get('paused') parse_nzbfile = kwargs.get('parse_nzbfile') use_database = kwargs.get('use_database') # Fetch/Load Scan Script Configuration script_config = dict([(SCAN_OPTS_RE.match(k).group(1), v.strip()) \ for (k, v) in environ.items() if SCAN_OPTS_RE.match(k)]) if self.vvdebug: # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Print Global Script Varables to help debugging process # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- for k, v in script_config.items(): self.logger.vvdebug('%s%s=%s' % (SCAN_ENVIRO_ID, k, v)) # Merge Script Configuration With System Config self.system = dict(script_config.items() + self.system.items()) # self.directory # This is the path to the destination directory for downloaded files. if directory is None: self.directory = environ.get( '%sDIRECTORY' % SCAN_ENVIRO_ID, ) else: self.directory = directory # self.nzbname # User-friendly name of processed nzb-file as it is displayed by the # program. The file path and extension are removed. If download was # renamed, this parameter reflects the new name. if nzbname is None: self.nzbname = environ.get( '%sNZBNAME' % SCAN_ENVIRO_ID, ) else: self.nzbname = nzbname # self.filename # Name of file to be processed if filename is None: self.filename = environ.get( '%sFILENAME' % SCAN_ENVIRO_ID, ) else: self.filename = filename # self.category # Category assigned to nzb-file (can be empty string). if category is None: self.category = environ.get( '%sCATEGORY' % SCAN_ENVIRO_ID, ) else: self.category = category # self.priority # The priority of the nzb file being scanned if priority is None: self.priority = environ.get( '%sPRIORITY' % SCAN_ENVIRO_ID, ) else: self.priority = priority # self.top # Flag indicating that the file will be added to the top of queue if top is None: self.top = environ.get( '%sTOP' % SCAN_ENVIRO_ID, ) else: self.top = top # self.paused # Flag indicating that the file will be added as paused if paused is None: self.paused = environ.get( '%sPAUSED' % SCAN_ENVIRO_ID, ) else: self.paused = paused # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Error Handling # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- if self.filename: # absolute path names self.filename = abspath(self.filename) if parse_nzbfile: # Initialize information fetched from NZB-File # We intentionally allow existing nzbheaders to over-ride # any found in the nzbfile self.nzbheaders = dict( self.parse_nzbfile( self.filename, check_queued=True)\ .items() + self.pull_dnzb().items(), ) if self.directory: # absolute path names self.directory = abspath(self.directory) if not (self.directory and isdir(self.directory)): self.logger.debug('Process directory is missing: %s' % \ self.directory) else: try: chdir(self.directory) except OSError: self.logger.debug('Process directory is not accessible: %s' % \ self.directory) # Priority if not isinstance(self.priority, int): try: self.priority = int(self.priority) except: self.priority = PRIORITY.NORMAL if self.priority not in PRIORITIES: self.priority = PRIORITY.NORMAL # Top try: self.top = bool(int(self.top)) except: self.top = False # Paused try: self.paused = bool(int(self.paused)) except: self.paused = False # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Enforce system/global variables for script processing # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- self.system['DIRECTORY'] = self.directory if self.directory is not None: environ['%sDIRECTORY' % SCAN_ENVIRO_ID] = self.directory self.system['NZBNAME'] = self.nzbname if self.nzbname is not None: environ['%sNZBNAME' % SCAN_ENVIRO_ID] = self.nzbname self.system['FILENAME'] = self.filename if self.filename is not None: environ['%sFILENAME' % SCAN_ENVIRO_ID] = self.filename self.system['CATEGORY'] = self.category if self.category is not None: environ['%sCATEGORY' % SCAN_ENVIRO_ID] = self.category self.system['PRIORITY'] = self.priority if self.priority is not None: environ['%sPRIORITY' % SCAN_ENVIRO_ID] = str(self.priority) self.system['TOP'] = self.top if self.top is not None: environ['%sTOP' % SCAN_ENVIRO_ID] = str(int(self.top)) self.system['PAUSED'] = self.paused if self.paused is not None: environ['%sPAUSED' % SCAN_ENVIRO_ID] = str(int(self.paused)) # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # Create Database for set() and get() operations # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- if use_database: # database_key is inherited in the parent class # future calls of set() and get() will allow access # to the database now try: self.database_key = basename(self.filename) self.logger.info('Connected to SQLite Database') except AttributeError: pass
def _insert_kube_env(self): return { key: value for (key, value) in environ.items() if "KUBERNETES" in key }
from requests import post from requests_oauthlib import OAuth2Session from git import ( Getter, is_authenticated, repo_exists, split_branch_path, get_circle_artifacts, select_path, _LONGTIME, get_branch_info, ERR_TESTS_PENDING, ERR_TESTS_FAILED, skip_webhook_payload, get_webhook_commit_info, post_github_status ) from href import needs_redirect, get_redirect, absolute_url from util import errors_logged, nice_relative_time, parse_webhook_config, err_codes from git import github_client_id, github_client_secret, FAKE_TOKEN, GithubDisallowed from git import ERR_NO_REPOSITORY, ERR_TESTS_PENDING, ERR_TESTS_FAILED, ERR_NO_REF_STATUS flask_secret_key = environ.get('FLASK_SECRET') or 'poop' webhook_config = parse_webhook_config(*[val for (key, val) in environ.items() if key.startswith('WEBHOOK_CONFIG_') or key == 'WEBHOOK_CONFIG']) app = Flask(__name__) # if sys.argv[0] != 'test.py': # getLogger('precog').info('SSLifying') # SSLify(app) app.secret_key = flask_secret_key app.jinja_env.filters['nice_relative_time'] = nice_relative_time app.config['HOOK_SECRETS_TOKENS'] = webhook_config @app.before_first_request def adjust_log_level(): getLogger('precog').setLevel(DEBUG if app.debug else INFO)
def create_global_comm(nprocs, dir=None): """ Figures out global communicator through external mpi call. """ from sys import executable from tempfile import NamedTemporaryFile from subprocess import PIPE from os.path import exists from os import environ from os import remove, getcwd from re import finditer from .. import mpirun_exe, modify_global_comm, do_multiple_mpi_programs, \ figure_out_machines as script, launch_program as launch from ..misc import local_path from ..error import ConfigError import pylada from ..process import logger logger.debug('process/mpi: create_global_comm: entry') logger.debug('process/mpi: create_global_comm: nprocs: %s' % nprocs) logger.debug('process/mpi: create_global_comm: dir: \"%s\"' % dir) logger.debug('process/mpi: create_global_comm: script: \"%s\"' % script) if not do_multiple_mpi_programs: return if nprocs <= 0: raise MPISizeError(nprocs) if dir is None: dir = getcwd() # each proc prints its name to the standard output. local_path(dir).ensure(dir=True) filename = None try: with NamedTemporaryFile(delete=False, dir=dir) as file: file.write(str(script).encode('utf-8')) filename = file.name formatter = Communicator(n=nprocs).copy() formatter['program'] = executable + ' ' + filename logger.debug("process.mpi: create_global_comm: formatter: %s" % formatter) logger.debug("process.mpi: filename: \"%s\"" % filename) logger.debug("===content:===\n%s===end===" % open(filename).read()) logger.debug("process.mpi: create_global_comm: mpirun_exe: %s " % mpirun_exe) logger.debug("process.mpi: *** launch ***") process = launch(mpirun_exe, stdout=PIPE, formatter=formatter, stderr=PIPE, env=environ) logger.debug("process.mpi: create_global_comm: process: %s" % process) logger.debug("process.mpi: *** start process.communicate ***") stdout, stderr = process.communicate() logger.debug("process.mpi: === start stdout ===\n%s\n=== end ===" % stdout) logger.debug("process.mpi: === start stderr ===\n%s\n=== end ===" % stderr) logger.debug("process.mpi: *** start process.communicate ***") finally: if filename is not None and exists(filename): try: remove(filename) except: pass # we use that to deduce the number of machines and processes per machine. processes = [line.group(1) for line in finditer('PYLADA MACHINE HOSTNAME:\s*(\S*)', stdout)] logger.debug(" process.mpi: create_global_comm: processes: %s" % processes) logger.debug("process.mpi: nprocs: create_global_comm: %s" % nprocs) # sanity check. if nprocs != len(processes): envstring = '' for key, value in environ.items(): envstring += ' {0} = {1!r}\n'.format(key, value) raise ConfigError('Pylada could not determine host machines.\n' 'Standard output reads as follows:\n{0}\n' 'Standard error reads as follows:\n{1}\n' 'environment variables were set as follows:\n{2}\n' 'following script was run:\n{3}\n' .format(stdout, stderr, envstring, script)) # now set up the default communicator. machines = set(processes) pylada.default_comm = Communicator(pylada.default_comm) pylada.default_comm['n'] = nprocs pylada.default_comm.machines = {} for machine in machines: pylada.default_comm.machines[machine] = processes.count(machine) # called when we need to change the machine names on some supercomputers. modify_global_comm(pylada.default_comm)
''' Created on Jan 19, 2019 @author: dad ''' if __name__ == '__main__': pass import os # get environment from os import environ env_items = environ.items() env_values = {(k, v) for (k, v) in environ.items()} import io from sys import argv with open('somefile.txt', 'rb') as inf: txtdata = inf.read() if txtdata.startswith(b'\xff\xd8'): text = u'This is a Text file (%d bytes long)\n' else: text = u'This is a random file (%d bytes long)\n' with io.open('summary.txt', 'w', encoding='utf-8') as outf: outf.write(text % len(txtdata)) env_filtered = [ env_item for env_item in environ.items()
def test_main(self) -> None: """ Tests that main will be called """ with self.assertRaises(SystemExit) as e, patch( "sys.stdout", new_callable=StringIO) as out, patch( "sys.stderr", new_callable=StringIO) as err, patch( "sys.argv", [sys.executable]): self.assertIsNone(main()) help_text, _usage, engine_help_text = err.getvalue().rpartition( "usage") engine_help_text = _usage + engine_help_text engines = help_text[(lambda k: help_text.index(k) + len(k) + 2 )("--engine"):help_text.index("\n") - 2] self.assertEqual( engine_help_text, "usage: python -m ml_params [-h] [--version] [--engine {{{engines}}}]\n" "python -m ml_params: error: --engine must be provided," " and from installed ml-params-* options\n".format( engines=engines), ) self.assertEqual(*map( lambda s: s.replace(" ", "").replace("\n", ""), (( help_text, "usage: python -m ml_params [-h] [--version] [--engine {{{engines}}}]\n\n" "Consistent CLI for every popular ML framework.\n\n" "{option_help}:\n" " -h, --help show this help message and exit\n" " --version show program's version number and exit\n" " --engine {{{engines}}}" ' ML engine, e.g., "TensorFlow", "JAX", "pytorch"\n' .format( option_help="options" if sys.version_info[:2] > (3, 9) else "optional arguments", engines=engines, ), )), )) self.assertEqual(e.exception.code, SystemExit(2).code) self.assertEqual(out.getvalue(), "") # With engine set mod = "ml-params-tensorflow" if mod not in sys.modules: sys.modules[mod] = ml_params # TODO: Some fake test module if not TF_SUPPORTED: return env_backup = dict(environ.items()) environ["ML_PARAMS_ENGINE"] = "tensorflow" _argv = ["--engine", "tensorflow", "-h"] with self.assertRaises(SystemExit) as e, patch( "sys.stdout", new_callable=StringIO) as out, patch( "sys.stderr", new_callable=StringIO) as err, patch( "sys.argv", [sys.executable] + _argv), patch( "os.environ", environ): self.assertIsNone(main(_argv)) self.assertEqual(e.exception.code, SystemExit(0).code) self.assertEqual(err.getvalue(), "") self.assertEqual( out.getvalue(), "Adding CLI parser: load_data_parser ;\n" "Adding CLI parser: load_model_parser ;\n" "Adding CLI parser: train_parser ;\n" "usage: python -m ml_params [-h] [--version] [--engine {{{engines}}}]" "\n{rest}".format( engines=engines, rest= " {load_data,load_model,train} ...\n\n" "Consistent CLI for every popular ML framework.\n\n" "positional arguments:\n" " {load_data,load_model,train}\n" " subcommand to run. Hacked to be chainable.\n\n" "optional arguments:\n" " -h, --help show this help message and exit\n" " --version show program's version number and exit\n" " " + "--engine {{{engines}}}\n" ' ML engine, e.g., "TensorFlow", "JAX", "pytorch"\n' .format(engines=engines), ), ) del environ["ML_PARAMS_ENGINE"] environ.update(env_backup)