Exemple #1
0
def _argmax_base(command):
    base_size = 2048
    for c in command:
        base_size += len(command) + 1
    for k, v in environ.iteritems():
        base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
    return base_size
Exemple #2
0
def loadConfig(path):
    spec = conf_spec.splitlines()
    config = ConfigObj(path, configspec=spec)
    validator = Validator()
    config.validate(validator)
    # TODO: Find a better way to do this (If the default value for a list is an empty list, it just doesn't set the value at all)
    if 'head_nodes' not in config:
        config["head_nodes"] = []
    for key, value in config.iteritems():
        # If any config option has the special value __unique_port__, compute a unique port for it by hashing the user running
        # the experiment, the experiment name and the config option name.
        if value == '__unique_port__':
            md5sum = md5()
            md5sum.update(getuser())
            md5sum.update(config['experiment_name'])
            md5sum.update(key)
            config[key] = int(md5sum.hexdigest()[-16:], 16) % 20000 + 20000

    # Override config options with env. variables.
    revalidate = False
    for key, value in environ.iteritems():
        if key.startswith("GUMBY_"):
            name = key[6:].lower()  # "GUMBY_".len()
            config[name] = value
            revalidate = True
    if revalidate:
        config.validate(validator)
    return config
Exemple #3
0
    def parse_environ(self):
        """Check the environment variables for options."""

        config = {}

        for key, value in environ.iteritems():
            if key.startswith(self.prefix):
                name = key[len(self.prefix):].lower()
                config[name] = value

        self.update(config)
Exemple #4
0
 def __init__(self):
     ''' Parse os.environ and set all members. The environment variables can have the same
         format as the variable names below, but prefixed by LUNCHTIME_
     '''
     self.ALLOWED_SLACK_ROOMS = ['lunch', 'lunch-test']
     self.CHERRYPY_PORT = 9999
     self.SLACK_REQUEST_TOKEN = ''
     self.INCOMING_WEBOOK_URL = ''
     for name, value in environ.iteritems():
         name = name.replace(ENVIRONMENT_VAR_PREFIX, '')
         if hasattr(self, name):
             setattr(self, name, value)
Exemple #5
0
def __geek():
    response.content_type = 'text/plain; charset=utf-8'
    ret =  'Hello world, I\'m %s!\n\n' % os.getpid()
    ret += 'Request vars:\n'
    for k, v in request.environ.iteritems():
        if 'bottle.' in k:
            continue
        ret += '%s=%s\n' % (k, v)

    ret += '\n'
    ret += 'Environment vars:\n'

    for k, v in env.iteritems():
        if 'bottle.' in k:
            continue
        ret += '%s=%s\n' % (k, v)

    return ret
Exemple #6
0
def _setup():
    import json
    import re
    from collections import OrderedDict
    from ast import literal_eval
    from os import path, environ
    from warnings import warn

    environ_var = re.compile('YOUTUBE_TO_POCKET_([A-Z_]+)')

    def relopen(name, *args, **kwargs):
        return open(path.join(path.dirname(__file__), name), *args, **kwargs)

    with relopen('default.json') as default:
        config = json.load(default, object_pairs_hook=OrderedDict)
    try:
        with relopen('config.json') as config_fp:
            config.update(json.load(config_fp, object_pairs_hook=OrderedDict))
    except IOError:
        warn('user config is missing')

    # Load options from environment and parse them as Python literals
    for key, value in environ.iteritems():
        mo = environ_var.match(key)
        if mo is not None:
            option = mo.group(1).upper()
            try:
                config[option] = literal_eval(value)
            except (ValueError, SyntaxError):
                config[option] = value

    config_keys = config.iterkeys

    def save():
        for key in config_keys():
            config[key] = globals()[key]
        with relopen('config.json', 'w') as config_fp:
            json.dump(config, config_fp, indent=4)

    globals()['save'] = save
    globals().update(config)
 def __prepare(self):
     try:
         self.job.updateStatus("Running", "PreparingInputData", hostname=gethostname(), batchId=self.batchId)
     except Exception as err:
         self.logThis("EXCEPTION: %s", err)
     # first, set all variables
     self.logThis("current environment settings")
     for key, value in sorted(environ.iteritems()):
         print "%s = %s"%(key,value)
     self.logThis("end of environment dump")
     # log.info("\n".join(["%s: %s"%(key,value) for key, value in sorted(environ.iteritems())]))
     for fi in self.job.InputFiles:
         src = expandvars(fi['source'])
         tg = expandvars(fi['target'])
         self.logThis("Staging %s --> %s",src,oPjoin(abspath(self.pwd),tg))
         try:
             safe_copy(src, tg, attempts=4, sleep='4s', checksum=True)
         except IOError, e:
             try:
                 self.job.updateStatus("Running" if self.debug else "Failed", camelize(e))
             except Exception as err:
                 self.logThis("EXCEPTION: %s", err)
                 self.job.logError(err)
             if not self.debug: return 4
def index():
  response.content_type = 'text/text; charset=utf-8'
  
  ret =  'Hi there, I\'m process {0}!\n\n'.format(os.getpid())

  sentence = 'Now is better than never.'
  ret += 'Testing TextBlob ngram (n=3) with sentence: \n "{0}" \n'.format(sentence)
  blob = TextBlob(sentence)
  for word_list in blob.ngrams(n=3):
    ret += (' '.join(word_list) + '\n')
  
  data = pd.DataFrame({'A': np.random.randn(3), 'B': np.random.randn(3)})
  func = "pd.DataFrame({'A': np.random.randn(3), 'B': np.random.randn(3)})"
  ret += '\nTesting Numpy and Pandas with command: \n {0} \n{1} \n'.format(func, data.to_json())
    
  ret += '\nCode at: \n https://github.com/alyssaq/bottle-heroku-skeleton \n'
  ret += '\nEnvironment vars:\n'

  for k, v in env.iteritems():
    if 'bottle.' in k:
      continue
    ret += '%s=%s\n' % (k, v)

  return ret
    timeout = None
    if opts.timeout != None:
        timeout = opts.timeout
    print "timeout =", timeout

    exit_status = 0 #so we can make flags for non-zero statuses later.
    if opts.exit_status != None:
        exit_status = opts.exit_status

    print "exit_status =", exit_status

    print >> stdout, "Stdout: CHECK!"
    print >> stderr, '#' * 80
    print >> stderr, "Args:\n", "\n".join(argv)
    print >> stderr, '#' * 80
    print >> stderr, "Envs:\n", "\n".join(["%s=%s" %(key, val) for key, val in environ.iteritems()])

    #drop other diagnostics here
    if timeout != None:
        run_time = timeout
    else:
        run_time = randrange(90, 150)
    print "Starting simulator_run for %s seconds." % run_time
    sleep(run_time)
    print "run completed successfully!"

    if exit_status == 0:
        print >> stderr, "Exiting with status 0."
        exit(0)
    else:
        print >> stderr, "Like the Not-an-exit exit, I Fail!"
Exemple #10
0
def create_global_comm(nprocs, dir=None):
    """ Figures out global communicator through external mpi call. """
    from sys import executable
    from tempfile import NamedTemporaryFile
    from subprocess import PIPE
    from os.path import exists
    from os import environ
    from os import remove, getcwd
    from re import finditer
    from .. import (
        mpirun_exe,
        modify_global_comm,
        do_multiple_mpi_programs,
        figure_out_machines as script,
        launch_program as launch,
    )
    from ..misc import Changedir
    from ..error import ConfigError
    import pylada
    from pylada.misc import bugLev

    if not do_multiple_mpi_programs:
        return
    if nprocs <= 0:
        raise MPISizeError(nprocs)
    if dir is None:
        dir = getcwd()

    # each proc prints its name to the standard output.
    with Changedir(dir) as pwd:
        pass
    try:
        with NamedTemporaryFile(delete=False, dir=dir) as file:
            file.write(script)
            filename = file.name

        formatter = Communicator(n=nprocs).copy()
        formatter["program"] = executable + " " + filename

        if bugLev >= 5:
            print "process.mpi: create_global_comm: formatter: ", formatter
            print 'process.mpi: filename: "%s"' % (filename,)
            print "===content:===\n%s===end===" % (open(filename).read(),)
            print "process.mpi: create_global_comm: mpirun_exe: ", mpirun_exe
            print "process.mpi: *** launch ***"

        process = launch(mpirun_exe, stdout=PIPE, formatter=formatter, stderr=PIPE, env=environ)
        if bugLev >= 5:
            print "process.mpi: create_global_comm: process: ", process
            print "process.mpi: *** start process.communicate ***"

        stdout, stderr = process.communicate()
    finally:
        if exists(filename):
            try:
                remove(filename)
            except:
                pass
    # we use that to deduce the number of machines and processes per machine.
    processes = [line.group(1) for line in finditer("PYLADA MACHINE HOSTNAME:\s*(\S*)", stdout)]
    if bugLev >= 5:
        for p in processes:
            print "  process.mpi: create_global_comm: process: ", p
        print "process.mpi: nprocs: create_global_comm: ", nprocs
    # sanity check.
    if nprocs != len(processes):
        envstring = ""
        for key, value in environ.iteritems():
            envstring += "  {0} = {1!r}\n".format(key, value)
        raise ConfigError(
            "Pylada could not determine host machines.\n"
            "Standard output reads as follows:\n{0}\n"
            "Standard error reads as follows:\n{1}\n"
            "environment variables were set as follows:\n{2}\n"
            "following script was run:\n{3}\n".format(stdout, stderr, envstring, script)
        )
    # now set up the default communicator.
    machines = set(processes)
    pylada.default_comm = Communicator(pylada.default_comm)
    pylada.default_comm["n"] = nprocs
    pylada.default_comm.machines = {}
    for machine in machines:
        pylada.default_comm.machines[machine] = processes.count(machine)

    # called when we need to change the machine names on some supercomputers.
    modify_global_comm(pylada.default_comm)
def generate(env):
    from os import environ
    if 'CCC_ANALYZER_OUTPUT_FORMAT' in environ:
        env['CC'] = environ['CC']
        env['CXX'] = environ['CXX']
        env['ENV'].update(item for item in environ.iteritems() if item[0].startswith('CCC_'))