def update(self):
# Read per-probe settings from the environment
for k, v in environ.iteritems():
if not k.startswith('OBSRVBL_IPFIX_PROBE_'):
continue
parts = k.rsplit('_', 2)
__, index, attr = parts
try:
int(index)
except ValueError:
logging.error('Probe index must be integer: %s', k)
continue
self.probe_config[index][attr] = v
# YAF-specific probes
if environ.get('OBSRVBL_YAF_CAPTURER', 'false') != 'true':
return
all_ifaces = sorted(environ.get('OBSRVBL_PNA_IFACES', '').split())
yaf_start_port = int(
environ.get(ENV_YAF_START_PORT, DEFAULT_YAF_START_PORT))
for i, iface in enumerate(all_ifaces):
self.probe_config[iface]['TYPE'] = 'ipfix'
self.probe_config[iface]['PORT'] = '{}'.format(yaf_start_port + i)
self.probe_config[iface]['PROTOCOL'] = 'tcp'
self.probe_config[iface]['localhost_only'] = True
def _argmax_base(command):
base_size = 2048
for c in command:
base_size += len(command) + 1
for k, v in environ.iteritems():
base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
return base_size
def loadConfig(path):
spec = conf_spec.splitlines()
config = ConfigObj(path, configspec=spec)
validator = Validator()
config.validate(validator)
# TODO: Find a better way to do this (If the default value for a list is an empty list, it just doesn't set the value at all)
if 'head_nodes' not in config:
config["head_nodes"] = []
for key, value in config.iteritems():
# If any config option has the special value __unique_port__, compute a unique port for it by hashing the user running
# the experiment, the experiment name and the config option name.
if value == '__unique_port__':
md5sum = md5()
md5sum.update(getuser())
md5sum.update(config['experiment_name'])
md5sum.update(key)
config[key] = int(md5sum.hexdigest()[-16:], 16) % 20000 + 20000
# Override config options with env. variables.
revalidate = False
for key, value in environ.iteritems():
if key.startswith("GUMBY_"):
name = key[6:].lower() # "GUMBY_".len()
config[name] = value
revalidate = True
if revalidate:
config.validate(validator)
return config
def _argmax_base(command):
base_size = 2048
for c in command:
base_size += len(command) + 1
for k, v in environ.iteritems():
base_size += len(k) + len(v) + 2 + sizeof(c_void_p)
return base_size
def init_app(self, app, prefix=DEFAULT_ENV_PREFIX):
for key, value in environ.iteritems():
if key.startswith(prefix):
key = key[len(prefix):]
try:
value = ast.literal_eval(value)
except (ValueError, SyntaxError):
pass
app.config[key] = value
def environ_to_condor():
s = ''
for k, v in environ.iteritems():
if any([
x in k
for x in ['PANDA', 'SUBMIT', 'SCRAM_ARCH', 'CMSSW_VERSION']
]):
s += '%s=%s ' % (k, v)
return s
def params(self):
"""
Returns json params from parent environment
"""
params = {}
for param, value in env.iteritems():
params[param] = value
params = dict((k.lower(), v) for k, v in params.iteritems())
return params
def parse_environ(self):
"""Check the environment variables for options."""
config = {}
for key, value in environ.iteritems():
if key.startswith(self.prefix):
name = key[len(self.prefix):].lower()
config[name] = value
self.update(config)
def parse_environ(self):
"""Check the environment variables for options."""
config = {}
for key, value in environ.iteritems():
if key.startswith(self.prefix):
name = key[len(self.prefix):].lower()
config[name] = value
self.update(config)
def __init__(self):
''' Parse os.environ and set all members. The environment variables can have the same
format as the variable names below, but prefixed by LUNCHTIME_
'''
self.ALLOWED_SLACK_ROOMS = ['lunch', 'lunch-test']
self.CHERRYPY_PORT = 9999
self.SLACK_REQUEST_TOKEN = ''
self.INCOMING_WEBOOK_URL = ''
for name, value in environ.iteritems():
name = name.replace(ENVIRONMENT_VAR_PREFIX, '')
if hasattr(self, name):
setattr(self, name, value)
def __init__(self):
''' Parse os.environ and set all members. The environment variables can have the same
format as the variable names below, but prefixed by LUNCHTIME_
'''
self.ALLOWED_SLACK_ROOMS = ['lunch', 'lunch-test']
self.CHERRYPY_PORT = 9999
self.SLACK_REQUEST_TOKEN = ''
self.INCOMING_WEBOOK_URL = ''
for name, value in environ.iteritems():
name = name.replace(ENVIRONMENT_VAR_PREFIX, '')
if hasattr(self, name):
setattr(self, name, value)
def execute(self, phase):
cmds = getattr(self, phase, None)
if cmds == None: return
print("#" * 80)
print("# executing %s '%s' phase" % (self.name, phase))
print("#")
dev_null = file("/dev/null")
logdir = "doc/buildlogs"
if not os.path.isdir(logdir): os.mkdir(logdir)
logpath = logdir + "/%s-%s.log" % (self.name, phase)
T = subprocess.Popen(["tee", logpath], stdin=subprocess.PIPE)
try:
import time
log = lambda msg="": print(msg, file=T.stdin)
log(time.asctime())
log()
log("\n".join("%s=%s" % nv
for nv in filter(lambda i: i[0] in (
"TOPDIR", "PREFIX", "BINDIR", "LIBDIR",
"MACHTYPE", "NCPUS_ONLN"), env.iteritems())))
log()
if type(cmds) not in (list, tuple):
cmds = cmds, # make it a 1-tuple
for cmd in cmds:
log("$ cd " + self.subdir)
log("$ " + "\n ".join(str(cmd).split("\n")))
log()
if type(cmd) == str:
ret = subprocess.call(cmd, shell=True, cwd=self.subdir,
stdin=dev_null, stdout=T.stdin, stderr=T.stdin)
elif type(cmd) == MethodType:
def tgt():
sys.stdin = dev_null
sys.stdout = sys.stderr = T.stdin
os.chdir(self.subdir)
return cmd()
sp = multiprocessing.Process(target=tgt)
sp.start()
sp.join()
ret = sp.exitcode
else:
raise Exception("Invalid '%s' phase command: %s" % (
phase, repr(cmd)))
log()
log(time.asctime())
if ret != 0:
raise Exception("%s '%s' phase failed -- exit code %d" % (
self.name, phase, ret))
finally:
T.stdin.close()
T.wait()
def index():
response.content_type = 'text/plain; charset=utf-8'
ret = 'Hello world, I\'m %s!\n\n' % os.getpid()
ret += 'Request vars:\n'
for k, v in request.environ.iteritems():
if 'bottle.' in k:
continue
ret += '%s=%s\n' % (k, v)
ret += '\n'
ret += 'Environment vars:\n'
for k, v in env.iteritems():
if 'bottle.' in k:
continue
ret += '%s=%s\n' % (k, v)
return ret
def __geek():
response.content_type = 'text/plain; charset=utf-8'
ret = 'Hello world, I\'m %s!\n\n' % os.getpid()
ret += 'Request vars:\n'
for k, v in request.environ.iteritems():
if 'bottle.' in k:
continue
ret += '%s=%s\n' % (k, v)
ret += '\n'
ret += 'Environment vars:\n'
for k, v in env.iteritems():
if 'bottle.' in k:
continue
ret += '%s=%s\n' % (k, v)
return ret
def _setup():
import json
import re
from collections import OrderedDict
from ast import literal_eval
from os import path, environ
from warnings import warn
environ_var = re.compile('YOUTUBE_TO_POCKET_([A-Z_]+)')
def relopen(name, *args, **kwargs):
return open(path.join(path.dirname(__file__), name), *args, **kwargs)
with relopen('default.json') as default:
config = json.load(default, object_pairs_hook=OrderedDict)
try:
with relopen('config.json') as config_fp:
config.update(json.load(config_fp, object_pairs_hook=OrderedDict))
except IOError:
warn('user config is missing')
# Load options from environment and parse them as Python literals
for key, value in environ.iteritems():
mo = environ_var.match(key)
if mo is not None:
option = mo.group(1).upper()
try:
config[option] = literal_eval(value)
except (ValueError, SyntaxError):
config[option] = value
config_keys = config.iterkeys
def save():
for key in config_keys():
config[key] = globals()[key]
with relopen('config.json', 'w') as config_fp:
json.dump(config, config_fp, indent=4)
globals()['save'] = save
globals().update(config)
def __prepare(self):
try:
self.job.updateStatus("Running", "PreparingInputData", hostname=gethostname(), batchId=self.batchId)
except Exception as err:
self.logThis("EXCEPTION: %s", err)
# first, set all variables
self.logThis("current environment settings")
for key, value in sorted(environ.iteritems()):
print "%s = %s"%(key,value)
self.logThis("end of environment dump")
# log.info("\n".join(["%s: %s"%(key,value) for key, value in sorted(environ.iteritems())]))
for fi in self.job.InputFiles:
src = expandvars(fi['source'])
tg = expandvars(fi['target'])
self.logThis("Staging %s --> %s",src,oPjoin(abspath(self.pwd),tg))
try:
safe_copy(src, tg, attempts=4, sleep='4s', checksum=True)
except IOError, e:
try:
self.job.updateStatus("Running" if self.debug else "Failed", camelize(e))
except Exception as err:
self.logThis("EXCEPTION: %s", err)
self.job.logError(err)
if not self.debug: return 4
def index():
response.content_type = 'text/text; charset=utf-8'
ret = 'Hi there, I\'m process {0}!\n\n'.format(os.getpid())
sentence = 'Now is better than never.'
ret += 'Testing TextBlob ngram (n=3) with sentence: \n "{0}" \n'.format(sentence)
blob = TextBlob(sentence)
for word_list in blob.ngrams(n=3):
ret += (' '.join(word_list) + '\n')
data = pd.DataFrame({'A': np.random.randn(3), 'B': np.random.randn(3)})
func = "pd.DataFrame({'A': np.random.randn(3), 'B': np.random.randn(3)})"
ret += '\nTesting Numpy and Pandas with command: \n {0} \n{1} \n'.format(func, data.to_json())
ret += '\nCode at: \n https://github.com/alyssaq/bottle-heroku-skeleton \n'
ret += '\nEnvironment vars:\n'
for k, v in env.iteritems():
if 'bottle.' in k:
continue
ret += '%s=%s\n' % (k, v)
return ret
#!/usr/bin/python
from os import environ
print "Content-Type: text/html"
print
print """
<table>
<thead>
<tr>
<th>Name</th>
<th>Value</th>
</tr>
</thead>
<tbody>
"""
for x in environ.iteritems():
print "<tr><td>%s</td><td>%s</td></tr>" % x
print """
</tbody>
</table>
"""
def generate(env):
from os import environ
if 'CCC_ANALYZER_OUTPUT_FORMAT' in environ:
env['CC'] = environ['CC']
env['CXX'] = environ['CXX']
env['ENV'].update(item for item in environ.iteritems() if item[0].startswith('CCC_'))
def environ_to_condor():
s = ''
for k, v in environ.iteritems():
if any([x in k for x in ['PANDA', 'SUBMIT', 'USER']]):
s += '%s=%s ' % (k, v)
return s
def create_global_comm(nprocs, dir=None):
""" Figures out global communicator through external mpi call. """
from sys import executable
from tempfile import NamedTemporaryFile
from subprocess import PIPE
from os.path import exists
from os import environ
from os import remove, getcwd
from re import finditer
from .. import mpirun_exe, modify_global_comm, do_multiple_mpi_programs, \
figure_out_machines as script, launch_program as launch
from ..misc import Changedir
from ..error import ConfigError
import pylada
from pylada.misc import bugLev
if bugLev >= 1:
print 'process/mpi: create_global_comm: entry'
print 'process/mpi: create_global_comm: nprocs: %s' % (nprocs, )
print 'process/mpi: create_global_comm: dir: \"%s\"' % (dir, )
print 'process/mpi: create_global_comm: script: \"%s\"' % (script, )
if not do_multiple_mpi_programs: return
if nprocs <= 0: raise MPISizeError(nprocs)
if dir is None: dir = getcwd()
# each proc prints its name to the standard output.
with Changedir(dir) as pwd:
pass
try:
with NamedTemporaryFile(delete=False, dir=dir) as file:
file.write(script)
filename = file.name
formatter = Communicator(n=nprocs).copy()
formatter['program'] = executable + ' ' + filename
if bugLev >= 5:
print "process.mpi: create_global_comm: formatter: ", formatter
print "process.mpi: filename: \"%s\"" % (filename, )
print "===content:===\n%s===end===" % (open(filename).read(), )
print "process.mpi: create_global_comm: mpirun_exe: ", mpirun_exe
print "process.mpi: *** launch ***"
process = launch(mpirun_exe,
stdout=PIPE,
formatter=formatter,
stderr=PIPE,
env=environ)
if bugLev >= 5:
print "process.mpi: create_global_comm: process: ", process
print "process.mpi: *** start process.communicate ***"
stdout, stderr = process.communicate()
if bugLev >= 5:
print "process.mpi: === start stdout ===\n%s\n=== end ===" % (
stdout, )
print "process.mpi: === start stderr ===\n%s\n=== end ===" % (
stderr, )
print "process.mpi: *** start process.communicate ***"
finally:
if exists(filename):
try:
remove(filename)
except:
pass
# we use that to deduce the number of machines and processes per machine.
processes = [ line.group(1) for line \
in finditer('PYLADA MACHINE HOSTNAME:\s*(\S*)', stdout) ]
if bugLev >= 5:
for p in processes:
print " process.mpi: create_global_comm: process: ", p
print "process.mpi: nprocs: create_global_comm: ", nprocs
# sanity check.
if nprocs != len(processes):
envstring = ''
for key, value in environ.iteritems():
envstring += ' {0} = {1!r}\n'.format(key, value)
raise ConfigError( 'Pylada could not determine host machines.\n' \
'Standard output reads as follows:\n{0}\n' \
'Standard error reads as follows:\n{1}\n' \
'environment variables were set as follows:\n{2}\n' \
'following script was run:\n{3}\n' \
.format(stdout, stderr, envstring, script) )
# now set up the default communicator.
machines = set(processes)
pylada.default_comm = Communicator(pylada.default_comm)
pylada.default_comm['n'] = nprocs
pylada.default_comm.machines = {}
for machine in machines:
pylada.default_comm.machines[machine] = processes.count(machine)
# called when we need to change the machine names on some supercomputers.
modify_global_comm(pylada.default_comm)
def environ_to_condor():
s = ''
for k, v in environ.iteritems():
if 'SUBMIT' in k or 'USER' in k:
s += '%s=%s ' % (k, v)
return s
timeout = None
if opts.timeout != None:
timeout = opts.timeout
print "timeout =", timeout
exit_status = 0 #so we can make flags for non-zero statuses later.
if opts.exit_status != None:
exit_status = opts.exit_status
print "exit_status =", exit_status
print >> stdout, "Stdout: CHECK!"
print >> stderr, '#' * 80
print >> stderr, "Args:\n", "\n".join(argv)
print >> stderr, '#' * 80
print >> stderr, "Envs:\n", "\n".join(["%s=%s" %(key, val) for key, val in environ.iteritems()])
#drop other diagnostics here
if timeout != None:
run_time = timeout
else:
run_time = randrange(90, 150)
print "Starting simulator_run for %s seconds." % run_time
sleep(run_time)
print "run completed successfully!"
if exit_status == 0:
print >> stderr, "Exiting with status 0."
exit(0)
else:
print >> stderr, "Like the Not-an-exit exit, I Fail!"
def create_global_comm(nprocs, dir=None):
""" Figures out global communicator through external mpi call. """
from sys import executable
from tempfile import NamedTemporaryFile
from subprocess import PIPE
from os.path import exists
from os import environ
from os import remove, getcwd
from re import finditer
from .. import (
mpirun_exe,
modify_global_comm,
do_multiple_mpi_programs,
figure_out_machines as script,
launch_program as launch,
)
from ..misc import Changedir
from ..error import ConfigError
import pylada
from pylada.misc import bugLev
if not do_multiple_mpi_programs:
return
if nprocs <= 0:
raise MPISizeError(nprocs)
if dir is None:
dir = getcwd()
# each proc prints its name to the standard output.
with Changedir(dir) as pwd:
pass
try:
with NamedTemporaryFile(delete=False, dir=dir) as file:
file.write(script)
filename = file.name
formatter = Communicator(n=nprocs).copy()
formatter["program"] = executable + " " + filename
if bugLev >= 5:
print "process.mpi: create_global_comm: formatter: ", formatter
print 'process.mpi: filename: "%s"' % (filename,)
print "===content:===\n%s===end===" % (open(filename).read(),)
print "process.mpi: create_global_comm: mpirun_exe: ", mpirun_exe
print "process.mpi: *** launch ***"
process = launch(mpirun_exe, stdout=PIPE, formatter=formatter, stderr=PIPE, env=environ)
if bugLev >= 5:
print "process.mpi: create_global_comm: process: ", process
print "process.mpi: *** start process.communicate ***"
stdout, stderr = process.communicate()
finally:
if exists(filename):
try:
remove(filename)
except:
pass
# we use that to deduce the number of machines and processes per machine.
processes = [line.group(1) for line in finditer("PYLADA MACHINE HOSTNAME:\s*(\S*)", stdout)]
if bugLev >= 5:
for p in processes:
print " process.mpi: create_global_comm: process: ", p
print "process.mpi: nprocs: create_global_comm: ", nprocs
# sanity check.
if nprocs != len(processes):
envstring = ""
for key, value in environ.iteritems():
envstring += " {0} = {1!r}\n".format(key, value)
raise ConfigError(
"Pylada could not determine host machines.\n"
"Standard output reads as follows:\n{0}\n"
"Standard error reads as follows:\n{1}\n"
"environment variables were set as follows:\n{2}\n"
"following script was run:\n{3}\n".format(stdout, stderr, envstring, script)
)
# now set up the default communicator.
machines = set(processes)
pylada.default_comm = Communicator(pylada.default_comm)
pylada.default_comm["n"] = nprocs
pylada.default_comm.machines = {}
for machine in machines:
pylada.default_comm.machines[machine] = processes.count(machine)
# called when we need to change the machine names on some supercomputers.
modify_global_comm(pylada.default_comm)