def read_config(self, **kw):
"""Initial state of a runner. The module-runtime.cfg is read
and the relevent runner section is applied. The environment
list is created and setup in order to construct the
environments next
"""
self.config = AttrDict(executable="", environments="")
self.sections = {}
self.debug("BaseRunner: read_config")
# Load any bisque related variable into this runner
runtime_bisque_cfg = find_config_path('runtime-bisque.cfg')
if os.path.exists(runtime_bisque_cfg):
self.bisque_cfg = ConfigFile(runtime_bisque_cfg)
self.load_section(None, self.bisque_cfg)
self.config['files'] = self.config.setdefault(
'files', '') + ',' + runtime_bisque_cfg
else:
self.info("BaseRunner: missing runtime-bisque.cfg")
module_dir = kw.get('module_dir', os.getcwd())
runtime_module_cfg = os.path.join(module_dir, 'runtime-module.cfg')
if not os.path.exists(runtime_module_cfg):
self.info("BaseRunner: missing %s", runtime_module_cfg)
return
self.module_cfg = ConfigFile(runtime_module_cfg)
# Process Command section
self.load_section(None, self.module_cfg) # Globals
def choose_runner(self, **kw):
curdir = os.getcwd()
module_dir = kw.get('module_dir', curdir)
log.debug('Choosing runner : cwd: %s module %s', curdir, module_dir)
# Choose a runner
runtime_module_cfg = os.path.join(module_dir, 'runtime-module.cfg')
if os.path.exists(runtime_module_cfg):
cfg = ConfigFile(runtime_module_cfg)
launcher = cfg.get(None, 'launcher')
if launcher and os.path.abspath(
sys.argv[0]) != os.path.abspath(launcher):
if os.path.exists(launcher):
# Given launcher
cmd = launcher.split(' ') # pylint: disable=no-member
cmd.extend(sys.argv[1:])
log.debug("Calling %s" % cmd)
subprocess.call(cmd)
return None
log.warn("Missing launcher: %s", launcher)
return None
else:
log.error("Missing %s", runtime_module_cfg)
return None
runners = cfg.get(None, 'runtime.platforms')
if runners is None:
raise ModuleEnvironmentError(
"Must define legal platforms: runtime.platforms in module config"
)
log.debug('Module runners: %s', runners)
self.module_runners = [r.strip() for r in runners.split(',')] # pylint: disable=no-member
runtime_bisque = 'runtime-bisque.cfg'
if not os.path.exists(runtime_bisque):
runtime_bisque = find_config_path(runtime_bisque)
if os.path.exists(runtime_bisque):
cfg = ConfigFile(runtime_bisque)
runners = cfg.get(None, 'runtime.platforms')
log.debug('System runners: %s', runners)
self.system_runners = [r.strip() for r in runners.split(',')] # pylint: disable=no-member
else:
log.warn("Could not file %s", runtime_bisque)
# Default to module runners ? They may not be availble
self.system_runners = self.module_runners
# Determine best platform for module by comparing system platforms
# and module platforms. The platforms are listed in order
# of preference based on the system preferences
for run_platform in self.system_runners:
if run_platform in self.module_runners:
log.debug("Choosing Runtime Platform: %s", run_platform)
return RUNNER_MAP[run_platform]
return None
def read_config(filename, section=None):
if not os.path.exists(filename):
filename = find_config_path(filename)
print "READING", filename
conf = ConfigFile(filename).get(section, asdict=True)
config_dir = os.path.dirname(filename)
conf['bisque.path.config'] = config_dir
print "CONFIG", config_dir
return conf
def python_setup(scripts,
package_scripts=True,
dependency_dir='pydist',
params=None):
"""compile python dependencies into a package using pyinstaller
if package_script is true then a runner scripts will be generated
for each script
"""
cmd = [
'pyinstaller', '--clean', '--noconfirm', '--distpath', dependency_dir
]
#fr = bbfreeze.Freezer(dependency_dir)
#fr.include_py = False
if not isinstance(scripts, list):
scripts = [scripts]
# check if python code is newer than dependency_dir (pydist)
#if not any (needs_update(dependency_dir, x) for x in scripts):
# print "Skipping python compilation %s is newer than %s" % (dependency_dir, scripts)
# return 0
# if we need to compile, then compile cleanly
if os.path.exists(dependency_dir):
shutil.rmtree(dependency_dir)
data = dict(params or {})
for script in scripts:
cmd.append(script)
check_call(cmd)
# read default path from site.cfg (TG config not initialized yet when this runs)
#site_config = read_config('site.cfg', "app:main")
#filename = os.path.abspath(os.path.join(site_config['bisque.paths.default'], 'templates/python_launcher.tmpl').replace('\\', '/'))
filename = find_config_path('templates/python_launcher.tmpl',
config_dir='config-defaults').replace(
'\\', '/')
print "TEMPLATE", filename
for script in scripts:
script_name = os.path.splitext(script)[0]
data['script'] = os.path.join('.', dependency_dir, script_name,
script_name)
# THIS os.path.join needs to be replaced by pkg_resources or pkg_util
# when the toplevel is packaged
try:
template = Template(filename=filename)
with open(script_name, 'wb') as f:
f.write(template.render(script=data['script']))
os.chmod(script_name, 0744)
return 0
except Exception, e:
print("Could not create python launcher script %s" % e)
def upgrade():
"Find resource references in string and migrate to new resource_uniq id's"
print("Sending migration info to migratelog.txt")
logfile = open('migratelog.txt', 'wb')
parser = ConfigParser()
parser.read(find_config_path("site.cfg"))
bisque_root = parser.get('servers', 'h1.url')
print("Using new bisque root %s" % bisque_root)
print("ROOT", bisque_root, file=logfile)
root = urlparse.urlsplit(bisque_root)
cntxt = context.get_context()
SessionMaker = sessionmaker(bind=cntxt.bind)
# toplevel = DBSession.query(Taggable).filter(Taggable.resource_parent_id == None, Taggable.resource_uniq == None)
# for resource in toplevel:
# uid = make_short_uuid()
# #resource.resource_uniq = "00-%s" % uid
# print resource.resource_type, uid
# resource.resource_uniq = "00-%s" % uid
Base = declarative_base()
metadata = MetaData(bind=cntxt.bind)
class Taggable(Base):
__table__ = Table('taggable', metadata, autoload=True)
def __str__(self):
return u"<%s id=%s name=%s value=%s ts=%s document=%s>" % (
self.resource_type, self.id, self.resource_name,
self.resource_value, self.ts, self.document_id)
DBSession = SessionMaker()
Tag = Taggable
alltags = DBSession.query(Tag).filter(
and_(Tag.resource_type == 'tag', Tag.resource_value.like('http%')))
for tag in alltags:
url = tag.resource_value
parts = list(urlparse.urlsplit(url))
# (scheme, host, path, ...)
path = parts[2].split('/')
ident = None
try:
while path:
x = path.pop(0)
if x and x[0].isdigit():
ident = int(x)
break
except (ValueError, IndexError):
print("badvalue %s %s" % (x, url), file=logfile)
continue
if ident is None:
print("badvalue %s" % tag, file=logfile)
continue
target = DBSession.query(Taggable).filter_by(id=ident).first()
if target:
if target.resource_uniq is None:
print("badpointer %s points to uniq=None %s" % (tag, target),
file=logfile)
continue
#print "mapping %s -> %s /data_service/%s/%s" % (ident, target, target.resource_uniq, "/".join(path))
parts[0] = root[0]
parts[1] = root[1]
parts[2] = posixpath.join("/data_service", target.resource_uniq,
"/".join(path)).rstrip('/')
newurl = urlparse.urlunsplit(parts)
print("mapping %s -> %s" % (tag, newurl), file=logfile)
tag.resource_value = newurl
else:
print("unknown mapping %s" % tag, file=logfile)
def operation(command, options, *args):
"""Run a multi-server command to start several bisque jobs
"""
def verbose(msg):
if options.verbose:
print msg
cfg_file=options.site or SITE_CFG
site_cfg = find_config_path(cfg_file)
if site_cfg is None:
print "Cannot find %s.. please make sure you are in the bisque dir" % cfg_file
return
site_dir = os.path.dirname(os.path.abspath(site_cfg))
verbose('using config : %s' % site_cfg)
try:
config = readhostconfig(site_cfg)
verbose("SERVERS %s" % (config['servers'].keys()))
processes = []
cfgopt = {} # {'root': config['root']}
cfgopt['site_dir'] = site_dir
cfgopt['site_cfg'] = site_cfg
cfgopt['virtualenv'] = os.getenv('VIRTUAL_ENV')
for f in ('log_dir', 'pid_dir', 'logging_server'):
if f in config:
cfgopt[f] = config[f]
if not os.path.exists (config['run_dir']):
print "Please check your site.cfg. bisque.paths.run is not set correctly"
return
os.chdir (config['run_dir'])
# Get the backend
backend = config.get('backend', None)
verbose("using backend: " + str(backend))
if backend == None:
verbose("Backend not configured. defaulting to paster")
backend = 'paster'
backend_command = BACKENDS[backend]
# Process non-server commands
if command == 'list':
print "ARGS: %s" % (args,)
for server, params in config['servers'].items():
print "server %s : %s " % (server, params)
return
if not args and 'logging_server' in cfgopt:
if command in ('restart'):
logger_command ('stop', cfgopt, processes)
if command in ('start', 'restart'):
logger_command ('start', cfgopt, processes)
for key, serverspec in sorted(config['servers'].items()):
if args and key not in args:
continue
print ING[command], key
cfgopt['services_enabled'] = ','.join([
l.strip() for l in serverspec.pop('services_enabled', '').split(',')])
cfgopt['services_disabled'] = ','.join([
l.strip() for l in serverspec.pop('services_disabled', '').split(',')])
cfgopt['url'] = serverspec.pop('url')
fullurl = urlparse (cfgopt['url'])
cfgopt['host'] = fullurl[1].split(':')[0]
cfgopt['port'] = str(fullurl.port)
cfgopt['logfile'] = os.path.join(config['log_dir'], LOG_TEMPL % cfgopt['port'])
cfgopt['pidfile'] = os.path.join(config['pid_dir'], PID_TEMPL % cfgopt['port'])
cfgopt['uwsgi'] = serverspec.pop('uwsgi', None)
cfg_file_name = "%s_%s.cfg" % (key, backend)
cfg_file = find_config_path(cfg_file_name)
if not cfg_file:
print ("Cannot find config file %s" % cfg_file_name)
print ("Please run bq-admin setup configuration")
return
if command in ('stop', 'restart'):
error = None
backend_command ('stop', options, cfgopt, processes, cfg_file)
for proc in processes:
try:
proc.wait()
except KeyboardInterrupt:
error = True
if error is not None and backend == 'paster':
#print 'Paste windows error while stopping process, re-running'
processes = paster_command('stop', options, cfgopt, processes, args)
for proc in processes:
proc.wait()
print 'Recovered Paste windows error and successfully stopped daemon'
processes = []
if command in ('start', 'restart'):
if os.path.exists(cfgopt['pidfile']):
try:
pid= int (open (cfgopt['pidfile']).read())
if psutil.pid_exists (pid):
if options.force:
print 'old pid file: %s exists! restarting...' % cfgopt['pidfile']
operation("stop", options, cfg_file, *args)
time.sleep(5)
else:
print "Can't start because of existing PID file"
sys.exit(2)
except ValueError:
pass
if os.path.exists(cfgopt['pidfile']):
os.remove (cfgopt['pidfile'])
processes = backend_command('start', options, cfgopt, processes, cfg_file)
if not args and 'logging_server' in cfgopt:
if command in ('stop'):
logger_command ('stop', cfgopt, processes)
if options.wait:
for proc in processes:
proc.wait()
except KeyboardInterrupt:
pass
def load_config(filename, section=None):
if not os.path.exists(filename):
filename = find_config_path(filename)
print "LOADING", filename
return ConfigFile(filename)