def read_config(self, **kw): """Initial state of a runner. The module-runtime.cfg is read and the relevent runner section is applied. The environment list is created and setup in order to construct the environments next """ self.config = AttrDict(executable="", environments="") self.sections = {} self.debug("BaseRunner: read_config") # Load any bisque related variable into this runner runtime_bisque_cfg = find_config_path('runtime-bisque.cfg') if os.path.exists(runtime_bisque_cfg): self.bisque_cfg = ConfigFile(runtime_bisque_cfg) self.load_section(None, self.bisque_cfg) self.config['files'] = self.config.setdefault( 'files', '') + ',' + runtime_bisque_cfg else: self.info("BaseRunner: missing runtime-bisque.cfg") module_dir = kw.get('module_dir', os.getcwd()) runtime_module_cfg = os.path.join(module_dir, 'runtime-module.cfg') if not os.path.exists(runtime_module_cfg): self.info("BaseRunner: missing %s", runtime_module_cfg) return self.module_cfg = ConfigFile(runtime_module_cfg) # Process Command section self.load_section(None, self.module_cfg) # Globals
def choose_runner(self, **kw): curdir = os.getcwd() module_dir = kw.get('module_dir', curdir) log.debug('Choosing runner : cwd: %s module %s', curdir, module_dir) # Choose a runner runtime_module_cfg = os.path.join(module_dir, 'runtime-module.cfg') if os.path.exists(runtime_module_cfg): cfg = ConfigFile(runtime_module_cfg) launcher = cfg.get(None, 'launcher') if launcher and os.path.abspath( sys.argv[0]) != os.path.abspath(launcher): if os.path.exists(launcher): # Given launcher cmd = launcher.split(' ') # pylint: disable=no-member cmd.extend(sys.argv[1:]) log.debug("Calling %s" % cmd) subprocess.call(cmd) return None log.warn("Missing launcher: %s", launcher) return None else: log.error("Missing %s", runtime_module_cfg) return None runners = cfg.get(None, 'runtime.platforms') if runners is None: raise ModuleEnvironmentError( "Must define legal platforms: runtime.platforms in module config" ) log.debug('Module runners: %s', runners) self.module_runners = [r.strip() for r in runners.split(',')] # pylint: disable=no-member runtime_bisque = 'runtime-bisque.cfg' if not os.path.exists(runtime_bisque): runtime_bisque = find_config_path(runtime_bisque) if os.path.exists(runtime_bisque): cfg = ConfigFile(runtime_bisque) runners = cfg.get(None, 'runtime.platforms') log.debug('System runners: %s', runners) self.system_runners = [r.strip() for r in runners.split(',')] # pylint: disable=no-member else: log.warn("Could not file %s", runtime_bisque) # Default to module runners ? They may not be availble self.system_runners = self.module_runners # Determine best platform for module by comparing system platforms # and module platforms. The platforms are listed in order # of preference based on the system preferences for run_platform in self.system_runners: if run_platform in self.module_runners: log.debug("Choosing Runtime Platform: %s", run_platform) return RUNNER_MAP[run_platform] return None
def read_config(filename, section=None): if not os.path.exists(filename): filename = find_config_path(filename) print "READING", filename conf = ConfigFile(filename).get(section, asdict=True) config_dir = os.path.dirname(filename) conf['bisque.path.config'] = config_dir print "CONFIG", config_dir return conf
def python_setup(scripts, package_scripts=True, dependency_dir='pydist', params=None): """compile python dependencies into a package using pyinstaller if package_script is true then a runner scripts will be generated for each script """ cmd = [ 'pyinstaller', '--clean', '--noconfirm', '--distpath', dependency_dir ] #fr = bbfreeze.Freezer(dependency_dir) #fr.include_py = False if not isinstance(scripts, list): scripts = [scripts] # check if python code is newer than dependency_dir (pydist) #if not any (needs_update(dependency_dir, x) for x in scripts): # print "Skipping python compilation %s is newer than %s" % (dependency_dir, scripts) # return 0 # if we need to compile, then compile cleanly if os.path.exists(dependency_dir): shutil.rmtree(dependency_dir) data = dict(params or {}) for script in scripts: cmd.append(script) check_call(cmd) # read default path from site.cfg (TG config not initialized yet when this runs) #site_config = read_config('site.cfg', "app:main") #filename = os.path.abspath(os.path.join(site_config['bisque.paths.default'], 'templates/python_launcher.tmpl').replace('\\', '/')) filename = find_config_path('templates/python_launcher.tmpl', config_dir='config-defaults').replace( '\\', '/') print "TEMPLATE", filename for script in scripts: script_name = os.path.splitext(script)[0] data['script'] = os.path.join('.', dependency_dir, script_name, script_name) # THIS os.path.join needs to be replaced by pkg_resources or pkg_util # when the toplevel is packaged try: template = Template(filename=filename) with open(script_name, 'wb') as f: f.write(template.render(script=data['script'])) os.chmod(script_name, 0744) return 0 except Exception, e: print("Could not create python launcher script %s" % e)
def upgrade(): "Find resource references in string and migrate to new resource_uniq id's" print("Sending migration info to migratelog.txt") logfile = open('migratelog.txt', 'wb') parser = ConfigParser() parser.read(find_config_path("site.cfg")) bisque_root = parser.get('servers', 'h1.url') print("Using new bisque root %s" % bisque_root) print("ROOT", bisque_root, file=logfile) root = urlparse.urlsplit(bisque_root) cntxt = context.get_context() SessionMaker = sessionmaker(bind=cntxt.bind) # toplevel = DBSession.query(Taggable).filter(Taggable.resource_parent_id == None, Taggable.resource_uniq == None) # for resource in toplevel: # uid = make_short_uuid() # #resource.resource_uniq = "00-%s" % uid # print resource.resource_type, uid # resource.resource_uniq = "00-%s" % uid Base = declarative_base() metadata = MetaData(bind=cntxt.bind) class Taggable(Base): __table__ = Table('taggable', metadata, autoload=True) def __str__(self): return u"<%s id=%s name=%s value=%s ts=%s document=%s>" % ( self.resource_type, self.id, self.resource_name, self.resource_value, self.ts, self.document_id) DBSession = SessionMaker() Tag = Taggable alltags = DBSession.query(Tag).filter( and_(Tag.resource_type == 'tag', Tag.resource_value.like('http%'))) for tag in alltags: url = tag.resource_value parts = list(urlparse.urlsplit(url)) # (scheme, host, path, ...) path = parts[2].split('/') ident = None try: while path: x = path.pop(0) if x and x[0].isdigit(): ident = int(x) break except (ValueError, IndexError): print("badvalue %s %s" % (x, url), file=logfile) continue if ident is None: print("badvalue %s" % tag, file=logfile) continue target = DBSession.query(Taggable).filter_by(id=ident).first() if target: if target.resource_uniq is None: print("badpointer %s points to uniq=None %s" % (tag, target), file=logfile) continue #print "mapping %s -> %s /data_service/%s/%s" % (ident, target, target.resource_uniq, "/".join(path)) parts[0] = root[0] parts[1] = root[1] parts[2] = posixpath.join("/data_service", target.resource_uniq, "/".join(path)).rstrip('/') newurl = urlparse.urlunsplit(parts) print("mapping %s -> %s" % (tag, newurl), file=logfile) tag.resource_value = newurl else: print("unknown mapping %s" % tag, file=logfile)
def operation(command, options, *args): """Run a multi-server command to start several bisque jobs """ def verbose(msg): if options.verbose: print msg cfg_file=options.site or SITE_CFG site_cfg = find_config_path(cfg_file) if site_cfg is None: print "Cannot find %s.. please make sure you are in the bisque dir" % cfg_file return site_dir = os.path.dirname(os.path.abspath(site_cfg)) verbose('using config : %s' % site_cfg) try: config = readhostconfig(site_cfg) verbose("SERVERS %s" % (config['servers'].keys())) processes = [] cfgopt = {} # {'root': config['root']} cfgopt['site_dir'] = site_dir cfgopt['site_cfg'] = site_cfg cfgopt['virtualenv'] = os.getenv('VIRTUAL_ENV') for f in ('log_dir', 'pid_dir', 'logging_server'): if f in config: cfgopt[f] = config[f] if not os.path.exists (config['run_dir']): print "Please check your site.cfg. bisque.paths.run is not set correctly" return os.chdir (config['run_dir']) # Get the backend backend = config.get('backend', None) verbose("using backend: " + str(backend)) if backend == None: verbose("Backend not configured. defaulting to paster") backend = 'paster' backend_command = BACKENDS[backend] # Process non-server commands if command == 'list': print "ARGS: %s" % (args,) for server, params in config['servers'].items(): print "server %s : %s " % (server, params) return if not args and 'logging_server' in cfgopt: if command in ('restart'): logger_command ('stop', cfgopt, processes) if command in ('start', 'restart'): logger_command ('start', cfgopt, processes) for key, serverspec in sorted(config['servers'].items()): if args and key not in args: continue print ING[command], key cfgopt['services_enabled'] = ','.join([ l.strip() for l in serverspec.pop('services_enabled', '').split(',')]) cfgopt['services_disabled'] = ','.join([ l.strip() for l in serverspec.pop('services_disabled', '').split(',')]) cfgopt['url'] = serverspec.pop('url') fullurl = urlparse (cfgopt['url']) cfgopt['host'] = fullurl[1].split(':')[0] cfgopt['port'] = str(fullurl.port) cfgopt['logfile'] = os.path.join(config['log_dir'], LOG_TEMPL % cfgopt['port']) cfgopt['pidfile'] = os.path.join(config['pid_dir'], PID_TEMPL % cfgopt['port']) cfgopt['uwsgi'] = serverspec.pop('uwsgi', None) cfg_file_name = "%s_%s.cfg" % (key, backend) cfg_file = find_config_path(cfg_file_name) if not cfg_file: print ("Cannot find config file %s" % cfg_file_name) print ("Please run bq-admin setup configuration") return if command in ('stop', 'restart'): error = None backend_command ('stop', options, cfgopt, processes, cfg_file) for proc in processes: try: proc.wait() except KeyboardInterrupt: error = True if error is not None and backend == 'paster': #print 'Paste windows error while stopping process, re-running' processes = paster_command('stop', options, cfgopt, processes, args) for proc in processes: proc.wait() print 'Recovered Paste windows error and successfully stopped daemon' processes = [] if command in ('start', 'restart'): if os.path.exists(cfgopt['pidfile']): try: pid= int (open (cfgopt['pidfile']).read()) if psutil.pid_exists (pid): if options.force: print 'old pid file: %s exists! restarting...' % cfgopt['pidfile'] operation("stop", options, cfg_file, *args) time.sleep(5) else: print "Can't start because of existing PID file" sys.exit(2) except ValueError: pass if os.path.exists(cfgopt['pidfile']): os.remove (cfgopt['pidfile']) processes = backend_command('start', options, cfgopt, processes, cfg_file) if not args and 'logging_server' in cfgopt: if command in ('stop'): logger_command ('stop', cfgopt, processes) if options.wait: for proc in processes: proc.wait() except KeyboardInterrupt: pass
def load_config(filename, section=None): if not os.path.exists(filename): filename = find_config_path(filename) print "LOADING", filename return ConfigFile(filename)