def test_yakonfig_default(): yakonfig.set_default_config([yakonfig]) try: c = yakonfig.get_global_config() assert 'yakonfig' in c finally: yakonfig.clear_global_config()
def as_child(cls, global_config, parent=None): '''Run a single job in a child process. This method never returns; it always calls :func:`sys.exit` with an error code that says what it did. ''' try: setproctitle('rejester worker') random.seed() # otherwise everyone inherits the same seed yakonfig.set_default_config([yakonfig, dblogger, rejester], config=global_config) worker = cls(yakonfig.get_global_config(rejester.config_name)) worker.register(parent=parent) did_work = worker.run(set_title=True) worker.unregister() if did_work: sys.exit(cls.EXIT_SUCCESS) else: sys.exit(cls.EXIT_BORED) except Exception, e: # There's some off chance we have logging. # You will be here if redis is down, for instance, # and the yakonfig dblogger setup runs but then # the get_work call fails with an exception. if len(logging.root.handlers) > 0: logger.critical('failed to do any work', exc_info=e) sys.exit(cls.EXIT_EXCEPTION)
def start_logging(gconfig, logpath): '''Turn on logging and set up the global config. This expects the :mod:`yakonfig` global configuration to be unset, and establishes it. It starts the log system via the :mod:`dblogger` setup. In addition to :mod:`dblogger`'s defaults, if `logpath` is provided, a :class:`logging.handlers.RotatingFileHandler` is set to write log messages to that file. This should not be called if the target worker is :class:`coordinate.workers.ForkWorker`, since that manages logging on its own. :param dict gconfig: the :mod:`yakonfig` global configuration :param str logpath: optional location to write a log file ''' yakonfig.set_default_config([coordinate, dblogger], config=gconfig) if logpath: formatter = dblogger.FixedWidthFormatter() # TODO: do we want byte-size RotatingFileHandler or # TimedRotatingFileHandler? handler = logging.handlers.RotatingFileHandler( logpath, maxBytes=10000000, backupCount=3) handler.setFormatter(formatter) logging.getLogger('').addHandler(handler)
def test_fill_in_kvp(): yakonfig.set_default_config([ConfigurableLike], { 'key': 'value' }) try: c = yakonfig.get_global_config() assert sorted(iterkeys(c)) == ['configurable'] cc = c['configurable'] assert sorted(iterkeys(cc)) == ['key', 'type'] assert cc['type'] == 'class' assert cc['key'] == 'value' finally: yakonfig.clear_global_config()
def test_default_file(request): yakonfig.set_default_config( [ConfigurableArgs()], filename=str(request.fspath.dirpath('argconfig.yaml'))) try: c = yakonfig.get_global_config() assert sorted(iterkeys(c)) == ['config'] cc = c['config'] assert sorted(iterkeys(cc)) == ['k'] assert cc['k'] == 'x' finally: yakonfig.clear_global_config()
def test_dont_validate(): yakonfig.set_default_config([ConfigurableArgs()], params={'key': 'longer than one char'}, validate=False) try: c = yakonfig.get_global_config() assert sorted(iterkeys(c)) == ['config'] cc = c['config'] assert sorted(iterkeys(cc)) == ['k'] assert cc['k'] == 'longer than one char' finally: yakonfig.clear_global_config()
def test_two_level(): yakonfig.set_default_config([ConfigurableTop]) try: c = yakonfig.get_global_config() assert sorted(iterkeys(c)) == ['top'] c = c['top'] assert sorted(iterkeys(c)) == ['aaa', 'bottom'] assert c['aaa'] == 'bbb' c = c['bottom'] assert sorted(iterkeys(c)) == ['zzz'] assert c['zzz'] == '-32768' finally: yakonfig.clear_global_config()
def test_two_level_args(): yakonfig.set_default_config([ConfigurableTop], { 'aaa': 'a', 'bbb': 'b', 'zzz': 'z' }) try: c = yakonfig.get_global_config() assert sorted(iterkeys(c)) == ['top'] c = c['top'] assert sorted(iterkeys(c)) == ['aaa', 'bottom'] assert c['aaa'] == 'a' c = c['bottom'] assert sorted(iterkeys(c)) == ['zzz'] assert c['zzz'] == 'z' finally: yakonfig.clear_global_config()
def test_prog_yaml(): the_yaml = """ configurable: key: yaml """ yakonfig.set_default_config([ConfigurableLike], yaml=the_yaml) try: c = yakonfig.get_global_config() assert sorted(iterkeys(c)) == ['configurable'] cc = c['configurable'] assert sorted(iterkeys(cc)) == ['key', 'type'] assert cc['type'] == 'class' assert cc['key'] == 'yaml' finally: yakonfig.clear_global_config()
def main(): p = argparse.ArgumentParser( description='SortingDesk report generation tool') p.add_argument('-c', '--config', required=True, help='dossier stack YAML config file') p.add_argument('-o', '--output', required=True, help='path to write Excel workbook file') p.add_argument('-u', '--user', default='unknown', help='user name (default=ALL)') p.add_argument('folder', help='folder name') p.add_argument('subfolder', nargs='?', default=None, help='subfolder name (default=ALL)') args = p.parse_args() config = yakonfig.set_default_config([kvlayer], filename=args.config) factory = Factory(config) store = factory.create(Store) # Instantiate and run report generator. folders = Folders(kvlayer.client()) gen = ReportGenerator(store, folders, args.folder, subfolder_name=args.subfolder, user=args.user) with open(args.output, 'wb+') as out: gen.run(out)
def log_spewer(self, gconfig, fd): '''Child process to manage logging. This reads pairs of lines from `fd`, which are alternating priority (Python integer) and message (unformatted string). ''' setproctitle('rejester fork_worker log task') yakonfig.set_default_config([yakonfig, dblogger], config=gconfig) try: while True: prefix = os.read(fd, struct.calcsize('ii')) level, msglen = struct.unpack('ii', prefix) msg = os.read(fd, msglen) logger.log(level, msg) except Exception, e: logger.critical('log writer failed', exc_info=e) raise
def test_prog_yaml(): # like from the command line, values via params overrides explicit yaml the_yaml = """ configurable: key: yaml """ yakonfig.set_default_config([ConfigurableLike], params={'key': 'value'}, yaml=the_yaml) try: c = yakonfig.get_global_config() assert sorted(iterkeys(c)) == ['configurable'] cc = c['configurable'] assert sorted(iterkeys(cc)) == ['key', 'type'] assert cc['type'] == 'class' assert cc['key'] == 'value' finally: yakonfig.clear_global_config()
def go(gconfig, args): '''Actually run the worker. This does some required housekeeping, like setting up logging for :class:`~coordinate.workers.MultiWorker` and establishing the global :mod:`yakonfig` configuration. This expects to be called with the :mod:`yakonfig` configuration unset. :param dict gconfig: the :mod:`yakonfig` global configuration :param args: command-line arguments ''' rconfig = gconfig['coordinate'] which_worker = rconfig.get('worker', 'fork_worker') if which_worker == 'fork_worker': yakonfig.set_default_config([coordinate], config=gconfig) else: start_logging(gconfig, args.logpath) return start_worker(which_worker, rconfig)
def main(): p = argparse.ArgumentParser( description='SortingDesk report generation tool') p.add_argument('-c', '--config', required=True, help='Dossier stack YAML config file') p.add_argument('folder_name', nargs='?', default=None, help='Folder name') p.add_argument('-u', '--user', default=None,help='user name (default=ALL)') args = p.parse_args() config = yakonfig.set_default_config([kvlayer], filename=args.config) factory = Factory(config) store = factory.create(Store) label_store = factory.create(LabelStore) folders = Folders(store, label_store) list_projects(folders, args.folder_name, args.user)
def main(): p = argparse.ArgumentParser( description='SortingDesk report generation tool') p.add_argument('-c', '--config', required=True, help='Dossier stack YAML config file') p.add_argument('folder_name', nargs='?', default=None, help='Folder name') p.add_argument('-u', '--user', default=None, help='user name (default=ALL)') args = p.parse_args() config = yakonfig.set_default_config([kvlayer], filename=args.config) factory = Factory(config) store = factory.create(Store) label_store = factory.create(LabelStore) folders = Folders(store, label_store) list_projects(folders, args.folder_name, args.user)
def main(): p = argparse.ArgumentParser( description='SortingDesk report generation tool') p.add_argument('-c', '--config', required=True, help='dossier stack YAML config file') p.add_argument('-o', '--output', required=True, help='path to write Excel workbook file') p.add_argument( '-u', '--user', default='unknown', help='user name (default=ALL)') p.add_argument('folder', help='folder name') p.add_argument('subfolder', nargs='?', default=None, help='subfolder name (default=ALL)') args = p.parse_args() config = yakonfig.set_default_config([kvlayer], filename=args.config) factory = Factory(config) store = factory.create(Store) # Instantiate and run report generator. folders = Folders(kvlayer.client()) gen = ReportGenerator(store, folders, args.folder, subfolder_name=args.subfolder, user=args.user) with open(args.output, 'wb+') as out: gen.run(out)
def global_yakonfig(a_configurable): yakonfig.set_default_config([a_configurable]) yield yakonfig.get_global_config() yakonfig.clear_global_config()