#!/usr/bin/env python import sys import argparse from lya import AttrDict if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('-f', '--file', action='store', default='/etc/goodrain/docker-compose.yaml', help='yaml file, default is %(default)s') parser.add_argument('-u', '--update', action='store_true', help="update yaml config") parser.add_argument('-d', '--delete', action='store', help="delete a service") parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), default=sys.stdin) args = parser.parse_args() yaml_file = args.file cfg = AttrDict.from_yaml(yaml_file) if args.update: cfg.update_yaml(args.infile) with open(yaml_file, 'w') as f: cfg.dump(f) elif args.delete: service = args.delete if 'services' in cfg: if service in cfg.services: cfg.services.pop(service) if not cfg.services: cfg.pop('services') with open(yaml_file, 'w') as f: cfg.dump(f)
def main(): import argparse parser = argparse.ArgumentParser( description='Collect and dispatch various metrics to destinations.') parser.add_argument('-t', '--destination', metavar='host[:port]', help='host[:port] (default port: 2003, can be overidden' ' via config file) of sink destination endpoint (e.g. carbon' ' linereceiver tcp port, by default).') parser.add_argument('-i', '--interval', type=int, metavar='seconds', help='Interval between collecting and sending the datapoints.') parser.add_argument('-e', '--collector-enable', action='append', metavar='collector', default=list(), help='Enable only the specified metric collectors,' ' can be specified multiple times.') parser.add_argument('-d', '--collector-disable', action='append', metavar='collector', default=list(), help='Explicitly disable specified metric collectors,' ' can be specified multiple times. Overrides --collector-enable.') parser.add_argument('-s', '--sink-enable', action='append', metavar='sink', default=list(), help='Enable only the specified datapoint sinks,' ' can be specified multiple times.') parser.add_argument('-x', '--sink-disable', action='append', metavar='sink', default=list(), help='Explicitly disable specified datapoint sinks,' ' can be specified multiple times. Overrides --sink-enable.') parser.add_argument('-p', '--processor-enable', action='append', metavar='processor', default=list(), help='Enable only the specified datapoint processors,' ' can be specified multiple times.') parser.add_argument('-z', '--processor-disable', action='append', metavar='processor', default=list(), help='Explicitly disable specified datapoint processors,' ' can be specified multiple times. Overrides --processor-enable.') parser.add_argument('-c', '--config', action='append', metavar='path', default=list(), help='Configuration files to process.' ' Can be specified more than once.' ' Values from the latter ones override values in the former.' ' Available CLI options override the values in any config.') parser.add_argument('-a', '--xattr-emulation', metavar='db-path', help='Emulate filesystem extended attributes (used in' ' some collectors like sysstat or cron_log), storing per-path' ' data in a simple shelve db.') parser.add_argument('-n', '--dry-run', action='store_true', help='Do not actually send data.') parser.add_argument('--debug', action='store_true', help='Verbose operation mode.') optz = parser.parse_args() # Read configuration files cfg = AttrDict.from_yaml('{}.yaml'.format( os.path.splitext(os.path.realpath(__file__))[0] )) for k in optz.config: cfg.update_yaml(k) # Logging import logging configure_logging( cfg.logging, logging.DEBUG if optz.debug else logging.WARNING ) if not cfg.logging.tracebacks: class NoTBLogger(logging.Logger): def exception(self, *argz, **kwz): self.error(*argz, **kwz) logging.setLoggerClass(NoTBLogger) log = logging.getLogger(__name__) # Fill "auto-detected" blanks in the configuration, CLI overrides try: if optz.destination: cfg.sinks._default.host = optz.destination cfg.sinks._default.host = cfg.sinks._default.host.rsplit(':', 1) if len(cfg.sinks._default.host) == 1: cfg.sinks._default.host =\ cfg.sinks._default.host[0], cfg.sinks._default.default_port else: cfg.sinks._default.host[1] = int(cfg.sinks._default.host[1]) except KeyError: pass if optz.interval: cfg.loop.interval = optz.interval if optz.dry_run: cfg.debug.dry_run = optz.dry_run if optz.xattr_emulation: cfg.core.xattr_emulation = optz.xattr_emulation # Fake "xattr" module, if requested if cfg.core.xattr_emulation: import shelve xattr_db = shelve.open(cfg.core.xattr_emulation, 'c') class xattr_path(object): def __init__(self, base): assert isinstance(base, str) self.base = base def key(self, k): return '{}\0{}'.format(self.base, k) def __setitem__(self, k, v): xattr_db[self.key(k)] = v def __getitem__(self, k): return xattr_db[self.key(k)] def __del__(self): xattr_db.sync() class xattr_module(object): xattr = xattr_path sys.modules['xattr'] = xattr_module # Override "enabled" collector/sink parameters, based on CLI ep_conf = dict() for ep, enabled, disabled in\ [ ('collectors', optz.collector_enable, optz.collector_disable), ('processors', optz.processor_enable, optz.processor_disable), ('sinks', optz.sink_enable, optz.sink_disable) ]: conf = cfg[ep] conf_base = conf.pop('_default') if 'debug' not in conf_base: conf_base['debug'] = cfg.debug ep_conf[ep] = conf_base, conf, OrderedDict(), enabled, disabled # Init global cfg for collectors/sinks' usage from graphite_metrics import collectors, sinks, loops collectors.cfg = sinks.cfg = loops.cfg = cfg # Init pluggable components import pkg_resources for ep_type in 'collector', 'processor', 'sink': ep_key = '{}s'.format(ep_type) # a bit of a hack conf_base, conf, objects, enabled, disabled = ep_conf[ep_key] ep_dict = dict( (ep.name, ep) for ep in pkg_resources.iter_entry_points('graphite_metrics.{}'.format(ep_key)) ) eps = OrderedDict( (name, (ep_dict.pop(name), subconf or AttrDict())) for name, subconf in conf.viewitems() if name in ep_dict ) eps.update( (name, (module, conf_base)) for name, module in ep_dict.viewitems() ) for ep_name, (ep_module, subconf) in eps.viewitems(): if ep_name[0] == '_': log.debug( 'Skipping {} enty point,' ' prefixed by underscore: {}'.format(ep_type, ep_name) ) subconf.rebase(conf_base) # fill in "_default" collector parameters if enabled: if ep_name in enabled: subconf['enabled'] = True else: subconf['enabled'] = False if disabled and ep_name in disabled: subconf['enabled'] = False if subconf.get('enabled', True): log.debug('Loading {}: {}'.format(ep_type, ep_name)) try: obj = getattr(ep_module.load(), ep_type)(subconf) except Exception as err: log.exception('Failed to load/init {} ({}): {}'.format(ep_type, ep_name, err)) subconf.enabled = False obj = None if subconf.get('enabled', True): objects[ep_name] = obj else: log.debug(( '{} {} (entry point: {})' ' was disabled after init' ).format(ep_type.title(), obj, ep_name)) if ep_type != 'processor' and not objects: log.fatal('No {}s were properly enabled/loaded, bailing out'.format(ep_type)) sys.exit(1) log.debug('{}: {}'.format(ep_key.title(), objects)) loop = dict( (ep.name, ep) for ep in pkg_resources.iter_entry_points('graphite_metrics.loops') ) conf = AttrDict(**cfg.loop) if 'debug' not in conf: conf.debug = cfg.debug loop = loop[cfg.loop.name].load().loop(conf) collectors, processors, sinks = it.imap( op.itemgetter(2), op.itemgetter('collectors', 'processors', 'sinks')(ep_conf) ) log.debug( 'Starting main loop: {} ({} collectors, {} processors, {} sinks)'\ .format(loop, len(collectors), len(processors), len(sinks)) ) loop.start(collectors, processors, sinks)
def main(): import argparse parser = argparse.ArgumentParser( description='Check integrity of mirrored files.') parser.add_argument('-c', '--config', action='append', metavar='path', default=list(), help='Configuration files to process.' ' Can be specified more than once.' ' Values from the latter ones override values in the former.' ' Available CLI options override the values in any config.') parser.add_argument('-m', '--mtime-after', type=int, metavar='unix_time', help='Only act on files with' ' mtime larger than the given value (unix timestamp).') parser.add_argument('-l', '--list', nargs='?', metavar='types', default=False, help='Instead of usual action, just dump current state of all the files and exit.' 'Can take an optional "type" (of mirrors wrt file) argument (comma-separated type(s)):' ' consistent, inconsistent, unavailable, undergoal (default: inconsistent, unavailable).') parser.add_argument('-n', '--skip-nx', action='store_true', help='Do not retry known-404 mirrors.') parser.add_argument('--debug', action='store_true', help='Verbose operation mode.') optz = parser.parse_args() ## Read configuration files from lya import AttrDict cfg = AttrDict.from_yaml('{}.yaml'.format( os.path.splitext(os.path.realpath(__file__))[0] )) for k in optz.config: cfg.update_yaml(k) ## Logging import logging logging.basicConfig( level=logging.WARNING if not optz.debug else logging.DEBUG ) global log log = logging.getLogger() ## Modules modules_manifest_db = dict(dbm=ManifestDBM) modules_remote = dict( (k, ft.partial(v, conf=cfg.checks.get(k.split('__', 1)[0], dict()))) for k,v in dict( gentoo_portage=check_portage, rsync=check_rsync, rsync__batched=check_rsync_batched, mirrors=check_mirror ).viewitems() ) ## Catch-up with local fs check_fs_ts = time() manifest_db = modules_manifest_db[cfg.manifest.type](cfg.manifest) log.debug('Updating manifest-db with hashes of local files') for path in cfg.local: check_fs( path, manifest_db, hashes=cfg.manifest.hashes, local_changes_warn=cfg.goal.warn.local_changes, ts=check_fs_ts, ts_min=optz.mtime_after ) log.debug('Manifest-db cleanup') ts_gc_min = check_fs_ts - cfg.goal.gc_timeout * 24 * 3600 if optz.mtime_after and optz.mtime_after < ts_gc_min: check_gc(manifest_db, ts_min=ts_gc_min) ## List of remotes in order of preference remotes = list() for rtype, urls in cfg.remote.viewitems(): for url in urls or list(): remotes.append((rtype, url)) ## Build a set of excluded filename-patterns exclude = set(cfg.exclude.patterns or set()) for src in cfg.exclude.from_files or list(): with open(src, 'rb') as src: for pat in it.ifilter( None, it.imap(op.methodcaller('strip'), src) ): exclude.add(pat) ## Build a large-enough list of stuff to be checked qratio = int(len(remotes) * cfg.goal.query.ratio) qmin = min(len(remotes), cfg.goal.query.hard_min or qratio) qmax = min(len(remotes), max(qmin, cfg.goal.query.hard_max or qratio)) limit = (cfg.goal.limit.files or None) if not optz.list else None undergoal = manifest_db.undergoal_order( qmin, qmax, limit=limit, remotes=remotes, ts_min=optz.mtime_after ) # Check that each listed path actually exists now and isn't conf-excluded drop = set() for path in undergoal: for pat in exclude: if fnmatch(pat, basename(path)): drop.add(path) continue try: meta = manifest_db[path] if meta['ts'] != check_fs_ts and not os.path.exists(path): raise KeyError(path) except KeyError: log.debug('Skipping check for unlisted/nx path: %s', path) drop.add(path) undergoal = list(path for path in undergoal if path not in drop) ## Just dump the list, if requested if optz.list is not False: log.debug('Just listing all the remotes') if not optz.list: optz.list = 'inconsistent, unavailable' mtypes = sorted(it.imap(op.methodcaller('strip'), optz.list.split(','))) mtypes_err = set(mtypes).difference([ 'consistent', 'inconsistent', 'unavailable', 'undergoal' ]) if mtypes_err: parser.error('Unrecognized check states: {}'.format(', '.join(mtypes_err))) if 'undergoal' in mtypes: undergoal_check = True mtypes = sorted( set(mtypes).difference(['undergoal'])\ .union(['consistent', 'inconsistent', 'unavailable']) ) else: undergoal_check = False for path in undergoal: meta_remotes = manifest_db[path].get('remotes', dict()) if undergoal_check and not ( len(set(meta_remotes.get( 'consistent', set() )).intersection(remotes)) < qmin\ or set(meta_remotes.get( 'inconsistent', set() )).intersection(remotes) ): continue if not meta_remotes: print('Path: {}\n No consistency data available'.format(path)) continue path_line = False for mtype in mtypes: mtype_remotes = set(meta_remotes.get(mtype, set())).intersection(remotes) if not mtype_remotes: continue if not path_line: print('Path: {}'.format(path)) path_line = True print(' {}{}'.format(mtype, ' ({}{}):\n {}'.format( len(mtype_remotes), '' if mtype != 'consistent' else ', min/max: {}/{}'.format(qmin, qmax), '\n '.join(it.starmap('({}) {}'.format, sorted(mtype_remotes))) ))) sys.exit() ## Check against remotes log.debug('Checking manifest-db against remotes') check_remotes( undergoal, remotes, manifest_db, modules_remote, thresh_err=min( cfg.goal.warn.inconsistency.max, int(len(remotes) * cfg.goal.warn.inconsistency.ratio) ), thresh_nx=cfg.goal.warn.unavailable, thresh_bug=cfg.goal.limit.errors, skip_nx=optz.skip_nx, warn_skip=cfg.goal.warn.skipped_remote ) log.debug('Done')
def __init__(self): self._settings = AttrDict.from_yaml(settings_default.as_posix()) if settings_local.exists(): self._settings.update_yaml(settings_local.as_posix())
def __init__(self, master=None): """Creates application main window with sizes self.WIDTH and self.HEIGHT. :param master: instance - Tkinter.Tk instance """ super(Application, self).__init__(master) self.master.title('Engine Game') self.master.geometry('{}x{}'.format(self.WIDTH, self.HEIGHT)) self.master.protocol('WM_DELETE_WINDOW', self.exit) self.source = None self._map = None self.points = None self.lines = None self.captured_point = None self.x0 = None self.y0 = None self.scale_x = None self.scale_y = None self.font_size = None self.coordinates = {} self.captured_lines = {} self.canvas_obj = AttrDict() self.icons = { 0: PhotoImage(file=join('icons', 'player_city.png')), 1: PhotoImage(file=join('icons', 'city.png')), 2: PhotoImage(file=join('icons', 'market.png')), 3: PhotoImage(file=join('icons', 'store.png')), 4: PhotoImage(file=join('icons', 'point.png')), 5: PhotoImage(file=join('icons', 'player_train.png')), 6: PhotoImage(file=join('icons', 'train.png')), 7: PhotoImage(file=join('icons', 'crashed_train.png')), 8: PhotoImage(file=join('icons', 'collision.png')), 9: PhotoImage(file=join('icons', 'play.png')), 10: PhotoImage(file=join('icons', 'play_pressed.png')), 11: PhotoImage(file=join('icons', 'stop.png')), 12: PhotoImage(file=join('icons', 'stop_pressed.png')) } self.queue_requests = { 0: self.set_status_bar, 1: self.set_player_idx, 2: self.build_map, 3: self.refresh_map, 4: self.set_available_games, 99: self.bot_control } self.settings_window = None if exists(expanduser(self.DEFAULTS)): with open(expanduser(self.DEFAULTS), 'r') as cfg: defaults = DefaultsDict.from_yaml(cfg) self.host = None if not defaults.host else str(defaults.host) self.port = None if not defaults.port else int(defaults.port) self.timeout = None if not defaults.timeout else int(defaults.timeout) self.username = None if not defaults.username else str(defaults.username) self.password = None if not defaults.password else str(defaults.password) else: self.host, self.port, self.timeout, self.username, self.password = None, None, None, None, None self.player_idx = None self.posts = {} self.trains = {} self.select_game_window = False self.available_games = None self.game = None self.num_players = None self.num_turns = None self.bot = Bot() self.bot_thread = None self.menu = Menu(self) filemenu = Menu(self.menu) filemenu.add_command(label='Open file', command=self.file_open) filemenu.add_command(label='Server settings', command=self.open_server_settings) filemenu.add_command(label='Select game', command=self.select_game) filemenu.add_command(label='Exit', command=self.exit) self.menu.add_cascade(label='Menu', menu=filemenu) master.config(menu=self.menu) self._status_bar = StringVar() self.label = Label(master, textvariable=self._status_bar) self.label.pack() self.frame = Frame(self) self.frame.bind('<Configure>', self._resize_frame) self.canvas = Canvas(self.frame, bg=self.BG, scrollregion=(0, 0, self.winfo_width(), self.winfo_height())) self.canvas.bind('<Button-1>', self._capture_point) self.canvas.bind('<Motion>', self._move_point) self.canvas.bind('<B1-ButtonRelease>', self._release_point) self.canvas.bind('<Configure>', self._resize_canvas) hbar = Scrollbar(self.frame, orient=HORIZONTAL) hbar.pack(side=BOTTOM, fill=X) hbar.config(command=self.canvas.xview) vbar = Scrollbar(self.frame, orient=VERTICAL) vbar.pack(side=RIGHT, fill=Y) vbar.config(command=self.canvas.yview) self.canvas.config(xscrollcommand=hbar.set, yscrollcommand=vbar.set) self.canvas.pack(fill=BOTH, expand=True) self.play = Label(self.canvas, bg='white') self.play.configure(image=self.icons[9]) self.play.bind('<Button-1>', self._play_press) self.play.bind('<B1-ButtonRelease>', self._play_release) self.stop = Label(self.canvas, bg='white') self.stop.configure(image=self.icons[11]) self.stop.bind('<Button-1>', self._stop_press) self.stop.bind('<B1-ButtonRelease>', self._stop_release) self.frame.pack(fill=BOTH, expand=True) self.weighted = IntVar(value=1) self.weighted_check = Checkbutton(self, text='Proportionally to length', variable=self.weighted, command=self._proportionally) self.weighted_check.pack(side=LEFT) self.show_weight = IntVar() self.show_weight_check = Checkbutton(self, text='Show length', variable=self.show_weight, command=self.show_weights) self.show_weight_check.pack(side=LEFT) self.pack(fill=BOTH, expand=True) self.requests_executor() self.get_available_games() self.set_status_bar('Click Play to start the game') self.play.place(rely=0.5, relx=0.5, anchor=CENTER)