def setup(): LogOptions.set_stderr_log_level('NONE') app.add_option('--iface', default='eth0', type=str, help='The interface to sniff on') app.add_option('--port', default=2889, type=int, help='The ZAB port used by the leader') app.add_option('-c', '--colors', default=False, action='store_true', help='Color each learner/leader stream differently') app.add_option('--dump-bad-packet', default=False, action='store_true', help='Dump packets that cannot be deserialized') app.add_option( '--include-pings', default=False, action='store_true', help='Whether to include pings send from learners to the leader') app.add_option('--version', default=False, action='store_true')
def setup(): LogOptions.set_stderr_log_level('NONE') app.add_option('--iface', default='eth0', type=str) app.add_option('--port', default=3888, type=int) app.add_option('-c', '--colors', default=False, action='store_true') app.add_option('--dump-bad-packet', default=False, action='store_true') app.add_option('--version', default=False, action='store_true')
def setup(): LogOptions.set_stderr_log_level('NONE') app.add_option('--iface', default='eth0', type=str) app.add_option('--port', default=3888, type=int) app.add_option('-c', '--colors', default=False, action='store_true') app.add_option('--dump-bad-packet', default=False, action='store_true') app.add_option('--version', default=False, action='store_true')
def run(self, lock): if self.options.dry_run: print "****** Dry Run ******" logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or "info").upper()) logdir = self.options.logdir or self.config.get("goals", "logdir", default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init("goals") else: init() logger = log if self.options.recursive_directory: log.warn("--all-recursive is deprecated, use a target spec with the form [dir]:: instead") for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn("--all is deprecated, use a target spec with the form [dir]: instead") for dir in self.options.target_directory: self.add_target_directory(dir) context = Context( self.config, self.options, self.targets, requested_goals=self.requested_goals, lock=lock, log=logger, timer=self.timer if self.options.time else None, ) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print ("Unknown goal(s): %s" % " ".join(phase.name for phase in unknown)) print ("") return Phase.execute(context, "goals") if logger: logger.debug("Operating on targets: %s", self.targets) ret = Phase.attempt(context, self.phases) if self.options.time: print ("Timing report") print ("=============") self.timer.print_timings() return ret
def run(self, lock): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # Enable standard python logging for code with no handle to a context/work-unit. if self.options.log_level: LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) log.init('goals') else: log.init() # Update the reporting settings, now that we have flags etc. def is_console_task(): for phase in self.phases: for goal in phase.goals(): if issubclass(goal.task_type, ConsoleTask): return True return False is_explain = self.options.explain update_reporting(self.options, is_console_task() or is_explain, self.run_tracker) if self.options.dry_run: print('****** Dry Run ******') context = Context( self.config, self.options, self.run_tracker, self.targets, requested_goals=self.requested_goals, lock=lock) if self.options.recursive_directory: context.log.warn( '--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: context.log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: _list_goals(context, 'Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) return 1 return Goal._execute(context, self.phases, print_timing=self.options.time)
def run(self, lock): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # Enable standard python logging for code with no handle to a context/work-unit. if self.options.log_level: LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) log.init('goals') else: log.init() # Update the reporting settings, now that we have flags etc. def is_console_task(): for phase in self.phases: for goal in phase.goals(): if issubclass(goal.task_type, ConsoleTask): return True return False is_explain = self.options.explain update_reporting(self.options, is_console_task() or is_explain, self.run_tracker) if self.options.dry_run: print('****** Dry Run ******') context = Context( self.config, self.options, self.run_tracker, self.targets, requested_goals=self.requested_goals, lock=lock) if self.options.recursive_directory: context.log.warn( '--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: context.log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: _list_goals(context, 'Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) return 1 return Goal._execute(context, self.phases, print_timing=self.options.time)
def run(self, lock): with self.check_errors("Target contains a dependency cycle") as error: for target in self.targets: try: InternalTarget.check_cycles(target) except InternalTarget.CycleException as e: error(target.id) timer = None if self.options.time: class Timer(object): def now(self): return time.time() def log(self, message): print(message) timer = Timer() logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) context = Context(self.config, self.options, self.targets, lock=lock, log=logger) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) return Phase.attempt(context, self.phases, timer=timer)
def execute(self): def add_targets(dir, buildfile): try: self.targets.extend(Target.get(addr) for addr in Target.get_all_addresses(buildfile)) except (TypeError, ImportError): error(dir, include_traceback=True) except (IOError, SyntaxError): error(dir) if self.options.recursive_directory: with self.check_errors('There was a problem scanning the ' 'following directories for targets:') as error: for dir in self.options.recursive_directory: for buildfile in BuildFile.scan_buildfiles(self.root_dir, dir): add_targets(dir, buildfile) if self.options.target_directory: with self.check_errors("There was a problem loading targets " "from the following directory's BUILD files") as error: for dir in self.options.target_directory: add_targets(dir, BuildFile(self.root_dir, dir)) timer = None if self.options.time: class Timer(object): def now(self): return time.time() def log(self, message): print(message) timer = Timer() logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.config.get('goals', 'logdir') if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') logger = log context = Context(self.config, self.options, self.targets, log=logger) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print() return Phase.execute(context, 'goals') return Phase.attempt(context, self.phases, timer=timer)
def run(self, lock): with self.check_errors("Target contains a dependency cycle") as error: with self.timer.timing("parse:check_cycles"): for target in self.targets: try: InternalTarget.check_cycles(target) except InternalTarget.CycleException as e: error(target.id) logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or "info").upper()) logdir = self.options.logdir or self.config.get("goals", "logdir", default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init("goals") else: init() logger = log if self.options.recursive_directory: log.warn("--all-recursive is deprecated, use a target spec with the form [dir]:: instead") for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn("--all is deprecated, use a target spec with the form [dir]: instead") for dir in self.options.target_directory: self.add_target_directory(dir) context = Context(self.config, self.options, self.targets, lock=lock, log=logger) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print("Unknown goal(s): %s" % " ".join(phase.name for phase in unknown)) print("") return Phase.execute(context, "goals") if logger: logger.debug("Operating on targets: %s", self.targets) ret = Phase.attempt(context, self.phases, timer=self.timer if self.options.time else None) if self.options.time: print("Timing report") print("=============") self.timer.print_timings() return ret
def run(self, lock): if self.options.dry_run: print '****** Dry Run ******' logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) context = Context( self.config, self.options, self.targets, lock=lock, log=logger, timer=self.timer if self.options.time else None) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) ret = Phase.attempt(context, self.phases) if self.options.time: print('Timing report') print('=============') self.timer.print_timings() return ret
def run(self, lock): with self.check_errors("Target contains a dependency cycle") as error: with self.timer.timing('parse:check_cycles'): for target in self.targets: try: InternalTarget.check_cycles(target) except InternalTarget.CycleException as e: error(target.id) logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead') for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn('--all is deprecated, use a target spec with the form [dir]: instead') for dir in self.options.target_directory: self.add_target_directory(dir) context = Context(self.config, self.options, self.targets, lock=lock, log=logger) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) ret = Phase.attempt(context, self.phases, timer=self.timer if self.options.time else None) if self.options.time: print('Timing report') print('=============') self.timer.print_timings() return ret
def setup(): LogOptions.set_stderr_log_level('NONE') app.add_option('--packet-filter', default='tcp', type=str, help='pcap filter string. e.g. "tcp portrange 11221-32767" for JUnit tests') app.add_option('-c', '--colors', default=False, action='store_true') app.add_option('--dump-bad-packet', default=False, action='store_true') app.add_option('--include-pings', default=False, action='store_true', help='Whether to include ZAB/ZK pings') app.add_option('--offline', default=None, type=str, help='offline mode with a pcap file') app.add_option('--version', default=False, action='store_true')
def setup(): LogOptions.set_stderr_log_level('NONE') app.add_option('--iface', default='eth0', type=str, help='The interface to sniff on') app.add_option('--port', default=2889, type=int, help='The ZAB port used by the leader') app.add_option('-c', '--colors', default=False, action='store_true', help='Color each learner/leader stream differently') app.add_option('--dump-bad-packet', default=False, action='store_true', help='Dump packets that cannot be deserialized') app.add_option('--include-pings', default=False, action='store_true', help='Whether to include pings send from learners to the leader') app.add_option('--version', default=False, action='store_true')
def run(self, lock): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # Enable standard python logging for code with no handle to a context/work-unit. if self.options.log_level: LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get( 'goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) log.init('goals') else: log.init() # Update the reporting settings, now that we have flags etc. def is_console_task(): for phase in self.phases: for goal in phase.goals(): if issubclass(goal.task_type, ConsoleTask): return True return False is_explain = self.options.explain update_reporting(self.options, is_console_task() or is_explain, self.run_tracker) context = Context(self.config, self.options, self.run_tracker, self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, lock=lock) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: context.log.error('Unknown goal(s): %s\n' % ' '.join(phase.name for phase in unknown)) return 1 engine = GroupEngine() return engine.execute(context, self.phases)
def run(self, lock): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # Enable standard python logging for code with no handle to a context/work-unit. if self.options.log_level: LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) log.init('goals') else: log.init() # Update the reporting settings, now that we have flags etc. def is_console_task(): for phase in self.phases: for goal in phase.goals(): if issubclass(goal.task_type, ConsoleTask): return True return False is_explain = self.options.explain update_reporting(self.options, is_console_task() or is_explain, self.run_tracker) context = Context( self.config, self.options, self.run_tracker, self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, lock=lock) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: context.log.error('Unknown goal(s): %s\n' % ' '.join(phase.name for phase in unknown)) return 1 engine = GroupEngine() return engine.execute(context, self.phases)
def setup(): LogOptions.set_stderr_log_level('NONE') app.add_option('--iface', default='eth0', type=str, metavar='<iface>', help='The interface to sniff on') app.add_option('--client-port', default=0, type=int, metavar='<client_port>', help='The client port to filter by') app.add_option('--zookeeper-port', default=2181, type=int, metavar='<server_port>', help='The ZooKeeper server port to filter by') app.add_option('--max-queued-requests', default=10000, type=int, metavar='<max>', help='The maximum number of requests queued to be deserialized') app.add_option('--exclude-host', dest='excluded_hosts', metavar='<host>', default=[], action='append', help='Host that should be excluded (you can use this multiple times)') app.add_option('--include-host', dest='included_hosts', metavar='<host>', default=[], action='append', help='Host that should be included (you can use this multiple times)') app.add_option('--count-requests', default=0, type=int, metavar='<nreqs>', help='Count N requests and report a summary (default: group by path)') app.add_option('--measure-latency', default=0, type=int, metavar='<nreqs>', help='Measure latency of N pairs of requests and replies (default: group by path') app.add_option('--group-by', default='path', type=str, metavar='<group>', help='Used with --count-requests or --measure-latency. Possible values: path, type or client') app.add_option('--sort-by', default='avg', type=str, metavar='<sort>', help='Used with --measure-latency. Possible values: avg, p95 and p99') app.add_option("--aggregation-depth", default=0, type=int, metavar='<depth>', help="Aggregate paths up to a certain depth. Used with --count-requests or --measure-latency") app.add_option('--unpaired', default=False, action='store_true', help='Don\'t pair reqs/reps') app.add_option('-p', '--include-pings', default=False, action='store_true', help='Whether to include ping requests and replies') app.add_option('-c', '--colors', default=False, action='store_true', help='Color each client/server stream differently') app.add_option('--dump-bad-packet', default=False, action='store_true', help='If unable to to deserialize a packet, print it out') app.add_option('--version', default=False, action='store_true')
def setup(): LogOptions.set_stderr_log_level('NONE') app.add_option( '--packet-filter', default='tcp', type=str, help= 'pcap filter string. e.g. "tcp portrange 11221-32767" for JUnit tests') app.add_option('-c', '--colors', default=False, action='store_true') app.add_option('--dump-bad-packet', default=False, action='store_true') app.add_option('--include-pings', default=False, action='store_true', help='Whether to include ZAB/ZK pings') app.add_option('--offline', default=None, type=str, help='offline mode with a pcap file') app.add_option('--version', default=False, action='store_true')
# These are are side-effecting imports in that they register commands via # app.command. This is a poor code practice and should be fixed long-term # with the creation of twitter.common.cli that allows for argparse-style CLI # composition. from twitter.aurora.client.commands import ( core, help, run, ssh, ) from twitter.aurora.client.options import add_verbosity_options app.register_commands_from(core, run, ssh) app.register_commands_from(help) add_verbosity_options() def main(): app.help() LogOptions.set_stderr_log_level('INFO') LogOptions.disable_disk_logging() app.set_name('aurora-client') app.set_usage(generate_terse_usage()) def proxy_main(): app.main()
import zookeeper from twitter.common import log from twitter.common.log.options import LogOptions from twitter.common.zookeeper.client import ZooKeeper, ZooDefs from twitter.common.zookeeper.test_server import ZookeeperServer MAX_EVENT_WAIT_SECS = 30.0 MAX_EXPIRE_WAIT_SECS = 60.0 CONNECT_TIMEOUT_SECS = 10.0 CONNECT_RETRIES = 6 if os.getenv('ZOOKEEPER_TEST_DEBUG'): LogOptions.set_stderr_log_level('NONE') LogOptions.set_disk_log_level('DEBUG') LogOptions.set_log_dir('/tmp') log.init('client_test') def make_zk(server, **kw): return ZooKeeper('localhost:%d' % server.zookeeper_port, timeout_secs=CONNECT_TIMEOUT_SECS, max_reconnects=CONNECT_RETRIES, **kw) def test_client_connect(): with ZookeeperServer() as server: zk = make_zk(server)
def main(args, options): log.info("Options in use: %s", options) if not options.api_port: app.error('Must specify --port') if not options.mesos_master: app.error('Must specify --mesos_master') if not options.framework_user: app.error('Must specify --framework_user') if not options.executor_uri: app.error('Must specify --executor_uri') if not options.executor_cmd: app.error('Must specify --executor_cmd') if not options.zk_url: app.error('Must specify --zk_url') if not options.admin_keypath: app.error('Must specify --admin_keypath') if not options.scheduler_keypath: app.error('Must specify --scheduler_keypath') if options.verbose: LogOptions.set_stderr_log_level('google:DEBUG') try: election_timeout = parse_time(options.election_timeout) framework_failover_timeout = parse_time(options.framework_failover_timeout) except InvalidTime as e: app.error(e.message) try: _, zk_servers, zk_root = zookeeper.parse(options.zk_url) except Exception as e: app.error("Invalid --zk_url: %s" % e.message) web_assets_dir = os.path.join(options.work_dir, "web") pkgutil.unpack_assets(web_assets_dir, MYSOS_MODULE, ASSET_RELPATH) log.info("Extracted web assets into %s" % options.work_dir) fw_principal = None fw_secret = None if options.framework_authentication_file: try: with open(options.framework_authentication_file, "r") as f: cred = yaml.load(f) fw_principal = cred["principal"] fw_secret = cred["secret"] log.info("Loaded credential (principal=%s) for framework authentication" % fw_principal) except IOError as e: app.error("Unable to read the framework authentication key file: %s" % e) except (KeyError, yaml.YAMLError) as e: app.error("Invalid framework authentication key file format %s" % e) scheduler_key = None try: with open(options.scheduler_keypath, 'rb') as f: scheduler_key = f.read().strip() if not scheduler_key: raise ValueError("The key file is empty") except Exception as e: app.error("Cannot read --scheduler_keypath: %s" % e) log.info("Starting Mysos scheduler") kazoo = KazooClient(zk_servers) kazoo.start() if options.state_storage == 'zk': log.info("Using ZooKeeper (path: %s) for state storage" % zk_root) state_provider = ZooKeeperStateProvider(kazoo, zk_root) else: log.info("Using local disk for state storage") state_provider = LocalStateProvider(options.work_dir) try: state = state_provider.load_scheduler_state() except StateProvider.Error as e: app.error(e.message) if state: log.info("Successfully restored scheduler state") framework_info = state.framework_info if framework_info.HasField('id'): log.info("Recovered scheduler's FrameworkID is %s" % framework_info.id.value) else: log.info("No scheduler state to restore") framework_info = FrameworkInfo( user=options.framework_user, name=FRAMEWORK_NAME, checkpoint=True, failover_timeout=framework_failover_timeout.as_(Time.SECONDS), role=options.framework_role, hostname=options.hostname, webui_url="http://%s:%s/" % (options.hostname, options.api_port)) if fw_principal: framework_info.principal = fw_principal state = Scheduler(framework_info) state_provider.dump_scheduler_state(state) scheduler = MysosScheduler( state, state_provider, options.framework_user, options.executor_uri, options.executor_cmd, kazoo, options.zk_url, election_timeout, options.admin_keypath, scheduler_key, installer_args=options.installer_args, backup_store_args=options.backup_store_args, executor_environ=options.executor_environ, executor_source_prefix=options.executor_source_prefix, docker_image=options.docker_image, framework_role=options.framework_role) RootMetrics().register_observable('scheduler', scheduler) if fw_principal and fw_secret: cred = Credential(principal=fw_principal, secret=fw_secret) scheduler_driver = mesos.native.MesosSchedulerDriver( scheduler, framework_info, options.mesos_master, cred) else: scheduler_driver = mesos.native.MesosSchedulerDriver( scheduler, framework_info, options.mesos_master) scheduler_driver.start() metric_sampler = MetricSampler(RootMetrics()) metric_sampler.start() server = HttpServer() server.mount_routes(MysosServer(scheduler, web_assets_dir, metric_sampler)) et = ExceptionalThread( target=server.run, args=('0.0.0.0', options.api_port, 'cherrypy')) et.daemon = True et.start() try: # Wait for the scheduler to stop. # The use of 'stopped' event instead of scheduler_driver.join() is necessary to stop the # process with SIGINT. while not scheduler.stopped.wait(timeout=0.5): pass except KeyboardInterrupt: log.info('Interrupted, exiting.') else: log.info('Scheduler exited.') app.shutdown(1) # Mysos scheduler is supposed to be long-running thus the use of exit status 1.
def run(self, lock): if self.options.dry_run: print '****** Dry Run ******' logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get( 'goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn( '--all-recursive is deprecated, use a target spec with the form [dir]:: instead' ) for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn( '--all is deprecated, use a target spec with the form [dir]: instead' ) for dir in self.options.target_directory: self.add_target_directory(dir) context = Context(self.config, self.options, self.targets, requested_goals=self.requested_goals, lock=lock, log=logger, timer=self.timer if self.options.time else None) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) ret = Phase.attempt(context, self.phases) if self.options.cleanup_nailguns or self.config.get( 'nailgun', 'autokill', default=False): if log: log.debug('auto-killing nailguns') if NailgunTask.killall: NailgunTask.killall(log) if self.options.time: print('Timing report') print('=============') self.timer.print_timings() return ret
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from twitter.common import app from twitter.common.log.options import LogOptions from apache.aurora.admin import help as help_commands from apache.aurora.admin import admin, maintenance from .help import add_verbosity_options, generate_terse_usage app.register_commands_from(admin, help_commands, maintenance) add_verbosity_options() def main(): app.help() LogOptions.set_stderr_log_level('INFO') LogOptions.disable_disk_logging() app.set_name('aurora-admin') app.set_usage(generate_terse_usage()) def proxy_main(): app.main()
import os import pytest import threading import time import unittest import zookeeper from twitter.common.zookeeper.client import ZooKeeper, ZooDefs from twitter.common.zookeeper.test_server import ZookeeperServer from twitter.common.zookeeper.group.group import ActiveGroup, Group, Membership if os.getenv('ZOOKEEPER_TEST_DEBUG'): from twitter.common import log from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level('DEBUG') LogOptions.set_disk_log_level('NONE') LogOptions.set_log_dir('/tmp') log.init('client_test') class AlternateGroup(Group): MEMBER_PREFIX = 'herpderp_' class TestGroup(unittest.TestCase): GroupImpl = Group MAX_EVENT_WAIT_SECS = 30.0 CONNECT_TIMEOUT_SECS = 10.0 CONNECT_RETRIES = 6
def _do_run(self): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # TODO(Eric Ayers) We are missing log messages. Set the log level earlier # Enable standard python logging for code with no handle to a context/work-unit. if self.global_options.level: LogOptions.set_stderr_log_level((self.global_options.level or 'info').upper()) logdir = self.global_options.logdir or self.config.get( 'goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) prev_log_level = None # If quiet, temporarily change stderr log level to kill init's output. if self.global_options.quiet: prev_log_level = LogOptions.loglevel_name( LogOptions.stderr_log_level()) # loglevel_name can fail, so only change level if we were able to get the current one. if prev_log_level is not None: LogOptions.set_stderr_log_level( LogOptions._LOG_LEVEL_NONE_KEY) log.init('goals') if prev_log_level is not None: LogOptions.set_stderr_log_level(prev_log_level) else: log.init() # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False is_explain = self.global_options.explain update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker) context = Context(config=self.config, options=self.options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, spec_excludes=self.get_spec_excludes()) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown)) return 1 engine = RoundEngine() return engine.execute(context, self.goals)
def set_verbose(option, _1, _2, parser): setattr(parser.values, option.dest, 'verbose') LogOptions.set_stderr_log_level('DEBUG')
def set_quiet(option, _1, _2, parser): setattr(parser.values, option.dest, "quiet") LogOptions.set_stderr_log_level("NONE")
def run(self, lock): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # Enable standard python logging for code with no handle to a context/work-unit. if self.options.log_level: LogOptions.set_stderr_log_level((self.options.log_level or "info").upper()) logdir = self.options.logdir or self.config.get("goals", "logdir", default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) log.init("goals") else: log.init() # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False # Target specs are mapped to the patterns which match them, if any. This variable is a key for # specs which don't match any exclusion regexes. We know it won't already be in the list of # patterns, because the asterisks in its name make it an invalid regex. _UNMATCHED_KEY = "** unmatched **" def targets_by_pattern(targets, patterns): mapping = defaultdict(list) for target in targets: matched_pattern = None for pattern in patterns: if re.search(pattern, target.address.spec) is not None: matched_pattern = pattern break if matched_pattern is None: mapping[_UNMATCHED_KEY].append(target) else: mapping[matched_pattern].append(target) return mapping is_explain = self.options.explain update_reporting(self.options, is_quiet_task() or is_explain, self.run_tracker) if self.options.target_excludes: excludes = self.options.target_excludes log.debug("excludes:\n {excludes}".format(excludes="\n ".join(excludes))) by_pattern = targets_by_pattern(self.targets, excludes) self.targets = by_pattern[_UNMATCHED_KEY] # The rest of this if-statement is just for debug logging. log.debug( "Targets after excludes: {targets}".format(targets=", ".join(t.address.spec for t in self.targets)) ) excluded_count = sum(len(by_pattern[p]) for p in excludes) log.debug( "Excluded {count} target{plural}.".format( count=excluded_count, plural=("s" if excluded_count != 1 else "") ) ) for pattern in excludes: log.debug( "Targets excluded by pattern {pattern}\n {targets}".format( pattern=pattern, targets="\n ".join(t.address.spec for t in by_pattern[pattern]) ) ) context = Context( config=self.config, options=self.options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, lock=lock, ) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error("Unknown goal(s): %s\n" % " ".join(goal.name for goal in unknown)) return 1 engine = RoundEngine() return engine.execute(context, self.goals)
def run(self, lock): timer = None if self.options.time: class Timer(object): def now(self): return time.time() def log(self, message): print(message) timer = Timer() logger = None if self.options.log or self.options.log_level: from twitter.common.log import init from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper()) logdir = self.options.logdir or self.config.get( 'goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) init('goals') else: init() logger = log if self.options.recursive_directory: log.warn( '--all-recursive is deprecated, use a target spec with the form [dir]:: instead' ) for dir in self.options.recursive_directory: self.add_target_recursive(dir) if self.options.target_directory: log.warn( '--all is deprecated, use a target spec with the form [dir]: instead' ) for dir in self.options.target_directory: self.add_target_directory(dir) context = Context(self.config, self.options, self.targets, lock=lock, log=logger) unknown = [] for phase in self.phases: if not phase.goals(): unknown.append(phase) if unknown: print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown)) print('') return Phase.execute(context, 'goals') if logger: logger.debug('Operating on targets: %s', self.targets) return Phase.attempt(context, self.phases, timer=timer)
from twitter.common import log from twitter.common.log.options import LogOptions from twitter.common.zookeeper.client import ZooKeeper, ZooDefs from twitter.common.zookeeper.test_server import ZookeeperServer import mox MAX_EVENT_WAIT_SECS = 30.0 MAX_EXPIRE_WAIT_SECS = 60.0 CONNECT_TIMEOUT_SECS = 10.0 CONNECT_RETRIES = 6 if os.getenv('ZOOKEEPER_TEST_DEBUG'): LogOptions.set_stderr_log_level('NONE') LogOptions.set_disk_log_level('DEBUG') LogOptions.set_log_dir('/tmp') log.init('client_test') def make_zk(server, **kw): return ZooKeeper('localhost:%d' % server.zookeeper_port, timeout_secs=CONNECT_TIMEOUT_SECS, max_reconnects=CONNECT_RETRIES, **kw) def test_client_connect(): with ZookeeperServer() as server: zk = make_zk(server)
return TaskObserver(path_detector, interval=polling_interval) def handle_error(exc_type, value, traceback): """ Tear down the observer in case of unhandled errors. By using ExceptionalThread throughout the observer we have ensured that sys.excepthook will be called for every unhandled exception, even for those not originating in the main thread. """ log.error("An unhandled error occured. Tearing down.", exc_info=(exc_type, value, traceback)) # TODO: In Python 3.4 we will be able to use threading.main_thread() if not isinstance(threading.current_thread(), threading._MainThread): thread.interrupt_main() def main(_, options): observer = initialize(options) observer.start() root_server = configure_server(observer) server = ExceptionalThread(target=lambda: root_server.run(options.ip, options.port, "cherrypy")) server.daemon = True server.start() sleep_forever() sys.excepthook = handle_error LogOptions.set_stderr_log_level("google:INFO") app.main()
def run(self): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # Enable standard python logging for code with no handle to a context/work-unit. if self.global_options.level: LogOptions.set_stderr_log_level((self.global_options.level or 'info').upper()) logdir = self.global_options.logdir or self.config.get( 'goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) prev_log_level = None # If quiet, temporarily change stderr log level to kill init's output. if self.global_options.quiet: prev_log_level = LogOptions.loglevel_name( LogOptions.stderr_log_level()) # loglevel_name can fail, so only change level if we were able to get the current one. if prev_log_level is not None: LogOptions.set_stderr_log_level( LogOptions._LOG_LEVEL_NONE_KEY) log.init('goals') if prev_log_level is not None: LogOptions.set_stderr_log_level(prev_log_level) else: log.init() # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False # Target specs are mapped to the patterns which match them, if any. This variable is a key for # specs which don't match any exclusion regexes. We know it won't already be in the list of # patterns, because the asterisks in its name make it an invalid regex. _UNMATCHED_KEY = '** unmatched **' def targets_by_pattern(targets, patterns): mapping = defaultdict(list) for target in targets: matched_pattern = None for pattern in patterns: if re.search(pattern, target.address.spec) is not None: matched_pattern = pattern break if matched_pattern is None: mapping[_UNMATCHED_KEY].append(target) else: mapping[matched_pattern].append(target) return mapping is_explain = self.global_options.explain update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker) if self.global_options.exclude_target_regexp: excludes = self.global_options.exclude_target_regexp log.debug('excludes:\n {excludes}'.format( excludes='\n '.join(excludes))) by_pattern = targets_by_pattern(self.targets, excludes) self.targets = by_pattern[_UNMATCHED_KEY] # The rest of this if-statement is just for debug logging. log.debug('Targets after excludes: {targets}'.format( targets=', '.join(t.address.spec for t in self.targets))) excluded_count = sum(len(by_pattern[p]) for p in excludes) log.debug('Excluded {count} target{plural}.'.format( count=excluded_count, plural=('s' if excluded_count != 1 else ''))) for pattern in excludes: log.debug('Targets excluded by pattern {pattern}\n {targets}'. format(pattern=pattern, targets='\n '.join( t.address.spec for t in by_pattern[pattern]))) context = Context(config=self.config, new_options=self.new_options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, spec_excludes=self.get_spec_excludes()) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown)) return 1 engine = RoundEngine() return engine.execute(context, self.goals)
from twitter.common import app, log from twitter.common.exceptions import ExceptionalThread from twitter.common.http import HttpServer from twitter.common.log.options import LogOptions from twitter.common.quantity import Time from twitter.common.quantity.parse_simple import InvalidTime, parse_time import yaml FRAMEWORK_NAME = 'mysos' MYSOS_MODULE = 'mysos.scheduler' ASSET_RELPATH = 'assets' LogOptions.disable_disk_logging() LogOptions.set_stderr_log_level('google:INFO') def proxy_main(): app.add_option( '--port', dest='api_port', type='int', default=None, help='Port for the HTTP API server') app.add_option( '--mesos_master', dest='mesos_master', default=None, help='Mesos master address. It can be a ZooKeeper URL through which the master can be '
from mesos.interface.mesos_pb2 import Credential, FrameworkInfo import mesos.native from twitter.common import app, log from twitter.common.exceptions import ExceptionalThread from twitter.common.http import HttpServer from twitter.common.log.options import LogOptions from twitter.common.quantity import Time from twitter.common.quantity.parse_simple import InvalidTime, parse_time import yaml FRAMEWORK_NAME = 'mysos' MYSOS_MODULE = 'mysos.scheduler' ASSET_RELPATH = 'assets' LogOptions.disable_disk_logging() LogOptions.set_stderr_log_level('google:INFO') def proxy_main(): app.add_option('--port', dest='api_port', type='int', default=None, help='Port for the HTTP API server') app.add_option( '--mesos_master', dest='mesos_master', default=None, help= 'Mesos master address. It can be a ZooKeeper URL through which the master can be '
def set_quiet(option, _1, _2, parser): setattr(parser.values, option.dest, 'quiet') LogOptions.set_stderr_log_level('NONE')
def setup(): from twitter.common import app LogOptions.set_stderr_log_level('NONE') app.add_option('--iface', default='eth0', type=str, metavar='<iface>', help='The interface to sniff on') app.add_option('--client-port', default=0, type=int, metavar='<client_port>', help='The client port to filter by') app.add_option('--zookeeper-port', default=2181, type=int, metavar='<server_port>', help='The ZooKeeper server port to filter by') app.add_option( '--max-queued-requests', default=10000, type=int, metavar='<max>', help='The maximum number of requests queued to be deserialized') app.add_option( '--exclude-host', dest='excluded_hosts', metavar='<host>', default=[], action='append', help='Host that should be excluded (you can use this multiple times)') app.add_option( '--include-host', dest='included_hosts', metavar='<host>', default=[], action='append', help='Host that should be included (you can use this multiple times)') app.add_option( '--count-requests', default=0, type=int, metavar='<nreqs>', help='Count N requests and report a summary (default: group by path)') app.add_option( '--measure-latency', default=0, type=int, metavar='<nreqs>', help= 'Measure latency of N pairs of requests and replies (default: group by path' ) app.add_option( '--group-by', default='path', type=str, metavar='<group>', help= 'Used with --count-requests or --measure-latency. Possible values: path, type or client' ) app.add_option( '--sort-by', default='avg', type=str, metavar='<sort>', help='Used with --measure-latency. Possible values: avg, p95 and p99') app.add_option( "--aggregation-depth", default=0, type=int, metavar='<depth>', help= "Aggregate paths up to a certain depth. Used with --count-requests or --measure-latency" ) app.add_option('--unpaired', default=False, action='store_true', help='Don\'t pair reqs/reps') app.add_option('-p', '--include-pings', default=False, action='store_true', help='Whether to include ping requests and replies') app.add_option('-c', '--colors', default=False, action='store_true', help='Color each client/server stream differently') app.add_option('--dump-bad-packet', default=False, action='store_true', help='If unable to to deserialize a packet, print it out') app.add_option('--version', default=False, action='store_true')
def set_quiet(option, _1, _2, parser): setattr(parser.values, option.dest, 'quiet') LogOptions.set_stderr_log_level('NONE')
def set_verbose(option, _1, _2, parser): setattr(parser.values, option.dest, 'verbose') LogOptions.set_stderr_log_level('DEBUG')
from apache.aurora.executor.common.executor_timeout import ExecutorTimeout from apache.aurora.executor.common.health_checker import HealthCheckerProvider from apache.aurora.executor.common.sandbox import DirectorySandbox, SandboxProvider from apache.aurora.executor.common.status_checker import ChainedStatusChecker from apache.aurora.executor.common.task_runner import TaskError from apache.aurora.executor.status_manager import StatusManager from apache.aurora.executor.thermos_task_runner import ( DefaultThermosTaskRunnerProvider, ThermosTaskRunner) from apache.thermos.core.runner import TaskRunner from apache.thermos.monitoring.monitor import TaskMonitor from gen.apache.aurora.api.constants import AURORA_EXECUTOR_NAME from gen.apache.aurora.api.ttypes import AssignedTask, ExecutorConfig, JobKey, TaskConfig if 'THERMOS_DEBUG' in os.environ: LogOptions.set_stderr_log_level('google:DEBUG') LogOptions.set_simple(True) log.init('executor_logger') class FastThermosExecutor(AuroraExecutor): STOP_WAIT = Amount(0, Time.SECONDS) class FastStatusManager(StatusManager): POLL_WAIT = Amount(10, Time.MILLISECONDS) class DefaultTestSandboxProvider(SandboxProvider): def from_assigned_task(self, assigned_task, **kwargs): return DirectorySandbox(safe_mkdtemp(), **kwargs)
from apache.aurora.executor.aurora_executor import AuroraExecutor from apache.aurora.executor.common.executor_timeout import ExecutorTimeout from apache.aurora.executor.common.health_checker import HealthCheckerProvider from apache.aurora.executor.common.sandbox import DirectorySandbox, SandboxProvider from apache.aurora.executor.common.status_checker import ChainedStatusChecker from apache.aurora.executor.common.task_runner import TaskError from apache.aurora.executor.status_manager import StatusManager from apache.aurora.executor.thermos_task_runner import DefaultThermosTaskRunnerProvider, ThermosTaskRunner from apache.thermos.core.runner import TaskRunner from apache.thermos.monitoring.monitor import TaskMonitor from gen.apache.aurora.api.constants import AURORA_EXECUTOR_NAME from gen.apache.aurora.api.ttypes import AssignedTask, ExecutorConfig, Identity, JobKey, TaskConfig if "THERMOS_DEBUG" in os.environ: LogOptions.set_stderr_log_level("google:DEBUG") LogOptions.set_simple(True) log.init("executor_logger") class FastThermosExecutor(AuroraExecutor): STOP_WAIT = Amount(0, Time.SECONDS) class FastStatusManager(StatusManager): POLL_WAIT = Amount(10, Time.MILLISECONDS) class DefaultTestSandboxProvider(SandboxProvider): def from_assigned_task(self, assigned_task): return DirectorySandbox(safe_mkdtemp())
from mysos.scheduler.state import LocalStateProvider, Scheduler from kazoo.handlers.threading import SequentialThreadingHandler from mesos.interface.mesos_pb2 import DRIVER_STOPPED, FrameworkInfo from twitter.common import log from twitter.common.concurrent import deadline from twitter.common.dirutil import safe_mkdtemp from twitter.common.metrics import RootMetrics from twitter.common.quantity import Amount, Time from zake.fake_client import FakeClient from zake.fake_storage import FakeStorage if 'MYSOS_DEBUG' in os.environ: from twitter.common.log.options import LogOptions LogOptions.set_stderr_log_level('google:DEBUG') LogOptions.set_simple(True) log.init('mysos_tests') def test_scheduler_runs(): """ Verifies that the scheduler successfully launches 3 "no-op" MySQL tasks. NOTE: Due to the limitation of zake the scheduler's ZK operations are not propagated to executors in separate processes but they are unit-tested separately. """ import mesos.native # Make sure fake_mysos_executor.pex is available to be fetched by Mesos slave. assert os.path.isfile('dist/fake_mysos_executor.pex')
def _do_run(self): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # TODO(Eric Ayers) We are missing log messages. Set the log level earlier # Enable standard python logging for code with no handle to a context/work-unit. if self.global_options.level: LogOptions.set_stderr_log_level((self.global_options.level or 'info').upper()) logdir = self.global_options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) prev_log_level = None # If quiet, temporarily change stderr log level to kill init's output. if self.global_options.quiet: prev_log_level = LogOptions.loglevel_name(LogOptions.stderr_log_level()) # loglevel_name can fail, so only change level if we were able to get the current one. if prev_log_level is not None: LogOptions.set_stderr_log_level(LogOptions._LOG_LEVEL_NONE_KEY) log.init('goals') if prev_log_level is not None: LogOptions.set_stderr_log_level(prev_log_level) else: log.init() # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False is_explain = self.global_options.explain update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker) context = Context( config=self.config, options=self.options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, spec_excludes=self.get_spec_excludes() ) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown)) return 1 engine = RoundEngine() return engine.execute(context, self.goals)
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from apache.aurora.client.base import generate_terse_usage from apache.aurora.client.commands import admin, help from apache.aurora.client.options import add_verbosity_options from twitter.common import app from twitter.common.log.options import LogOptions app.register_commands_from(admin, help) add_verbosity_options() def main(): app.help() LogOptions.set_stderr_log_level("INFO") LogOptions.disable_disk_logging() app.set_name("aurora-admin") app.set_usage(generate_terse_usage()) def proxy_main(): app.main()
ScheduleStatusResult, ServerInfo, SessionKey, TaskConfig, TaskConstraint, TaskQuery, ValueConstraint, ) # Debug output helper -> enables log.* in source. if "UPDATER_DEBUG" in environ: from twitter.common import log from twitter.common.log.options import LogOptions LogOptions.set_disk_log_level("NONE") LogOptions.set_stderr_log_level("DEBUG") log.init("test_updater") SERVER_INFO = ServerInfo(thriftAPIVersion=THRIFT_API_VERSION) def make_response(code, msg="test"): return Response(responseCode=code, serverInfo=SERVER_INFO, details=[ResponseDetail(message=msg)]) class FakeConfig(object): def __init__(self, role, name, env, update_config): self._role = role self._env = env self._name = name self._update_config = update_config
ScheduleStatusResult, ScheduledTask, TaskConfig, TaskQuery, ) from mox import MockObject, Replay, Verify from pytest import raises # Debug output helper -> enables log.* in source. if 'UPDATER_DEBUG' in environ: from twitter.common import log from twitter.common.log.options import LogOptions LogOptions.set_disk_log_level('NONE') LogOptions.set_stderr_log_level('DEBUG') log.init('test_updater') class FakeConfig(object): def __init__(self, role, name, env, update_config): self._role = role self._env = env self._name = name self._update_config = update_config self.job_config = None def role(self): return self._role def name(self): return self._name
def run(self): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # Enable standard python logging for code with no handle to a context/work-unit. if self.global_options.level: LogOptions.set_stderr_log_level((self.global_options.level or 'info').upper()) logdir = self.global_options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) prev_log_level = None # If quiet, temporarily change stderr log level to kill init's output. if self.global_options.quiet: prev_log_level = LogOptions.loglevel_name(LogOptions.stderr_log_level()) # loglevel_name can fail, so only change level if we were able to get the current one. if prev_log_level is not None: LogOptions.set_stderr_log_level(LogOptions._LOG_LEVEL_NONE_KEY) log.init('goals') if prev_log_level is not None: LogOptions.set_stderr_log_level(prev_log_level) else: log.init() # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False # Target specs are mapped to the patterns which match them, if any. This variable is a key for # specs which don't match any exclusion regexes. We know it won't already be in the list of # patterns, because the asterisks in its name make it an invalid regex. _UNMATCHED_KEY = '** unmatched **' def targets_by_pattern(targets, patterns): mapping = defaultdict(list) for target in targets: matched_pattern = None for pattern in patterns: if re.search(pattern, target.address.spec) is not None: matched_pattern = pattern break if matched_pattern is None: mapping[_UNMATCHED_KEY].append(target) else: mapping[matched_pattern].append(target) return mapping is_explain = self.global_options.explain update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker) if self.global_options.exclude_target_regexp: excludes = self.global_options.exclude_target_regexp log.debug('excludes:\n {excludes}'.format(excludes='\n '.join(excludes))) by_pattern = targets_by_pattern(self.targets, excludes) self.targets = by_pattern[_UNMATCHED_KEY] # The rest of this if-statement is just for debug logging. log.debug('Targets after excludes: {targets}'.format( targets=', '.join(t.address.spec for t in self.targets))) excluded_count = sum(len(by_pattern[p]) for p in excludes) log.debug('Excluded {count} target{plural}.'.format(count=excluded_count, plural=('s' if excluded_count != 1 else ''))) for pattern in excludes: log.debug('Targets excluded by pattern {pattern}\n {targets}'.format(pattern=pattern, targets='\n '.join(t.address.spec for t in by_pattern[pattern]))) context = Context( config=self.config, new_options=self.new_options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, spec_excludes=self.get_spec_excludes() ) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown)) return 1 engine = RoundEngine() return engine.execute(context, self.goals)