def _setup_stderr_logging():
filter = GenericFilter(lambda r_l: r_l >= LogOptions.stderr_log_level())
formatter = ProxyFormatter(LogOptions.stderr_log_scheme)
stderr_handler = logging.StreamHandler(sys.stderr)
stderr_handler.setFormatter(formatter)
stderr_handler.addFilter(filter)
return [stderr_handler]
def publish(ivyxml_path):
ivysettings = self.generate_ivysettings(published, publish_local=path)
args = [
'-settings', ivysettings,
'-ivy', ivyxml_path,
'-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.outdir,
'-publish', resolver,
'-publishpattern', '%s/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]' % self.outdir,
'-revision', newver.version(),
'-m2compatible',
]
if LogOptions.stderr_log_level() == logging.DEBUG:
args.append('-verbose')
if self.snapshot:
args.append('-overwrite')
try:
ivy = Bootstrapper.default_ivy()
ivy.execute(jvm_options=jvm_args, args=args,
workunit_factory=self.context.new_workunit, workunit_name = 'jar-publish')
except (Bootstrapper.Error, Ivy.Error) as e:
raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
def publish(ivyxml_path):
ivysettings = self.generate_ivysettings(published, publish_local=path)
args = [
'-settings', ivysettings,
'-ivy', ivyxml_path,
'-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
'-publish', resolver,
'-publishpattern', '%s/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
'-revision', newver.version(),
'-m2compatible',
]
if LogOptions.stderr_log_level() == logging.DEBUG:
args.append('-verbose')
if self.snapshot:
args.append('-overwrite')
try:
ivy = Bootstrapper.default_ivy()
ivy.execute(jvm_options=jvm_args, args=args,
workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
except (Bootstrapper.Error, Ivy.Error) as e:
raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
def init(filebase=None):
"""
Sets up default stderr logging and, if filebase is supplied, sets up disk logging using:
{--log_dir}/filebase.{INFO,WARNING,...}
"""
logging._acquireLock()
# set up permissive logger
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
# clear existing handlers
teardown_stderr_logging()
teardown_disk_logging()
for handler in root_logger.handlers:
root_logger.removeHandler(handler)
# setup INFO...FATAL handlers
if filebase:
for handler in _setup_disk_logging(filebase):
root_logger.addHandler(handler)
_DISK_LOGGERS.append(handler)
for handler in _setup_stderr_logging():
root_logger.addHandler(handler)
_STDERR_LOGGERS.append(handler)
logging._releaseLock()
if len(_DISK_LOGGERS) > 0 and LogOptions.stderr_log_level(
) != LogOptions.LOG_LEVEL_NONE:
print('Writing log files to disk in %s' % LogOptions.log_dir(),
file=sys.stderr)
return root_logger
def init(filebase):
"""
Set up default logging using:
{--log_dir}/filebase.{INFO,WARNING,...}
"""
logging._acquireLock()
# set up permissive logger
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
# clear existing handlers
teardown_stderr_logging()
teardown_disk_logging()
# setup INFO...FATAL handlers
num_disk_handlers = 0
for handler in _setup_disk_logging(filebase):
root_logger.addHandler(handler)
_DISK_LOGGERS.append(handler)
for handler in _setup_stderr_logging():
root_logger.addHandler(handler)
_STDERR_LOGGERS.append(handler)
logging._releaseLock()
if len(_DISK_LOGGERS) > 0 and LogOptions.stderr_log_level() != LogOptions.LOG_LEVEL_NONE:
print('Writing log files to disk in %s' % LogOptions.log_dir(), file=sys.stderr)
return root_logger
def publish(self, ivyxml_path, jar, entry, repo, published):
"""Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published
is a list of jars published so far (including this one). entry is a pushdb entry."""
jvm_args = self._ivy_jvm_args(repo)
resolver = repo['resolver']
path = repo.get('path')
try:
ivy = Bootstrapper.default_ivy()
except Bootstrapper.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
ivysettings = self.generate_ivysettings(ivy, published, publish_local=path)
args = [
'-settings', ivysettings,
'-ivy', ivyxml_path,
'-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
'-publish', resolver,
'-publishpattern', '%s/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
'-revision', entry.version().version(),
'-m2compatible',
]
if LogOptions.stderr_log_level() == logging.DEBUG:
args.append('-verbose')
if self.local_snapshot:
args.append('-overwrite')
try:
ivy.execute(jvm_options=jvm_args, args=args,
workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
except Ivy.Error as e:
raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
def _setup_stderr_logging(): filter = GenericFilter(lambda r_l: r_l >= LogOptions.stderr_log_level()) formatter = ProxyFormatter(LogOptions.stderr_log_scheme) stderr_handler = logging.StreamHandler(sys.stderr) stderr_handler.setFormatter(formatter) stderr_handler.addFilter(filter) return [stderr_handler]
def _set_log_level(self, log_level_override=''):
stderr_log_level = LogOptions.stderr_log_level()
# set default level to FATAL.
# we do this here (instead of add_option) to distinguish when an override is set.
if stderr_log_level == log.INFO and log_level_override != 'INFO':
stderr_log_level = log.FATAL
# default to using stderr logging level, setting override if applicable
log_level = getattr(log, log_level_override, stderr_log_level)
# set the logger
zk_log_level = ZookeeperLoggingSubsystem._ZK_LOG_LEVEL_MAP.get(
log_level, zookeeper.LOG_LEVEL_ERROR)
zookeeper.set_debug_level(zk_log_level)
def _set_log_level(self, log_level_override=''):
stderr_log_level = LogOptions.stderr_log_level()
# set default level to FATAL.
# we do this here (instead of add_option) to distinguish when an override is set.
if stderr_log_level == log.INFO and log_level_override != 'INFO':
stderr_log_level = log.FATAL
# default to using stderr logging level, setting override if applicable
log_level = getattr(log, log_level_override, stderr_log_level)
# set the logger
zk_log_level = ZookeeperLoggingSubsystem._ZK_LOG_LEVEL_MAP.get(
log_level, zookeeper.LOG_LEVEL_ERROR)
zookeeper.set_debug_level(zk_log_level)
def print_stderr(message):
"""Emit a message on standard error if logging to stderr is permitted."""
if LogOptions.stderr_log_level() != LogOptions.LOG_LEVEL_NONE:
print(message, file=sys.stderr)
def log_function(msg):
if _LOG_MODULE:
log.error(msg)
# ensure that at least one message goes to stdout/stderr
if not _LOG_MODULE or LogOptions.stderr_log_level() > logging.ERROR:
sys.stderr.write(msg)
def _do_run(self):
# TODO(John Sirois): Consider moving to straight python logging. The divide between the
# context/work-unit logging and standard python logging doesn't buy us anything.
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
if self.global_options.level:
LogOptions.set_stderr_log_level((self.global_options.level or 'info').upper())
logdir = self.global_options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
prev_log_level = None
# If quiet, temporarily change stderr log level to kill init's output.
if self.global_options.quiet:
prev_log_level = LogOptions.loglevel_name(LogOptions.stderr_log_level())
# loglevel_name can fail, so only change level if we were able to get the current one.
if prev_log_level is not None:
LogOptions.set_stderr_log_level(LogOptions._LOG_LEVEL_NONE_KEY)
log.init('goals')
if prev_log_level is not None:
LogOptions.set_stderr_log_level(prev_log_level)
else:
log.init()
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
is_explain = self.global_options.explain
update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker)
context = Context(
config=self.config,
options=self.options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.get_spec_excludes()
)
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
def run(self):
# TODO(John Sirois): Consider moving to straight python logging. The divide between the
# context/work-unit logging and standard python logging doesn't buy us anything.
# Enable standard python logging for code with no handle to a context/work-unit.
if self.global_options.level:
LogOptions.set_stderr_log_level((self.global_options.level or 'info').upper())
logdir = self.global_options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
prev_log_level = None
# If quiet, temporarily change stderr log level to kill init's output.
if self.global_options.quiet:
prev_log_level = LogOptions.loglevel_name(LogOptions.stderr_log_level())
# loglevel_name can fail, so only change level if we were able to get the current one.
if prev_log_level is not None:
LogOptions.set_stderr_log_level(LogOptions._LOG_LEVEL_NONE_KEY)
log.init('goals')
if prev_log_level is not None:
LogOptions.set_stderr_log_level(prev_log_level)
else:
log.init()
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
# Target specs are mapped to the patterns which match them, if any. This variable is a key for
# specs which don't match any exclusion regexes. We know it won't already be in the list of
# patterns, because the asterisks in its name make it an invalid regex.
_UNMATCHED_KEY = '** unmatched **'
def targets_by_pattern(targets, patterns):
mapping = defaultdict(list)
for target in targets:
matched_pattern = None
for pattern in patterns:
if re.search(pattern, target.address.spec) is not None:
matched_pattern = pattern
break
if matched_pattern is None:
mapping[_UNMATCHED_KEY].append(target)
else:
mapping[matched_pattern].append(target)
return mapping
is_explain = self.global_options.explain
update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker)
if self.global_options.exclude_target_regexp:
excludes = self.global_options.exclude_target_regexp
log.debug('excludes:\n {excludes}'.format(excludes='\n '.join(excludes)))
by_pattern = targets_by_pattern(self.targets, excludes)
self.targets = by_pattern[_UNMATCHED_KEY]
# The rest of this if-statement is just for debug logging.
log.debug('Targets after excludes: {targets}'.format(
targets=', '.join(t.address.spec for t in self.targets)))
excluded_count = sum(len(by_pattern[p]) for p in excludes)
log.debug('Excluded {count} target{plural}.'.format(count=excluded_count,
plural=('s' if excluded_count != 1 else '')))
for pattern in excludes:
log.debug('Targets excluded by pattern {pattern}\n {targets}'.format(pattern=pattern,
targets='\n '.join(t.address.spec for t in by_pattern[pattern])))
context = Context(
config=self.config,
new_options=self.new_options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.get_spec_excludes()
)
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
def log_function(msg):
if _LOG_MODULE:
log.error(msg)
# ensure that at least one message goes to stdout/stderr
if not _LOG_MODULE or LogOptions.stderr_log_level() > logging.ERROR:
sys.stderr.write(msg)
def _set_default_log_level(self):
log_level = LogOptions.stderr_log_level()
zk_log_level = ZookeeperLoggingSubsystem._ZK_LOG_LEVEL_MAP.get(
log_level, zookeeper.LOG_LEVEL_ERROR)
zookeeper.set_debug_level(zk_log_level)
def _do_run(self):
# TODO(John Sirois): Consider moving to straight python logging. The divide between the
# context/work-unit logging and standard python logging doesn't buy us anything.
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
if self.global_options.level:
LogOptions.set_stderr_log_level((self.global_options.level
or 'info').upper())
logdir = self.global_options.logdir or self.config.get(
'goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
prev_log_level = None
# If quiet, temporarily change stderr log level to kill init's output.
if self.global_options.quiet:
prev_log_level = LogOptions.loglevel_name(
LogOptions.stderr_log_level())
# loglevel_name can fail, so only change level if we were able to get the current one.
if prev_log_level is not None:
LogOptions.set_stderr_log_level(
LogOptions._LOG_LEVEL_NONE_KEY)
log.init('goals')
if prev_log_level is not None:
LogOptions.set_stderr_log_level(prev_log_level)
else:
log.init()
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
is_explain = self.global_options.explain
update_reporting(self.global_options,
is_quiet_task() or is_explain, self.run_tracker)
context = Context(config=self.config,
options=self.options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.get_spec_excludes())
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' %
' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
def _set_default_log_level(self):
log_level = LogOptions.stderr_log_level()
zk_log_level = ZookeeperLoggingSubsystem._ZK_LOG_LEVEL_MAP.get(
log_level, zookeeper.LOG_LEVEL_ERROR)
zookeeper.set_debug_level(zk_log_level)
def print_stderr(message):
"""Emit a message on standard error if logging to stderr is permitted."""
if LogOptions.stderr_log_level() != LogOptions.LOG_LEVEL_NONE:
print(message, file=sys.stderr)
def run(self):
# TODO(John Sirois): Consider moving to straight python logging. The divide between the
# context/work-unit logging and standard python logging doesn't buy us anything.
# Enable standard python logging for code with no handle to a context/work-unit.
if self.global_options.level:
LogOptions.set_stderr_log_level((self.global_options.level
or 'info').upper())
logdir = self.global_options.logdir or self.config.get(
'goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
prev_log_level = None
# If quiet, temporarily change stderr log level to kill init's output.
if self.global_options.quiet:
prev_log_level = LogOptions.loglevel_name(
LogOptions.stderr_log_level())
# loglevel_name can fail, so only change level if we were able to get the current one.
if prev_log_level is not None:
LogOptions.set_stderr_log_level(
LogOptions._LOG_LEVEL_NONE_KEY)
log.init('goals')
if prev_log_level is not None:
LogOptions.set_stderr_log_level(prev_log_level)
else:
log.init()
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
# Target specs are mapped to the patterns which match them, if any. This variable is a key for
# specs which don't match any exclusion regexes. We know it won't already be in the list of
# patterns, because the asterisks in its name make it an invalid regex.
_UNMATCHED_KEY = '** unmatched **'
def targets_by_pattern(targets, patterns):
mapping = defaultdict(list)
for target in targets:
matched_pattern = None
for pattern in patterns:
if re.search(pattern, target.address.spec) is not None:
matched_pattern = pattern
break
if matched_pattern is None:
mapping[_UNMATCHED_KEY].append(target)
else:
mapping[matched_pattern].append(target)
return mapping
is_explain = self.global_options.explain
update_reporting(self.global_options,
is_quiet_task() or is_explain, self.run_tracker)
if self.global_options.exclude_target_regexp:
excludes = self.global_options.exclude_target_regexp
log.debug('excludes:\n {excludes}'.format(
excludes='\n '.join(excludes)))
by_pattern = targets_by_pattern(self.targets, excludes)
self.targets = by_pattern[_UNMATCHED_KEY]
# The rest of this if-statement is just for debug logging.
log.debug('Targets after excludes: {targets}'.format(
targets=', '.join(t.address.spec for t in self.targets)))
excluded_count = sum(len(by_pattern[p]) for p in excludes)
log.debug('Excluded {count} target{plural}.'.format(
count=excluded_count,
plural=('s' if excluded_count != 1 else '')))
for pattern in excludes:
log.debug('Targets excluded by pattern {pattern}\n {targets}'.
format(pattern=pattern,
targets='\n '.join(
t.address.spec
for t in by_pattern[pattern])))
context = Context(config=self.config,
new_options=self.new_options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.get_spec_excludes())
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' %
' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
