def emit(self, record):
# type: (logging.LogRecord) -> None
log_entry = beam_fn_api_pb2.LogEntry()
log_entry.severity = self.map_log_level(record.levelno)
log_entry.message = self.format(record)
log_entry.thread = record.threadName
log_entry.log_location = '%s:%s' % (record.pathname or record.module,
record.lineno or record.funcName)
(fraction, seconds) = math.modf(record.created)
nanoseconds = 1e9 * fraction
log_entry.timestamp.seconds = int(seconds)
log_entry.timestamp.nanos = int(nanoseconds)
if record.exc_info:
log_entry.trace = ''.join(
traceback.format_exception(*record.exc_info))
instruction_id = statesampler.get_current_instruction_id()
if instruction_id:
log_entry.instruction_id = instruction_id
tracker = statesampler.get_current_tracker()
if tracker:
current_state = tracker.current_state()
if (current_state and current_state.name_context
and current_state.name_context.transform_id):
log_entry.transform_id = current_state.name_context.transform_id
try:
self._log_entry_queue.put(log_entry, block=False)
except queue.Full:
self._dropped_logs += 1
def __call__(self, value=_DEFAULT):
if value is _DEFAULT:
if self.default is _DEFAULT:
raise ValueError('Missing value for update of %s' %
self.metric_name)
value = self.default
tracker = get_current_tracker()
if tracker is not None:
tracker.update_metric(self.typed_metric_name, value)
def __call__(self, value=_DEFAULT):
# type: (Any) -> None
if value is _DEFAULT:
if self.default_value is _DEFAULT:
raise ValueError('Missing value for update of %s' %
self.typed_metric_name.fast_name)
value = self.default_value
if self.process_wide:
MetricsEnvironment.process_wide_container().get_metric_cell(
self.typed_metric_name).update(value)
else:
tracker = get_current_tracker()
if tracker is not None:
tracker.update_metric(self.typed_metric_name, value)
def current_container(self):
"""Returns the current MetricsContainer."""
sampler = statesampler.get_current_tracker()
if sampler is None:
return None
return sampler.current_state().metrics_container
def current_container(self):
"""Returns the current MetricsContainer."""
sampler = statesampler.get_current_tracker()
if sampler is None:
return self._old_style_container()
return sampler.current_state().metrics_container
def format(self, record):
"""Returns a JSON string based on a LogRecord instance.
Args:
record: A LogRecord instance. See below for details.
Returns:
A JSON string representing the record.
A LogRecord instance has the following attributes and is used for
formatting the final message.
Attributes:
created: A double representing the timestamp for record creation
(e.g., 1438365207.624597). Note that the number contains also msecs and
microsecs information. Part of this is also available in the 'msecs'
attribute.
msecs: A double representing the msecs part of the record creation
(e.g., 624.5970726013184).
msg: Logging message containing formatting instructions or an arbitrary
object. This is the first argument of a log call.
args: A tuple containing the positional arguments for the logging call.
levelname: A string. Possible values are: INFO, WARNING, ERROR, etc.
exc_info: None or a 3-tuple with exception information as it is
returned by a call to sys.exc_info().
name: Logger's name. Most logging is done using the default root logger
and therefore the name will be 'root'.
filename: Basename of the file where logging occurred.
funcName: Name of the function where logging occurred.
process: The PID of the process running the worker.
thread: An id for the thread where the record was logged. This is not a
real TID (the one provided by OS) but rather the id (address) of a
Python thread object. Nevertheless having this value can allow to
filter log statement from only one specific thread.
"""
output = {}
output['timestamp'] = {
'seconds': int(record.created),
'nanos': int(record.msecs * 1000000)
}
# ERROR. INFO, DEBUG log levels translate into the same for severity
# property. WARNING becomes WARN.
output['severity'] = (record.levelname
if record.levelname != 'WARNING' else 'WARN')
# msg could be an arbitrary object, convert it to a string first.
record_msg = str(record.msg)
# Prepare the actual message using the message formatting string and the
# positional arguments as they have been used in the log call.
if record.args:
try:
output['message'] = record_msg % record.args
except (TypeError, ValueError):
output['message'] = '%s with args (%s)' % (record_msg,
record.args)
else:
output['message'] = record_msg
# The thread ID is logged as a combination of the process ID and thread ID
# since workers can run in multiple processes.
output['thread'] = '%s:%s' % (record.process, record.thread)
# job ID and worker ID. These do not change during the lifetime of a worker.
output['job'] = self.job_id
output['worker'] = self.worker_id
# Stage, step and work item ID come from thread local storage since they
# change with every new work item leased for execution. If there is no
# work item ID then we make sure the step is undefined too.
data = per_thread_worker_data.get_data()
if 'work_item_id' in data:
output['work'] = data['work_item_id']
tracker = statesampler.get_current_tracker()
if tracker:
output['stage'] = tracker.stage_name
if tracker.current_state() and tracker.current_state(
).name_context:
output['step'] = tracker.current_state(
).name_context.logging_name()
# All logging happens using the root logger. We will add the basename of the
# file and the function name where the logging happened to make it easier
# to identify who generated the record.
output['logger'] = '%s:%s:%s' % (record.name, record.filename,
record.funcName)
# Add exception information if any is available.
if record.exc_info:
output['exception'] = ''.join(
traceback.format_exception(*record.exc_info))
return json.dumps(output)
def format(self, record):
"""Returns a JSON string based on a LogRecord instance.
Args:
record: A LogRecord instance. See below for details.
Returns:
A JSON string representing the record.
A LogRecord instance has the following attributes and is used for
formatting the final message.
Attributes:
created: A double representing the timestamp for record creation
(e.g., 1438365207.624597). Note that the number contains also msecs and
microsecs information. Part of this is also available in the 'msecs'
attribute.
msecs: A double representing the msecs part of the record creation
(e.g., 624.5970726013184).
msg: Logging message containing formatting instructions or an arbitrary
object. This is the first argument of a log call.
args: A tuple containing the positional arguments for the logging call.
levelname: A string. Possible values are: INFO, WARNING, ERROR, etc.
exc_info: None or a 3-tuple with exception information as it is
returned by a call to sys.exc_info().
name: Logger's name. Most logging is done using the default root logger
and therefore the name will be 'root'.
filename: Basename of the file where logging occurred.
funcName: Name of the function where logging occurred.
process: The PID of the process running the worker.
thread: An id for the thread where the record was logged. This is not a
real TID (the one provided by OS) but rather the id (address) of a
Python thread object. Nevertheless having this value can allow to
filter log statement from only one specific thread.
"""
output = {}
output['timestamp'] = {
'seconds': int(record.created),
'nanos': int(record.msecs * 1000000)}
# ERROR. INFO, DEBUG log levels translate into the same for severity
# property. WARNING becomes WARN.
output['severity'] = (
record.levelname if record.levelname != 'WARNING' else 'WARN')
# msg could be an arbitrary object, convert it to a string first.
record_msg = str(record.msg)
# Prepare the actual message using the message formatting string and the
# positional arguments as they have been used in the log call.
if record.args:
try:
output['message'] = record_msg % record.args
except (TypeError, ValueError):
output['message'] = '%s with args (%s)' % (record_msg, record.args)
else:
output['message'] = record_msg
# The thread ID is logged as a combination of the process ID and thread ID
# since workers can run in multiple processes.
output['thread'] = '%s:%s' % (record.process, record.thread)
# job ID and worker ID. These do not change during the lifetime of a worker.
output['job'] = self.job_id
output['worker'] = self.worker_id
# Stage, step and work item ID come from thread local storage since they
# change with every new work item leased for execution. If there is no
# work item ID then we make sure the step is undefined too.
data = per_thread_worker_data.get_data()
if 'work_item_id' in data:
output['work'] = data['work_item_id']
tracker = statesampler.get_current_tracker()
if tracker:
output['stage'] = tracker.stage_name
if tracker.current_state() and tracker.current_state().name_context:
output['step'] = tracker.current_state().name_context.logging_name()
# All logging happens using the root logger. We will add the basename of the
# file and the function name where the logging happened to make it easier
# to identify who generated the record.
output['logger'] = '%s:%s:%s' % (
record.name, record.filename, record.funcName)
# Add exception information if any is available.
if record.exc_info:
output['exception'] = ''.join(
traceback.format_exception(*record.exc_info))
return json.dumps(output)
