def store(self, updated_by=None):
"""Stores a new version of the config entity."""
# Create an incomplete key, to be completed by 'store_new_version'.
self.key = ndb.Key(self.__class__, None, parent=self._get_root_key())
self.updated_by = updated_by or auth.get_current_identity()
self.updated_ts = utils.utcnow()
return datastore_utils.store_new_version(self, self._get_root_model())
def store(self, updated_by=None): """Stores a new version of the config entity.""" # Create an incomplete key, to be completed by 'store_new_version'. self.key = ndb.Key(self.__class__, None, parent=self._get_root_key()) self.updated_by = updated_by or auth.get_current_identity() self.updated_ts = utils.utcnow() return datastore_utils.store_new_version(self, self._get_root_model())
def store(self, name):
"""Stores a new version of the instance."""
# Create an incomplete key.
self.key = ndb.Key(self.__class__,
None,
parent=self._gen_root_key(name))
self.who = auth.get_current_identity()
return datastore_utils.store_new_version(self, self.ROOT_MODEL)
def bot_event(event_type, bot_id, external_ip, authenticated_as, dimensions,
state, version, quarantined, task_id, task_name, **kwargs):
"""Records when a bot has queried for work.
Arguments:
- event: event type.
- bot_id: bot id.
- external_ip: IP address as seen by the HTTP handler.
- authenticated_as: bot identity as seen by the HTTP handler.
- dimensions: Bot's dimensions as self-reported. If not provided, keep
previous value.
- state: ephemeral state of the bot. It is expected to change constantly. If
not provided, keep previous value.
- version: swarming_bot.zip version as self-reported. Used to spot if a bot
failed to update promptly. If not provided, keep previous value.
- quarantined: bool to determine if the bot was declared quarantined.
- task_id: packed task id if relevant. Set to '' to zap the stored value.
- task_name: task name if relevant. Zapped when task_id is zapped.
- kwargs: optional values to add to BotEvent relevant to event_type.
- lease_id (in kwargs): ID assigned by Machine Provider for this bot.
- lease_expiration_ts (in kwargs): UTC seconds from epoch when Machine
Provider lease expires.
"""
if not bot_id:
return
# Retrieve the previous BotInfo and update it.
info_key = get_info_key(bot_id)
bot_info = info_key.get() or BotInfo(key=info_key)
bot_info.last_seen_ts = utils.utcnow()
bot_info.external_ip = external_ip
bot_info.authenticated_as = authenticated_as
if dimensions:
bot_info.dimensions_flat = dimensions_to_flat(dimensions)
if state:
bot_info.state = state
if quarantined is not None:
bot_info.quarantined = quarantined
if task_id is not None:
bot_info.task_id = task_id
if task_name:
bot_info.task_name = task_name
if version is not None:
bot_info.version = version
if kwargs.get('lease_id') is not None:
bot_info.lease_id = kwargs['lease_id']
if kwargs.get('lease_expiration_ts') is not None:
bot_info.lease_expiration_ts = kwargs['lease_expiration_ts']
if event_type in ('request_sleep', 'task_update'):
# Handle this specifically. It's not much of an even worth saving a BotEvent
# for but it's worth updating BotInfo. The only reason BotInfo is GET is to
# keep first_seen_ts. It's not necessary to use a transaction here since no
# BotEvent is being added, only last_seen_ts is really updated.
bot_info.put()
return
event = BotEvent(parent=get_root_key(bot_id),
event_type=event_type,
external_ip=external_ip,
authenticated_as=authenticated_as,
dimensions_flat=bot_info.dimensions_flat,
quarantined=bot_info.quarantined,
state=bot_info.state,
task_id=bot_info.task_id,
version=bot_info.version,
**kwargs)
if event_type in ('task_canceled', 'task_completed', 'task_error'):
# Special case to keep the task_id in the event but not in the summary.
bot_info.task_id = ''
datastore_utils.store_new_version(event, BotRoot, [bot_info])
def bot_event(
event_type, bot_id, external_ip, dimensions, state, version, quarantined,
task_id, task_name, **kwargs):
"""Records when a bot has queried for work.
Arguments:
- event: event type.
- bot_id: bot id.
- external_ip: IP address as seen by the HTTP handler.
- dimensions: Bot's dimensions as self-reported. If not provided, keep
previous value.
- state: ephemeral state of the bot. It is expected to change constantly. If
not provided, keep previous value.
- version: swarming_bot.zip version as self-reported. Used to spot if a bot
failed to update promptly. If not provided, keep previous value.
- quarantined: bool to determine if the bot was declared quarantined.
- task_id: packed task id if relevant. Set to '' to zap the stored value.
- task_name: task name if relevant. Zapped when task_id is zapped.
- kwargs: optional values to add to BotEvent relevant to event_type.
"""
if not bot_id:
return
# Retrieve the previous BotInfo and update it.
info_key = get_info_key(bot_id)
bot_info = info_key.get() or BotInfo(key=info_key)
bot_info.last_seen_ts = utils.utcnow()
bot_info.external_ip = external_ip
if dimensions:
bot_info.dimensions = dimensions
if state:
bot_info.state = state
if quarantined is not None:
bot_info.quarantined = quarantined
if task_id is not None:
bot_info.task_id = task_id
if task_name:
bot_info.task_name = task_name
if version is not None:
bot_info.version = version
if event_type in ('request_sleep', 'task_update'):
# Handle this specifically. It's not much of an even worth saving a BotEvent
# for but it's worth updating BotInfo. The only reason BotInfo is GET is to
# keep first_seen_ts. It's not necessary to use a transaction here since no
# BotEvent is being added, only last_seen_ts is really updated.
bot_info.put()
return
event = BotEvent(
parent=get_root_key(bot_id),
event_type=event_type,
external_ip=external_ip,
dimensions=bot_info.dimensions,
quarantined=bot_info.quarantined,
state=bot_info.state,
task_id=bot_info.task_id,
version=bot_info.version,
**kwargs)
if event_type in ('task_completed', 'task_error'):
# Special case to keep the task_id in the event but not in the summary.
bot_info.task_id = ''
datastore_utils.store_new_version(event, BotRoot, [bot_info])
def store(self, name): """Stores a new version of the instance.""" # Create an incomplete key. self.key = ndb.Key(self.__class__, None, parent=self._gen_root_key(name)) self.who = auth.get_current_identity() return datastore_utils.store_new_version(self, self.ROOT_MODEL)
def bot_event(event_type, bot_id, external_ip, authenticated_as, dimensions,
state, version, quarantined, maintenance_msg, task_id, task_name,
**kwargs):
"""Records when a bot has queried for work.
The sheer fact this event is happening means the bot is alive (not dead), so
this is good. It may be quarantined though, and in this case, it will be
evicted from the task queues.
If it's declaring maintenance, it will not be evicted from the task queues, as
maintenance is supposed to be temporary and expected to complete within a
reasonable time frame.
Arguments:
- event_type: event type, one of BotEvent.ALLOWED_EVENTS.
- bot_id: bot id.
- external_ip: IP address as seen by the HTTP handler.
- authenticated_as: bot identity as seen by the HTTP handler.
- dimensions: Bot's dimensions as self-reported. If not provided, keep
previous value.
- state: ephemeral state of the bot. It is expected to change constantly. If
not provided, keep previous value.
- version: swarming_bot.zip version as self-reported. Used to spot if a bot
failed to update promptly. If not provided, keep previous value.
- quarantined: bool to determine if the bot was declared quarantined.
- maintenance_msg: string describing why the bot is in maintenance.
- task_id: packed task id if relevant. Set to '' to zap the stored value.
- task_name: task name if relevant. Zapped when task_id is zapped.
- kwargs: optional values to add to BotEvent relevant to event_type.
Returns:
ndb.Key to BotEvent entity if one was added.
"""
if not bot_id:
return
# Retrieve the previous BotInfo and update it.
info_key = get_info_key(bot_id)
bot_info = info_key.get()
if not bot_info:
bot_info = BotInfo(key=info_key)
now = utils.utcnow()
bot_info.last_seen_ts = now
bot_info.external_ip = external_ip
bot_info.authenticated_as = authenticated_as
bot_info.maintenance_msg = maintenance_msg
if dimensions:
bot_info.dimensions_flat = task_queues.dimensions_to_flat(dimensions)
if state:
bot_info.state = state
if quarantined is not None:
bot_info.quarantined = quarantined
if task_id is not None:
bot_info.task_id = task_id
if task_name:
bot_info.task_name = task_name
if version is not None:
bot_info.version = version
if quarantined:
# Make sure it is not in the queue since it can't reap anything.
task_queues.cleanup_after_bot(info_key.parent())
try:
if event_type in ('request_sleep', 'task_update'):
# Handle this specifically. It's not much of an even worth saving a
# BotEvent for but it's worth updating BotInfo. The only reason BotInfo is
# GET is to keep first_seen_ts. It's not necessary to use a transaction
# here since no BotEvent is being added, only last_seen_ts is really
# updated.
bot_info.put()
return
event = BotEvent(parent=get_root_key(bot_id),
event_type=event_type,
external_ip=external_ip,
authenticated_as=authenticated_as,
dimensions_flat=bot_info.dimensions_flat,
quarantined=bot_info.quarantined,
maintenance_msg=bot_info.maintenance_msg,
state=bot_info.state,
task_id=bot_info.task_id,
version=bot_info.version,
**kwargs)
if event_type in ('task_completed', 'task_error', 'task_killed'):
# Special case to keep the task_id in the event but not in the summary.
bot_info.task_id = ''
datastore_utils.store_new_version(event, BotRoot, [bot_info])
return event.key
finally:
# Store the event in memcache to accelerate monitoring.
# key is at minute resolution, because that's the monitoring precision.
key = '%s:%s' % (bot_id, now.strftime('%Y-%m-%dT%H:%M'))
m = memcache.Client()
while True:
data = [event_type, now.second]
if m.add(key, data, time=3600, namespace='BotEvents'):
break
prev_val = m.get(key, for_cas=True, namespace='BotEvents')
if prev_val is None:
continue
data = prev_val + [event_type, now.second]
# Keep the data for one hour. If the cron job cannot reap it within 1h,
# it's probably broken.
if m.cas(key, data, time=3600, namespace='BotEvents'):
break
def bot_event(event_type, bot_id, external_ip, authenticated_as, dimensions,
state, version, quarantined, maintenance_msg, task_id, task_name,
**kwargs):
"""Records when a bot has queried for work.
The sheer fact this event is happening means the bot is alive (not dead), so
this is good. It may be quarantined though, and in this case, it will be
evicted from the task queues.
If it's declaring maintenance, it will not be evicted from the task queues, as
maintenance is supposed to be temporary and expected to complete within a
reasonable time frame.
Arguments:
- event_type: event type, one of BotEvent.ALLOWED_EVENTS.
- bot_id: bot id.
- external_ip: IP address as seen by the HTTP handler.
- authenticated_as: bot identity as seen by the HTTP handler.
- dimensions: Bot's dimensions as self-reported. If not provided, keep
previous value.
- state: ephemeral state of the bot. It is expected to change constantly. If
not provided, keep previous value.
- version: swarming_bot.zip version as self-reported. Used to spot if a bot
failed to update promptly. If not provided, keep previous value.
- quarantined: bool to determine if the bot was declared quarantined.
- maintenance_msg: string describing why the bot is in maintenance.
- task_id: packed task id if relevant. Set to '' to zap the stored value.
- task_name: task name if relevant. Zapped when task_id is zapped.
- kwargs: optional values to add to BotEvent relevant to event_type.
- lease_id (in kwargs): ID assigned by Machine Provider for this bot.
- lease_expiration_ts (in kwargs): UTC seconds from epoch when Machine
Provider lease expires.
- machine_type (in kwargs): ID of the lease_management.MachineType this
Machine Provider bot was leased for.
- machine_lease (in kwargs): ID of the lease_management.MachineType
corresponding to this bot.
"""
if not bot_id:
return
# Retrieve the previous BotInfo and update it.
info_key = get_info_key(bot_id)
bot_info = info_key.get()
if not bot_info:
bot_info = BotInfo(key=info_key)
bot_info.last_seen_ts = utils.utcnow()
bot_info.external_ip = external_ip
bot_info.authenticated_as = authenticated_as
bot_info.maintenance_msg = maintenance_msg
if dimensions:
bot_info.dimensions_flat = task_queues.dimensions_to_flat(dimensions)
if state:
bot_info.state = state
if quarantined is not None:
bot_info.quarantined = quarantined
if task_id is not None:
bot_info.task_id = task_id
if task_name:
bot_info.task_name = task_name
if version is not None:
bot_info.version = version
if kwargs.get('lease_id') is not None:
bot_info.lease_id = kwargs['lease_id']
if kwargs.get('lease_expiration_ts') is not None:
bot_info.lease_expiration_ts = kwargs['lease_expiration_ts']
if kwargs.get('machine_type') is not None:
bot_info.machine_type = kwargs['machine_type']
if kwargs.get('machine_lease') is not None:
bot_info.machine_lease = kwargs['machine_lease']
if quarantined:
# Make sure it is not in the queue since it can't reap anything.
task_queues.cleanup_after_bot(info_key.parent())
if event_type in ('request_sleep', 'task_update'):
# Handle this specifically. It's not much of an even worth saving a BotEvent
# for but it's worth updating BotInfo. The only reason BotInfo is GET is to
# keep first_seen_ts. It's not necessary to use a transaction here since no
# BotEvent is being added, only last_seen_ts is really updated.
bot_info.put()
return
event = BotEvent(parent=get_root_key(bot_id),
event_type=event_type,
external_ip=external_ip,
authenticated_as=authenticated_as,
dimensions_flat=bot_info.dimensions_flat,
quarantined=bot_info.quarantined,
maintenance_msg=bot_info.maintenance_msg,
state=bot_info.state,
task_id=bot_info.task_id,
version=bot_info.version,
**kwargs)
if event_type in ('task_completed', 'task_error', 'task_killed'):
# Special case to keep the task_id in the event but not in the summary.
bot_info.task_id = ''
datastore_utils.store_new_version(event, BotRoot, [bot_info])
def bot_event(event_type, bot_id, external_ip, authenticated_as, dimensions,
state, version, quarantined, maintenance_msg, task_id, task_name,
register_dimensions, **kwargs):
"""Records when a bot has queried for work.
This event happening usually means the bot is alive (not dead), except for
'bot_missing' event which is created by server. It may be quarantined, and
in this case, it will be evicted from the task queues.
If it's declaring maintenance, it will not be evicted from the task queues, as
maintenance is supposed to be temporary and expected to complete within a
reasonable time frame.
Arguments:
- event_type: event type, one of BotEvent.ALLOWED_EVENTS.
- bot_id: bot id.
- external_ip: IP address as seen by the HTTP handler.
- authenticated_as: bot identity as seen by the HTTP handler.
- dimensions: Bot's dimensions as self-reported. If not provided, keep
previous value.
- state: ephemeral state of the bot. It is expected to change constantly. If
not provided, keep previous value.
- version: swarming_bot.zip version as self-reported. Used to spot if a bot
failed to update promptly. If not provided, keep previous value.
- quarantined: bool to determine if the bot was declared quarantined.
- maintenance_msg: string describing why the bot is in maintenance.
- task_id: packed task id if relevant. Set to '' to zap the stored value.
- task_name: task name if relevant. Zapped when task_id is zapped.
- register_dimensions: bool to specify whether to register dimensions to
BotInfo.
- kwargs: optional values to add to BotEvent relevant to event_type.
Returns:
ndb.Key to BotEvent entity if one was added.
"""
if not bot_id:
return
# Retrieve the previous BotInfo and update it.
info_key = get_info_key(bot_id)
bot_info = info_key.get()
if not bot_info:
bot_info = BotInfo(key=info_key)
# Register only id and pool dimensions at the first handshake.
dimensions_flat = task_queues.bot_dimensions_to_flat(dimensions)
bot_info.dimensions_flat = [
d for d in dimensions_flat
if d.startswith('id:') or d.startswith('pool:')
]
now = utils.utcnow()
# bot_missing event is created by a server, not a bot.
# So it shouldn't update last_seen_ts, external_ip, authenticated_as,
# maintenance_msg.
# If the last_seen_ts gets updated, it would change the bot composite
# to alive. And if it clears maintenance_msg, it would change the composite
# to NOT_IN_MAINTENANCE and lose the message.
if event_type != 'bot_missing':
bot_info.last_seen_ts = now
bot_info.external_ip = external_ip
bot_info.authenticated_as = authenticated_as
bot_info.maintenance_msg = maintenance_msg
dimensions_updated = False
dimensions_flat = []
if dimensions:
dimensions_flat = task_queues.bot_dimensions_to_flat(dimensions)
if register_dimensions and bot_info.dimensions_flat != dimensions_flat:
logging.debug('bot_event: Updating dimensions. from: %s, to: %s',
bot_info.dimensions_flat, dimensions_flat)
bot_info.dimensions_flat = dimensions_flat
dimensions_updated = True
if state:
bot_info.state = state
if quarantined is not None:
bot_info.quarantined = quarantined
if task_id is not None:
bot_info.task_id = task_id
# Remove the task from the BotInfo summary in the following cases
# 1) When the task finishes (event_type=task_XXX)
# In these cases, the BotEvent shall have the task
# since the event still refers to it
# 2) When the bot is pooling (event_type=request_sleep)
# The bot has already finished the previous task.
# But it could have forgotten to remove the task from the BotInfo.
# So ensure the task is removed.
# 3) When the bot is missing
# We assume it can't process assigned task anymore.
if event_type in ('task_completed', 'task_error', 'task_killed',
'request_sleep', 'bot_missing'):
bot_info.task_id = None
bot_info.task_name = None
if task_name:
bot_info.task_name = task_name
if version is not None:
bot_info.version = version
if quarantined:
# Make sure it is not in the queue since it can't reap anything.
task_queues.cleanup_after_bot(info_key.parent())
try:
# Decide whether saving the event.
# It's not much of an even worth saving a BotEvent for but it's worth
# updating BotInfo. The only reason BotInfo is GET is to keep first_seen_ts.
# It's not necessary to use a transaction here since no BotEvent is being
# added, only last_seen_ts is really updated.
# crbug.com/1015365: It's useful saving BotEvent when dimensions updates.
# crbug.com/952984: It needs to save BotEvent when quarantined.
skip_save_event = (not dimensions_updated and not quarantined
and event_type in ('request_sleep', 'task_update'))
if skip_save_event:
bot_info.put()
return
# When it's a 'bot_*' or 'request_*' event, use the dimensions provided
# by the bot.
# When it's a 'task_*' event, use BotInfo.dimensios_flat since dimensions
# aren't provided by the bot.
event_dimensions_flat = dimensions_flat or bot_info.dimensions_flat
event = BotEvent(parent=get_root_key(bot_id),
event_type=event_type,
external_ip=external_ip,
authenticated_as=authenticated_as,
dimensions_flat=event_dimensions_flat,
quarantined=bot_info.quarantined,
maintenance_msg=bot_info.maintenance_msg,
state=bot_info.state,
task_id=task_id or bot_info.task_id,
version=bot_info.version,
**kwargs)
datastore_utils.store_new_version(event, BotRoot, [bot_info])
return event.key
finally:
# Store the event in memcache to accelerate monitoring.
# key is at minute resolution, because that's the monitoring precision.
key = '%s:%s' % (bot_id, now.strftime('%Y-%m-%dT%H:%M'))
m = memcache.Client()
while True:
data = [event_type, now.second]
if m.add(key, data, time=3600, namespace='BotEvents'):
break
prev_val = m.get(key, for_cas=True, namespace='BotEvents')
if prev_val is None:
continue
data = prev_val + [event_type, now.second]
# Keep the data for one hour. If the cron job cannot reap it within 1h,
# it's probably broken.
if m.cas(key, data, time=3600, namespace='BotEvents'):
break
