def _Merge(self):
"""Orchestrates the merge operation."""
# Acquire op-lock for source user (should already have op-lock for target user).
op_lock = yield gen.Task(Lock.Acquire,
self._client,
LockResourceType.Operation,
str(self._source_user_id),
owner_id=self._op.operation_id)
try:
# If checkpoint exists, may skip past viewpoint merge phase.
state = self._op.checkpoint['state'] if self._op.checkpoint else 'vp'
if state == 'vp':
# Make target user a follower of all source user's viewpoints.
yield self._MergeViewpoints()
yield Operation.TriggerFailpoint(self._client)
self._op.checkpoint = None
else:
assert state == 'id', state
# Re-bind identities of the source user to the target user.
yield self._MergeIdentities()
yield Operation.TriggerFailpoint(self._client)
# Terminate the source user.
yield gen.Task(User.TerminateAccountOperation,
self._client,
user_id=self._source_user_id,
merged_with=self._target_user_id)
yield Operation.TriggerFailpoint(self._client)
finally:
yield gen.Task(op_lock.Release, self._client)
def _AddOpMethod2(client, callback):
"""Create operation with lower op id."""
op_dict = self._CreateTestOpDict(user_id=1, handler=_AddOpMethod3)
op_dict['operation_id'] = Operation.ConstructOperationId(1, 5)
Operation.CreateFromKeywords(**op_dict).Update(client, callback)
# Explicitly call Execute for this op, since otherwise the UserOpManager won't "see" it, because
# it has an op-id that is lower than the currently executing op-id.
user_op_mgr.Execute(op_dict['operation_id'])
def _AddOpMethod3(client, callback):
"""Create operation with lower op id."""
with util.Barrier(callback) as b:
op_dict = self._CreateTestOpDict(user_id=1, handler=self._OpMethod)
op_dict['operation_id'] = Operation.ConstructOperationId(1, 1)
Operation.CreateFromKeywords(**op_dict).Update(client, b.Callback())
# Try to acquire lock, which has side effect of incrementing "acquire_failures" and triggering requery.
Lock.TryAcquire(self._client, LockResourceType.Operation, '1', b.Callback())
def _CreateTestOp(self, user_id, handler, **kwargs):
Operation.ConstructOperationId(1, self._id)
self._id += 1
op_dict = self._CreateTestOpDict(user_id, handler, **kwargs)
op = Operation.CreateFromKeywords(**op_dict)
op.Update(self._client, self.stop)
self.wait()
return op
def testOperationReplay(self):
"""Verify that multiple applies for the same operation ID only increment the stats once."""
act = Accounting.CreateViewpointOwnedBy('vp1', 1)
act.num_photos = 1
# Manually set the current operation.
op = Operation(1, 'o1')
with EnterOpContext(op):
# First write for this entry.
self._RunAsync(Accounting.ApplyAccounting, self._client, act)
accounting = self._RunAsync(Accounting.Query, self._client, act.hash_key, act.sort_key, None)
assert accounting.StatsEqual(act)
ids = accounting.op_ids.split(',')
assert len(ids) == 1, 'len(op_ids) == %d' % len(ids)
assert op.operation_id in ids
assert accounting.num_photos == 1, 'num_photos: %d' % accounting.num_photos
# Apply the same operation.
self._RunAsync(Accounting.ApplyAccounting, self._client, act)
accounting = self._RunAsync(Accounting.Query, self._client, act.hash_key, act.sort_key, None)
assert accounting.StatsEqual(act)
ids = accounting.op_ids.split(',')
assert len(ids) == 1, 'len(op_ids) == %d' % len(ids)
assert op.operation_id in ids
assert accounting.num_photos == 1, 'num_photos: %d' % accounting.num_photos
# New operation.
op = Operation(1, 'o2')
with EnterOpContext(op):
self._RunAsync(Accounting.ApplyAccounting, self._client, act)
accounting = self._RunAsync(Accounting.Query, self._client, act.hash_key, act.sort_key, None)
ids = accounting.op_ids.split(',')
assert len(ids) == 2, 'len(op_ids) == %d' % len(ids)
assert op.operation_id in ids
assert accounting.num_photos == 2, 'num_photos: %d' % accounting.num_photos
# Simulate a "repair missing accounting entry" by dbchk. This means that the stats will be there,
# but the op_ids field will be None.
accounting.op_ids = None
self._RunAsync(accounting.Update, self._client)
# New operation.
op = Operation(1, 'o3')
with EnterOpContext(op):
self._RunAsync(Accounting.ApplyAccounting, self._client, act)
accounting = self._RunAsync(Accounting.Query, self._client, act.hash_key, act.sort_key, None)
ids = accounting.op_ids.split(',')
# op_ids is back to a size of 1.
assert len(ids) == 1, 'len(op_ids) == %d' % len(ids)
assert op.operation_id in ids
assert accounting.num_photos == 3, 'num_photos: %d' % accounting.num_photos
self.stop()
def testCreateAndUpdate(self):
"""Creates a episode with id pre-allocated on mobile device. Then updates the episode."""
with EnterOpContext(Operation(1, 'o1')):
timestamp = time.time()
episode_id = Episode.ConstructEpisodeId(timestamp,
self._mobile_dev.device_id,
15)
ep_dict = {
'user_id': self._user.user_id,
'episode_id': episode_id,
'viewpoint_id': self._user.private_vp_id,
'timestamp': time.time(),
'publish_timestamp': time.time(),
'description': 'yada yada this is a episode',
'title': 'Episode #1'
}
episode = self._RunAsync(Episode.CreateNew, self._client,
**ep_dict)
episode._version = None
self.assertEqual(ep_dict, episode._asdict())
update_dict = {
'episode_id': episode.episode_id,
'user_id': episode.user_id,
'description': 'updated description',
'title': 'Episode #1a'
}
self._RunAsync(episode.UpdateExisting, self._client, **update_dict)
episode._version = None
ep_dict.update(update_dict)
self.assertEqual(ep_dict, episode._asdict())
def _ResolveContacts(self, contact_dicts, contact_ids, reason=None):
"""Creates a prospective user account for any contacts that are not yet Viewfinder users.
The "contact_ids" list should have been previously obtained by the caller via a call to
_ResolveContactIds, and items in it must correspond to "contact_dicts".
If specified, the "reason" string is passed to the CreateProspective op. This describes what caused the user
to be created (see db/analytics.py for payload details).
"""
for contact_dict, (user_exists, user_id,
webapp_dev_id) in zip(contact_dicts, contact_ids):
if not user_exists:
# Check if previous invocation of this operation already created the user.
user = yield gen.Task(User.Query,
self._client,
user_id,
None,
must_exist=False)
if user is None:
# Create prospective user.
request = {
'user_id': user_id,
'webapp_dev_id': webapp_dev_id,
'identity_key': contact_dict['identity'],
'reason': reason
}
yield Operation.CreateNested(
self._client, 'CreateProspectiveOperation.Execute',
request)
def AcquireLock(cls, client, viewpoint_id, callback):
"""Acquires a persistent global lock on the specified viewpoint."""
op = Operation.GetCurrent()
lock = yield gen.Task(Lock.Acquire, client, LockResourceType.Viewpoint,
viewpoint_id, op.operation_id)
ViewpointLockTracker.AddViewpointId(viewpoint_id)
callback(lock)
def _CreateProspective(self):
"""Create the prospective user and identity."""
self._new_user, _ = yield User.CreateProspective(self._client,
self._new_user_id,
self._webapp_dev_id,
self._identity_key,
self._op.timestamp)
# If system user is defined, then create the welcome conversation.
# For now, add a check to ensure the welcome conversation is not created in production.
if system_users.NARRATOR_USER is not None:
# Checkpoint the allocated asset id range used to create the welcome conversation.
if self._op.checkpoint is None:
# NOTE: Asset ids are allocated from the new user's ids. This is different than the
# usual practice of allocating from the sharer's ids.
self._unique_id_start = yield gen.Task(User.AllocateAssetIds,
self._client,
self._new_user_id,
CreateProspectiveOperation._ASSET_ID_COUNT)
checkpoint = {'id': self._unique_id_start}
yield self._op.SetCheckpoint(self._client, checkpoint)
else:
self._unique_id_start = self._op.checkpoint['id']
yield self._CreateWelcomeConversation()
# Add an analytics entry for this user.
analytics = Analytics.Create(entity='us:%d' % self._new_user_id,
type=Analytics.USER_CREATE_PROSPECTIVE,
timestamp=self._op.timestamp,
payload=self._reason)
yield gen.Task(analytics.Update, self._client)
yield Operation.TriggerFailpoint(self._client)
def _InsertDeleteContact(contact_to_insert, contact_to_delete):
"""Insert followed by delete (after insert is complete)."""
if contact_to_insert is not None:
yield gen.Task(contact_to_insert.Update, self._client)
yield Operation.TriggerFailpoint(self._client)
if contact_to_delete is not None:
yield gen.Task(contact_to_delete.Delete, self._client)
def TerminateAccountOperation(cls, client, user_id, merged_with=None):
"""Invokes User.TerminateAccount via operation execution."""
@gen.coroutine
def _VisitIdentity(identity_key):
"""Unlink this identity from the user."""
yield Identity.UnlinkIdentityOperation(client, user_id, identity_key.hash_key)
# Turn off alerts to all devices owned by the user.
yield gen.Task(Device.MuteAlerts, client, user_id)
# Unlink every identity attached to the user.
query_expr = ('identity.user_id={id}', {'id': user_id})
yield gen.Task(Identity.VisitIndexKeys, client, query_expr, _VisitIdentity)
# Add an analytics entry for this user.
timestamp = Operation.GetCurrent().timestamp
payload = 'terminate' if merged_with is None else 'merge=%s' % merged_with
analytics = Analytics.Create(entity='us:%d' % user_id,
type=Analytics.USER_TERMINATE,
timestamp=timestamp,
payload=payload)
yield gen.Task(analytics.Update, client)
# Terminate the user account.
yield gen.Task(User.TerminateAccount, client, user_id, merged_with=merged_with)
# Notify all friends that this user account has been terminated.
yield NotificationManager.NotifyTerminateAccount(client, user_id)
def _UpdateEpisode(self):
"""Orchestrates the update_episode operation by executing each of the phases in turn."""
# Get the viewpoint_id from the episode (which must exist).
self._episode = yield gen.Task(Episode.Query,
self._client,
self._episode_id,
None,
must_exist=False)
if not self._episode:
raise InvalidRequestError(
'Episode "%s" does not exist and so cannot be updated.' %
self._episode_id)
self._viewpoint_id = self._episode.viewpoint_id
lock = yield gen.Task(Viewpoint.AcquireLock, self._client,
self._viewpoint_id)
try:
yield self._Check()
self._client.CheckDBNotModified()
yield self._Update()
yield self._Account()
yield Operation.TriggerFailpoint(self._client)
yield self._Notify()
finally:
yield gen.Task(Viewpoint.ReleaseLock, self._client,
self._viewpoint_id, lock)
def _CreateBadOperation(self, user_id, device_id, callback):
"""Creates a photo share for a photo which doesn't exist."""
request = {
'user_id':
user_id,
'activity': {
'activity_id': 'a123',
'timestamp': time.time()
},
'viewpoint': {
'viewpoint_id': Viewpoint.ConstructViewpointId(100, 100),
'type': Viewpoint.EVENT
},
'episodes': [{
'existing_episode_id': 'eg8QVrk3S',
'new_episode_id': 'eg8QVrk3T',
'timestamp': time.time(),
'photo_ids': ['pg8QVrk3S']
}],
'contacts': [{
'identity': 'Local:testing1',
'name': 'Peter Mattis'
}]
}
Operation.CreateAndExecute(self._client, user_id, device_id,
'ShareNewOperation.Execute', request,
callback)
def _RemoveContacts(self):
"""Orchestrates the remove contacts operation by executing each of the phases in turn."""
yield self._Check()
self._client.CheckDBNotModified()
yield self._Update()
# No accounting for this operation.
yield Operation.TriggerFailpoint(self._client)
yield self._Notify()
def _FetchContactsOp(self):
"""Orchestrates the fetch contacts operation by executing each of the phases in turn."""
yield self._Check()
self._client.CheckDBNotModified()
if self._do_fetch_and_update:
yield self._Update()
yield Operation.TriggerFailpoint(self._client)
yield self._Notify()
def _ReplaceRemovedContact(contact_dict_to_insert,
removed_contact_to_delete):
contact_to_insert = Contact.CreateFromKeywords(
**contact_dict_to_insert)
yield gen.Task(contact_to_insert.Update, self._client)
if removed_contact_to_delete is not None:
yield Operation.TriggerFailpoint(self._client)
yield gen.Task(removed_contact_to_delete.Delete, self._client)
def UpdateOperation(cls, client, callback, user_dict, settings_dict):
"""Invokes User.Update via operation execution."""
yield gen.Task(User.UpdateWithSettings, client, user_dict, settings_dict)
timestamp = Operation.GetCurrent().timestamp
yield NotificationManager.NotifyUpdateUser(client, user_dict, settings_dict, timestamp)
callback()
def _OnSecondUploadOp(orig_op, op):
"""Wait for the second operation and on completion, query the
original operation which is still failing. It should have a retry.
"""
Operation.WaitForOp(
self._client, op.user_id, op.operation_id,
partial(Operation.Query, self._client, orig_op.user_id,
orig_op.operation_id, None, _OnQueryOrigOp))
def MigrateForward(self, client, message, callback):
from viewfinder.backend.db.operation import Operation
def _OnAllocateId(id):
message.dict['headers']['op_id'] = id
message.dict['headers']['op_timestamp'] = time.time()
callback()
Operation.AllocateSystemOperationId(client, _OnAllocateId)
def testDuplicateOperations(self):
"""Verify that inserting duplicate operations is a no-op."""
op_id = Operation.ConstructOperationId(self._mobile_dev.device_id, 100)
with util.Barrier(self.stop) as b:
for i in xrange(10):
Operation.CreateAndExecute(
self._client, self._user.user_id,
self._mobile_dev.device_id, 'HidePhotosOperation.Execute',
{
'headers': {
'op_id': op_id,
'op_timestamp': time.time(),
'synchronous': True
},
'user_id': self._user.user_id,
'episodes': []
}, b.Callback())
def _RemoveContact(contact_to_remove):
"""Insert a 'removed' contact with the same contact_id as the one being removed and then
delete the actual contact that's being removed."""
removed_contact = Contact.CreateRemovedContact(
contact_to_remove.user_id, contact_to_remove.contact_id,
self._notify_timestamp)
yield gen.Task(removed_contact.Update, self._client)
yield Operation.TriggerFailpoint(self._client)
yield gen.Task(contact_to_remove.Delete, self._client)
def UnlinkIdentityOperation(cls, client, user_id, identity):
"""Unlinks the specified identity from any associated viewfinder user."""
# All contacts created during UnlinkIdentity are based on the current operation's timestamp.
timestamp = Operation.GetCurrent().timestamp
yield Identity.UnlinkIdentity(client, user_id, identity, timestamp)
# Notify clients of any contacts that have been updated.
yield NotificationManager.NotifyUnlinkIdentity(client, user_id,
identity, timestamp)
def _InnerMethod(client, arg1, arg2):
self._method_count += 1
self.assertEqual(arg1, 1)
self.assertEqual(arg2, 'hello')
self.assertEqual(self._method_count, 2)
# Assert that nested operation is derived from parent op.
inner_op = Operation.GetCurrent()
self.assertEqual(inner_op.user_id, outer_op.user_id)
self.assertEqual(inner_op.timestamp, outer_op.timestamp)
self.assertEqual(inner_op.operation_id, '+%s' % outer_op.operation_id)
def _GetInstance(cls):
"""Ensures that a viewpoint lock tracker has been created and attached to the current
operation.
"""
op = Operation.GetCurrent()
lock_tracker = op.context.get('viewpoint_lock_tracker')
if lock_tracker is None:
lock_tracker = ViewpointLockTracker()
op.context['viewpoint_lock_tracker'] = lock_tracker
return lock_tracker
def _PostComment(self):
"""Orchestrates the post_comment operation by executing each of the phases in turn."""
lock = yield gen.Task(Viewpoint.AcquireLock, self._client, self._viewpoint_id)
try:
if not (yield self._Check()):
return
self._client.CheckDBNotModified()
yield self._Update()
yield self._Account()
yield Operation.TriggerFailpoint(self._client)
yield self._Notify()
finally:
yield gen.Task(Viewpoint.ReleaseLock, self._client, self._viewpoint_id, lock)
def _UpdateFollower(self):
"""Orchestrates the update follower operation by executing each of the phases in turn."""
lock = yield gen.Task(Viewpoint.AcquireLock, self._client,
self._viewpoint_id)
try:
yield self._Check()
self._client.CheckDBNotModified()
yield self._Update()
yield Operation.TriggerFailpoint(self._client)
yield self._Notify()
finally:
yield gen.Task(Viewpoint.ReleaseLock, self._client,
self._viewpoint_id, lock)
def _CreateTestOpDict(self, user_id, handler, **kwargs):
op_id = Operation.ConstructOperationId(1, self._id)
self._id += 1
op_dict = {'user_id': user_id,
'operation_id': op_id,
'device_id': 1,
'method': handler.__name__,
'json': json.dumps(kwargs, indent=True),
'timestamp': time.time(),
'attempts': 0}
return op_dict
def _UploadEpisode(self):
"""Orchestrates the upload_episode operation by executing each of the phases in turn."""
# Lock the viewpoint while sharing into it.
lock = yield gen.Task(Viewpoint.AcquireLock, self._client, self._user.private_vp_id)
try:
yield self._Check()
self._client.CheckDBNotModified()
yield self._Update()
yield self._Account()
yield Operation.TriggerFailpoint(self._client)
yield self._Notify()
finally:
yield gen.Task(Viewpoint.ReleaseLock, self._client, self._user.private_vp_id, lock)
def _CreateBlockedOperation(self, user_id, device_id, callback):
"""Creates a photo share after locking the viewpoint so that the operation will fail and get retried."""
photo_id = Photo.ConstructPhotoId(time.time(), device_id, 123)
self._RunAsync(self._UploadPhotoOperation,
user_id,
device_id,
1,
photo_id=photo_id)
self._RunAsync(Lock.Acquire, self._client, LockResourceType.Viewpoint,
'vp123', Operation.ConstructOperationId(device_id, 123))
request = {
'user_id':
user_id,
'activity': {
'activity_id': 'a123',
'timestamp': time.time()
},
'viewpoint': {
'viewpoint_id': 'vp123',
'type': Viewpoint.EVENT
},
'episodes': [{
'existing_episode_id': 'eg8QVrk3S',
'new_episode_id': 'eg8QVrk3T',
'timestamp': time.time(),
'photo_ids': [photo_id]
}],
'contacts': [{
'identity': 'Local:testing1',
'name': 'Peter Mattis'
}]
}
Operation.CreateAndExecute(self._client, user_id, device_id,
'ShareNewOperation.Execute', request,
callback)
def _HidePhotos(self):
"""Orchestrates the hide photos operation by executing each of the phases in turn."""
lock = yield gen.Task(Viewpoint.AcquireLock, self._client,
self._user.private_vp_id)
try:
yield self._Check()
self._client.CheckDBNotModified()
yield self._Update()
# No accounting changes for hide_photos.
yield Operation.TriggerFailpoint(self._client)
yield self._Notify()
finally:
yield gen.Task(Viewpoint.ReleaseLock, self._client,
self._user.private_vp_id, lock)