def Upgrade(callback):
"""Upgrades each table in 'options.options.tables'."""
if options.options.verbose_test_run:
logging.info(
'***** NOTE: upgrade is being run in verbose testing mode; run with '
'--verbose_test_run=False once changes have been verified')
Version.SetMutateItems(False)
if options.options.allow_s3_queries is not None:
logging.info('Setting allow_s3_queries=%r' %
options.options.allow_s3_queries)
Version.SetAllowS3Queries(options.options.allow_s3_queries)
OpManager.SetInstance(
OpManager(op_map=DB_OPERATION_MAP,
client=db_client.DBClient.Instance()))
tables = [vf_schema.SCHEMA.GetTable(n) for n in options.options.tables]
if not tables:
raise Exception(
'The --tables option has not been specified. ' +
'Tables to upgrade must be explicitly listed using this option.')
yield [
gen.Task(UpgradeTable, db_client.DBClient.Instance(), table)
for table in tables
]
callback()
def _StartWWW(run_callback, scan_ops):
"""Starts services necessary for operating in the Viewfinder WWW server environment. Invokes
'run_callback' asynchronously.
"""
client = db_client.DBClient.Instance()
# Log emails and texts to the console in local mode.
if options.options.local_services:
EmailManager.SetInstance(LoggingEmailManager())
SMSManager.SetInstance(LoggingSMSManager())
else:
EmailManager.SetInstance(SendGridEmailManager())
SMSManager.SetInstance(TwilioSMSManager())
# Set URL for local fileobjstores.
if options.options.fileobjstore:
# Import server for ssl and port options.
from viewfinder.backend.www import server
url_fmt_string = '%s://%s:%d/fileobjstore/' % (
'https' if options.options.ssl else 'http',
ServerEnvironment.GetHost(), options.options.port)
url_fmt_string += '%s/%%s'
for store_name in (ObjectStore.PHOTO, ObjectStore.USER_LOG,
ObjectStore.USER_ZIPS):
ObjectStore.GetInstance(store_name).SetUrlFmtString(
url_fmt_string % store_name)
OpManager.SetInstance(
OpManager(op_map=DB_OPERATION_MAP, client=client, scan_ops=scan_ops))
apns_feedback_handler = Device.FeedbackHandler(client)
APNS.SetInstance(
'dev', APNS(environment='dev', feedback_handler=apns_feedback_handler))
APNS.SetInstance(
'ent', APNS(environment='ent', feedback_handler=apns_feedback_handler))
APNS.SetInstance(
'prod', APNS(environment='prod',
feedback_handler=apns_feedback_handler))
http_client = AsyncHTTPClient()
ITunesStoreClient.SetInstance(
'dev', ITunesStoreClient(environment='dev', http_client=http_client))
ITunesStoreClient.SetInstance(
'prod', ITunesStoreClient(environment='prod', http_client=http_client))
# Ensure that system users are loaded.
yield LoadSystemUsers(client)
yield gen.Task(run_callback)
def testMultiNestedOp(self):
"""Test nested op within nested op."""
@gen.coroutine
def _InnererMethod(client, arg3):
self.assertTrue(Operation.GetCurrent().operation_id.startswith('++'))
self.assertEqual(arg3, 3)
self._method_count += 1
@gen.coroutine
def _InnerMethod(client, arg2):
self.assertEqual(arg2, 2)
self._method_count += 1
if self._method_count == 2:
yield Operation.CreateNested(client, '_InnererMethod', {'arg3': 3})
@gen.coroutine
def _OuterMethod(client, arg1):
self.assertEqual(arg1, 1)
self._method_count += 1
if self._method_count == 1:
yield Operation.CreateNested(client, '_InnerMethod', {'arg2': 2})
# Create custom OpManager and make it the current instance for duration of test.
op_mgr = self._CreateOpManager(handlers=[_OuterMethod, _InnerMethod, _InnererMethod])
OpManager.SetInstance(op_mgr)
outer_op = self._CreateTestOp(user_id=1, handler=_OuterMethod, arg1=1)
self._RunAsync(op_mgr.WaitForUserOps, self._client, 1)
self.assertEqual(self._method_count, 5)
def testNestedOpError(self):
"""Test nested op that fails with errors."""
@gen.coroutine
def _InnerMethod(client):
self._method_count += 1
if self._method_count < 8:
raise Exception('permanent error')
@gen.coroutine
def _OuterMethod(client):
self._method_count += 1
if self._method_count < 8:
yield Operation.CreateNested(client, '_InnerMethod', {})
self.assertEqual(Operation.GetCurrent().quarantine, 1)
# Create custom OpManager and make it the current instance for duration of test.
op_mgr = self._CreateOpManager(handlers=[_OuterMethod, _InnerMethod])
OpManager.SetInstance(op_mgr)
outer_op = self._CreateTestOp(user_id=1, handler=_OuterMethod)
op_mgr.MaybeExecuteOp(self._client, 1, None)
# Now run failed ops (they should eventually succeed due to method_count < 8 checks) and ensure
# that ops complete.
while len(self._RunAsync(Operation.RangeQuery, self._client, 1, None, None, None)) != 0:
pass
self.assertEqual(self._method_count, 9)
def testNestedOp(self):
"""Test creation and invocation of nested operation."""
@gen.coroutine
def _InnerMethod(client, arg1, arg2):
self._method_count += 1
self.assertEqual(arg1, 1)
self.assertEqual(arg2, 'hello')
self.assertEqual(self._method_count, 2)
# Assert that nested operation is derived from parent op.
inner_op = Operation.GetCurrent()
self.assertEqual(inner_op.user_id, outer_op.user_id)
self.assertEqual(inner_op.timestamp, outer_op.timestamp)
self.assertEqual(inner_op.operation_id, '+%s' % outer_op.operation_id)
@gen.coroutine
def _OuterMethod(client):
self._method_count += 1
if self._method_count == 1:
yield Operation.CreateNested(client, '_InnerMethod', {'arg1': 1, 'arg2': 'hello'})
# Create custom OpManager and make it the current instance for duration of test.
op_mgr = self._CreateOpManager(handlers=[_OuterMethod, _InnerMethod])
OpManager.SetInstance(op_mgr)
outer_op = self._CreateTestOp(user_id=1, handler=_OuterMethod)
self._RunAsync(op_mgr.WaitForUserOps, self._client, 1)
self.assertEqual(self._method_count, 3)
def WaitForOp(cls, client, user_id, operation_id, callback):
"""Waits for the specified operation to complete. WaitForOp behaves exactly like using the
"synchronous" option when submitting an operation. The callback will be invoked once the
operation has completed or if it's backed off due to repeated failure.
"""
OpManager.Instance().MaybeExecuteOp(client, user_id, operation_id,
callback)
def tearDown(self): # Restore original values. OpManager.SetInstance(self.prev_op_mgr) UserOpManager._INITIAL_BACKOFF_SECS = 8.0 OpManager._MAX_SCAN_ABANDONED_LOCKS_INTERVAL = timedelta(seconds=60) OpManager._MAX_SCAN_FAILED_OPS_INTERVAL = timedelta(hours=6) OpManager._SCAN_LIMIT = 10 super(OpManagerTestCase, self).tearDown()
def setUp(self):
if not hasattr(self, '_enable_xsrf'): self._enable_xsrf = True
if not hasattr(self, '_url_host') or self._url_host is None: self._url_host = 'www.goviewfinder.com'
super(ServiceBaseTestCase, self).setUp()
# TODO(spencer): remove this when switched to AsyncHTTPSTestCase.
basic_auth.BasicAuthHandler._HTTP_TEST_CASE = True
db_client.DBClient.SetInstance(local_client.LocalClient(vf_schema.SCHEMA))
self._client = db_client.DBClient.Instance()
email_mgr.EmailManager.SetInstance(email_mgr.TestEmailManager())
sms_mgr.SMSManager.SetInstance(sms_mgr.TestSMSManager())
self._backup_dir = tempfile.mkdtemp()
server_log.LogBatchPersistor.SetInstance(server_log.LogBatchPersistor(backup_dir=self._backup_dir))
APNS.SetInstance('test', APNS(environment='test',
feedback_handler=Device.FeedbackHandler(self._client)))
self._apns = TestService.Instance()
IdAllocator.ResetState()
# Do not freeze new account creation during testing (dy default).
options.options.freeze_new_accounts = False
# Set deterministic testing timestamp used in place of time.time() in server code.
util._TEST_TIME = time.time()
# Create validator and create some users and devices for testing convenience.
self._validator = DBValidator(self._client, self.stop, self.wait)
self._tester = ServiceTester(self.get_url(''), self.http_client, self._validator,
secrets.GetSecret('cookie_secret'), self.stop, self.wait)
self._test_id = 1
self._validate = True
# Skip model_db validation for specified tables. Ignored if _validate==False.
self._skip_validation_for = []
self._RunAsync(vf_schema.SCHEMA.VerifyOrCreate, self._client)
OpManager.SetInstance(OpManager(op_map=DB_OPERATION_MAP, client=self._client, scan_ops=True))
# Ensure that test users are created.
self._CreateTestUsers()
# Remove limit of number of auth messages that can be sent to a particular identity key.
auth_viewfinder.VerifyIdBaseHandler._MAX_MESSAGES_PER_MIN = 10000
auth_viewfinder.VerifyIdBaseHandler._MAX_MESSAGES_PER_DAY = 10000
def RunOnce():
db_client = DBClient.Instance()
http_client = AsyncHTTPClient()
ITunesStoreClient.SetInstance(
'prod', ITunesStoreClient(environment='prod', http_client=http_client))
OpManager.SetInstance(
OpManager(op_map=DB_OPERATION_MAP, client=db_client, scan_ops=False))
last_key = None
while True:
results, last_key = yield gen.Task(Subscription.Scan,
db_client,
None,
limit=1000,
excl_start_key=last_key)
for sub in results:
if ShouldProcess(sub):
yield ProcessSubscription(db_client, sub)
if last_key is None:
break
def setUp(self):
super(OpManagerTestCase, self).setUp()
self._id = 1
self._method_count = 0
self._wait_count = 0
# Speed up retries and decrease limits for testing.
UserOpManager._INITIAL_BACKOFF_SECS = 0.05
OpManager._MAX_SCAN_ABANDONED_LOCKS_INTERVAL = timedelta(seconds=0.5)
OpManager._MAX_SCAN_FAILED_OPS_INTERVAL = timedelta(seconds=0.5)
OpManager._SCAN_LIMIT = 1
# Remember the current OpManager instance in order to restore it after.
self.prev_op_mgr = OpManager.Instance()
def testOperationRetries(self):
"""Verify an operation is retried on failure."""
# Acquire lock ahead of creation and execution of the operation.
# This will cause the operation to fail, but not be aborted, so it can be retried.
self._RunAsync(
Lock.Acquire, self._client, LockResourceType.Viewpoint,
self._user.private_vp_id,
Operation.ConstructOperationId(self._mobile_dev.device_id, 123))
# Make request to upload an episode
op = self._RunAsync(self._UploadPhotoOperation, self._user.user_id,
self._mobile_dev.device_id, 1)
start = time.time()
while True:
# Wait for op to execute.
self._RunAsync(self.io_loop.add_timeout,
time.time() + UserOpManager._INITIAL_BACKOFF_SECS)
op = self._RunAsync(Operation.Query,
self._client,
op.user_id,
op.operation_id,
None,
must_exist=False)
if op.attempts < 3:
self.assertTrue(not op.quarantine)
elif op.attempts == 3:
# After 3 attempts, op should go into quarantine.
self.assertTrue(op.first_failure is not None)
self.assertTrue(op.last_failure is not None)
self.assertEqual(op.quarantine, 1)
# Manually goose the op manager to retry abandoned locks.
OpManager.Instance()._ScanFailedOps()
elif op.attempts == 4:
# No operations completed successfully.
self._CheckCounters(op.attempts, op.attempts - 1)
break
def tearDown(self):
# If validation is enabled, validate all viewpoint assets using the service API.
# If the test failed, skip validation so we don't get extra redundant error reports
# (The unittest framework reports errors in the main test and errors in tearDown separately)
validate = self._validate and self._GetUnittestErrorCount() == self._unittest_error_count
if validate:
self._ValidateAssets()
# Ensure that operations do not exist in the db.
from viewfinder.backend.db.operation import Operation
Operation.Scan(self._client, None, self.stop)
ops, last_key = self.wait()
if validate:
self.assertTrue(len(ops) == 0, ops)
self.assertTrue(last_key is None)
# Cleanup all assets created during tests.
self._tester.Cleanup(validate=validate, skip_validation_for=self._skip_validation_for)
self._RunAsync(server_log.LogBatchPersistor.Instance().close)
self._RunAsync(OpManager.Instance().Drain)
shutil.rmtree(self._backup_dir)
super(ServiceBaseTestCase, self).tearDown()
self.assertIs(Operation.GetCurrent().operation_id, None)
def _CreateOpManager(self, handlers):
op_map = {handler.__name__: OpMapEntry(handler, []) for handler in handlers}
return OpManager(op_map, client=self._client, scan_ops=True)
def _ValidateAutoSave(tester, user_id, device_id, request_dict):
"""Validates that the shared photos have been saved to the default viewpoint of any follower
that has the target viewpoint marked as "auto-save".
"""
from viewfinder.backend.www.test.save_photos_test import _ValidateSavePhotos
validator = tester.validator
# Validate auto-save.
follower_matches = lambda f: f.viewpoint_id == request_dict['viewpoint_id']
for follower in validator.QueryModelObjects(Follower,
predicate=follower_matches):
# Only validate current followers that have viewpoint marked as auto-save.
if not follower.ShouldAutoSave() or follower.IsRemoved():
continue
follower_user = tester._RunAsync(User.Query, validator.client,
follower.user_id, None)
# Skip validation of follower that is sharing from default viewpoint.
source_episodes = [
validator.GetModelObject(Episode, ep_dict['existing_episode_id'])
for ep_dict in request_dict['episodes']
]
if all(ep.viewpoint_id == follower_user.private_vp_id
for ep in source_episodes):
continue
# The share_existing op triggered a save_photos op, so wait for that to complete.
tester._RunAsync(OpManager.Instance().WaitForUserOps, validator.client,
follower.user_id)
# Get the ids of episodes that should have been created during the auto-save.
expected_asset_id = follower_user.asset_id_seq
save_request_dict = deepcopy(request_dict)
# Iterate backwards, since last episode should have used last asset id.
for ep_dict in reversed(save_request_dict['episodes']):
# The share_existing "new_episode_id" becomes the "existing_episode_id" for save_photos.
ep_dict['existing_episode_id'] = ep_dict['new_episode_id']
# If target episode already existed, that should have been used.
episode_matches = lambda e: e.user_id == follower.user_id and e.parent_ep_id == ep_dict[
'existing_episode_id']
episodes = validator.QueryModelObjects(Episode,
predicate=episode_matches)
if episodes:
ep_dict['new_episode_id'] = episodes[0].episode_id
else:
expected_asset_id -= 1
timestamp, _, _ = Episode.DeconstructEpisodeId(
ep_dict['existing_episode_id'])
ep_dict['new_episode_id'] = Episode.ConstructEpisodeId(
timestamp, follower_user.webapp_dev_id, expected_asset_id)
# Create expected activity dict for the save_photos op.
expected_asset_id -= 1
save_activity_id = Activity.ConstructActivityId(
request_dict['activity']['timestamp'], follower_user.webapp_dev_id,
expected_asset_id)
save_request_dict['activity']['activity_id'] = save_activity_id
# Create expected operation id for the save_photos op.
expected_asset_id -= 1
save_op_id = Operation.ConstructOperationId(
follower_user.webapp_dev_id, expected_asset_id)
save_request_dict['headers']['op_id'] = save_op_id
_ValidateSavePhotos(tester, follower.user_id,
follower_user.webapp_dev_id, save_request_dict)
def CreateAndExecute(
cls,
client,
user_id,
device_id,
method,
args,
callback,
message_version=message.MAX_SUPPORTED_MESSAGE_VERSION):
"""Creates a new operation with 'method' and 'args' describing the operation. After
successfully creating the operation, the operation is asynchronously executed. Returns
the op that was executed.
"""
# Get useful headers and strip all else.
headers = args.pop('headers', {})
synchronous = headers.pop('synchronous', False)
# Validate the op_id and op_timestamp fields.
op_id = headers.pop('op_id', None)
op_timestamp = headers.pop('op_timestamp', None)
assert (op_id is not None) == (op_timestamp
is not None), (op_id, op_timestamp)
# Validate that op_id is correctly formed and is allowed to be generated by the current device.
# No need to do this if the op_id was generated by the system as part of message upgrade.
if op_id is not None and headers.get(
'original_version',
0) >= message.Message.ADD_OP_HEADER_VERSION:
yield Operation.VerifyOperationId(client, user_id, device_id,
op_id)
# Use the op_id provided by the user, or generate a system op-id.
if op_id is None:
op_id = yield gen.Task(Operation.AllocateSystemOperationId, client)
# Possibly migrate backwards to a message version that is compatible with older versions of the
# server that may still be running.
op_message = message.Message(
args, default_version=message.MAX_MESSAGE_VERSION)
yield gen.Task(op_message.Migrate,
client,
migrate_version=message_version,
migrators=OpManager.Instance().op_map[method].migrators)
op = Operation(user_id, op_id)
op.device_id = device_id
op.method = method
op.json = json.dumps(args)
op.attempts = 0
# Set timestamp to header value if it was specified, or current timestamp if not.
if op_timestamp is not None:
op.timestamp = op_timestamp
else:
op.timestamp = util.GetCurrentTimestamp()
# Set expired backoff so that if this process fails before the op can be executed, in the worst
# case it will eventually get picked up by the OpManager's scan for failed ops. Note that in
# rare cases, this may mean that the op gets picked up immediately by another server (i.e. even
# though the current server has *not* failed), but that is fine -- it doesn't really matter what
# server executes the op, it just matters that the op gets executed in a timely manner.
op.backoff = 0
# Try to create the operation if it does not yet exist.
try:
yield gen.Task(op.Update, client, expected={'operation_id': False})
# Execute the op according to the 'synchronous' parameter. If 'synchronous' is True, the
# callback is invoked only after the operation has completed. Useful during unittests to
# ensure the mutations wrought by the operation are queryable.
logging.info('PERSIST: user: %d, device: %d, op: %s, method: %s' %
(user_id, device_id, op_id, method))
except Exception:
# Return existing op.
logging.warning('operation "%s" already exists', op_id)
existing_op = yield gen.Task(Operation.Query,
client,
user_id,
op_id,
None,
must_exist=False)
if existing_op is not None:
op = existing_op
# If not synchronous, we fire the callback, but continue to execute.
if not synchronous:
callback(op)
# Establish new "clean" context in which to execute the operation. The operation should not rely
# on any context, since it may end up run on a completely different machine. In addition, establish
# an exception barrier in order to handle any bugs or asserts, rather than letting the context
# established for the request handle it, since it will have already completed).
with stack_context.NullContext():
with util.ExceptionBarrier(util.LogExceptionCallback):
OpManager.Instance().MaybeExecuteOp(
client, user_id, op.operation_id)
else:
# Let exceptions flow up to request context so they'll be put into an error response.
OpManager.Instance().MaybeExecuteOp(client, user_id,
op.operation_id,
partial(callback, op))