def RefreshGoogleAccessToken(self, client, callback): """Refreshes an expired google access token using the refresh token. """ def _OnRefresh(response): try: response_dict = www_util.ParseJSONResponse(response) except web.HTTPError as e: if e.status_code == 400: logging.error( '%s: failed to refresh access token; clearing refresh token' % e) with util.ExceptionBarrier(util.LogExceptionCallback): self.refresh_token = None self.Update(client, util.NoCallback) raise self.access_token = response_dict['access_token'] self.expires = time.time() + response_dict['expires_in'] callback() body = urllib.urlencode({ 'refresh_token': self.refresh_token, 'client_id': secrets.GetSecret('google_client_id'), 'client_secret': secrets.GetSecret('google_client_secret'), 'grant_type': 'refresh_token' }) http_client = httpclient.AsyncHTTPClient() http_client.fetch(Identity._GOOGLE_REFRESH_URL, method='POST', callback=_OnRefresh, body=body)
def _OnErrorNoRetry(response): self.assertTrue(response.error) self.assertNotLogMatches('Retrying function after', 'Retry should not happen on HTTP 403 error') asyncS3 = AsyncS3Connection(host='unknown', aws_access_key_id=secrets.GetSecret('aws_access_key_id'), aws_secret_access_key=secrets.GetSecret('aws_secret_access_key'), retry_policy=S3RetryPolicy(max_tries=2, min_delay=0)) asyncS3.make_request('GET', self.bucket, self.key, callback=_OnErrorRetry)
def tearDown(self): def _OnCompletedDelete(response): self.stop() asyncS3 = AsyncS3Connection(aws_access_key_id=secrets.GetSecret('aws_access_key_id'), aws_secret_access_key=secrets.GetSecret('aws_secret_access_key')) asyncS3.make_request('DELETE', self.bucket, self.key, callback=_OnCompletedDelete) self.wait() super(AsyncS3TestCase, self).tearDown()
def DeleteFacebookTestUsers(self): logging.info('Deleting facebook users') http_client = httpclient.HTTPClient() users = FacebookUtils.QueryFacebookTestUsers( http_client, secrets.GetSecret('facebook_api_key'), secrets.GetSecret('facebook_secret'), self._access_token, limit=options.options.num_users) [FacebookUtils.DeleteTestUser(u) for u in users]
def __init__(self, schema): self._read_queues = dict([(t.name_in_db, RequestQueue(t.name, False, '%s reads' % (t.name), t.read_units)) \ for t in schema.GetTables()]) self._write_queues = dict([(t.name_in_db, RequestQueue(t.name, True, '%s writes' % (t.name), t.write_units)) \ for t in schema.GetTables()]) self._cp_read_only_queue = RequestQueue('ControlPlane', False, 'Control Plane R/O', 100) self._cp_mutate_queue = RequestQueue('ControlPlane', True, 'Control Plane Mutate', 1) self._paused = False self._asyncdynamo = AsyncDynamoDB(secrets.GetSecret('aws_access_key_id'), secrets.GetSecret('aws_secret_access_key'))
def __init__(self): url = _FACEBOOK_APP_ACCESS_TOKEN_URL + '?' + \ urllib.urlencode({'client_id': secrets.GetSecret('facebook_api_key'), 'client_secret': secrets.GetSecret('facebook_secret'), 'grant_type': 'client_credentials'}) http_client = httpclient.HTTPClient() response = http_client.fetch(url, request_timeout=100) try: self._access_token = urlparse.parse_qs( response.body)['access_token'][0] except: logging.error( 'unable to parse access token from response body: %s' % response.body) raise
def _TestMakeRequest(self, value): asyncS3 = AsyncS3Connection(host='s3.amazonaws.com', aws_access_key_id=secrets.GetSecret('aws_access_key_id'), aws_secret_access_key=secrets.GetSecret('aws_secret_access_key')) def _OnCompletedGet(response): self.assertEqual(response.body, value if type(value) is str else value.encode('utf-8')) self.assertTrue(1) self.assertEqual(response.headers['Content-Type'], 'text/plain; charset=utf-8') self.stop() def _OnCompletedPut(response): self.assertFalse(response.error) asyncS3.make_request('GET', self.bucket, self.key, callback=_OnCompletedGet) asyncS3.make_request('PUT', self.bucket, self.key, headers={'Content-Type' : 'text/plain; charset=utf-8'}, body=value, callback=_OnCompletedPut) self.wait(timeout=30)
def PollForAuthentication(self, callback): """Poll the google authorization service to find if the user code generated in a previous call to GetUserCode() has been used to authorize a google user account. If an account has been authorized, this method will use that authorization to log into the viewfinder service, thus retrieving the needed authentication cookie. The given callback will be invoked with a boolean parameter to indicate whether or not the authentication was successful. If authentication was not successful, then this method can be polled again until it is successful. """ if not hasattr(self, '_device_code'): raise ScenarioLoginError('Must call GetUserCode() on a device before using the ' 'PollForAuthentication() method.') http_client = httpclient.AsyncHTTPClient() def _OnLogin(response): if not response.code in (200, 302): raise ScenarioLoginError('Error during login process:%s' % response.error) self._user_cookie = self._GetUserCookieFromResponse(response) self._SaveAuthentication() callback(True) def _OnPollTokenEndpoint(response): json_response = www_util.ParseJSONResponse(response) if 'error' in json_response: callback(False) else: refresh_token = json_response.get('refresh_token') url = 'https://%s:%d/auth/google?refresh_token=%s' % \ (ServerEnvironment.GetHost(), options.options.port, refresh_token) http_client.fetch(url, method='POST', callback=_OnLogin, body=json.dumps({}), validate_cert=False, follow_redirects=False, headers={'Content-Type': 'application/json'}) url = _GOOGLE_OAUTH2_TOKEN_URL request_args = {'client_id': secrets.GetSecret('google_client_mobile_id'), 'client_secret': secrets.GetSecret('google_client_mobile_secret'), 'code': self._device_code, 'grant_type': 'http://oauth.net/grant_type/device/1.0'} http_client.fetch(url, method='POST', body=urllib.urlencode(request_args), callback=_OnPollTokenEndpoint)
def get_app(self): """Creates a web server which handles /service requests.""" options.options.localdb = True options.options.fileobjstore = True options.options.localdb_dir = '' options.options.devbox = True options.options.domain = 'goviewfinder.com' options.options.short_domain = 'short.goviewfinder.com' # Init secrets with the unencrypted 'goviewfinder.com' domain. secrets.InitSecretsForTest() object_store.InitObjectStore(temporary=True) environ.ServerEnvironment.InitServerEnvironment() # Set up photo object store. obj_store = object_store.ObjectStore.GetInstance(object_store.ObjectStore.PHOTO) obj_store.SetUrlFmtString(self.get_url('/fileobjstore/photo/%s')) # Set up user logs object store. user_log_obj_store = object_store.ObjectStore.GetInstance(object_store.ObjectStore.USER_LOG) user_log_obj_store.SetUrlFmtString(self.get_url('/fileobjstore/user_log/%s')) # Set up user_zips object store. user_zips_obj_store = object_store.ObjectStore.GetInstance(object_store.ObjectStore.USER_ZIPS) user_zips_obj_store.SetUrlFmtString(self.get_url('/fileobjectstore/user_zips/%s')) settings = { 'login_url': '/', 'cookie_secret': secrets.GetSecret('cookie_secret'), 'obj_store': obj_store, 'server_version': ServiceTester.SERVER_VERSION, 'google_client_id': secrets.GetSecret('google_client_id'), 'google_client_secret': secrets.GetSecret('google_client_secret'), 'google_client_mobile_id': secrets.GetSecret('google_client_mobile_id'), 'google_client_mobile_secret': secrets.GetSecret('google_client_mobile_secret'), 'facebook_api_key': secrets.GetSecret('facebook_api_key'), 'facebook_secret': secrets.GetSecret('facebook_secret'), 'template_path': ResourcesManager.Instance().template_path, 'ui_modules': uimodules, 'xsrf_cookies' : self._enable_xsrf, 'static_path': ResourcesManager.Instance().static_path, } # Start with the production webapp handlers and add several for testing. webapp_handlers = deepcopy(server.WEBAPP_HANDLERS + server.ADMIN_HANDLERS) webapp_handlers.append((r'/fileobjstore/photo/(.*)', file_object_store.FileObjectStoreHandler, { 'storename': object_store.ObjectStore.PHOTO, 'contenttype': 'image/jpeg' })) webapp_handlers.append((r'/fileobjstore/user_log/(.*)', file_object_store.FileObjectStoreHandler, { 'storename': object_store.ObjectStore.USER_LOG, 'contenttype': 'text/plain' })) webapp_handlers.append((r'/fileobjstore/user_zips/(.*)', file_object_store.FileObjectStoreHandler, { 'storename': object_store.ObjectStore.USER_ZIPS, 'contenttype': 'application/zip' })) # Fake viewfinder handler - added explicitly because it is not part of WEBAPP_HANDLERS. webapp_handlers.append((r'/(link|login|register)/fakeviewfinder', auth_viewfinder.FakeAuthViewfinderHandler)) application = web.Application(**settings) application.add_handlers(options.options.short_domain, server.SHORT_DOMAIN_HANDLERS) application.add_handlers('.*', webapp_handlers) return application
def DecodeUnsubscribeCookie(cls, unsubscribe_cookie): """Decode a user unsubscribe cookie that is passed as an argument to the unsubscribe handler. Returns the unsubscribe dict containing the user_id and email_type originally passed to CreateUnsubscribeCookie. """ value = web.decode_signed_value(secrets.GetSecret('invite_signing'), 'unsubscribe', unsubscribe_cookie) return None if value is None else json.loads(value)
def _Start(): assert options.options.id and options.options.data_key and options.options.data_value api_key = secrets.GetSecret(API_KEY_NAME) print 'API key: %s' % api_key g = GCM(api_key) data = {options.options.data_key: options.options.data_value} # Do not catch any of the exceptions for now, we'd like to see them. g.plaintext_request(registration_id=options.options.id, data=data)
def QueryFacebookTestUsers(self, limit): url = (_FACEBOOK_QUERY_TEST_USERS_URL % secrets.GetSecret('facebook_api_key')) + '?' + \ urllib.urlencode({'access_token': self._access_token, 'limit': limit}) http_client = httpclient.HTTPClient() response = http_client.fetch(url, request_timeout=100) try: json_data = json.loads(response.body) return json_data['data'] except: logging.error('unable to query facebook test users: %s' % response.body) raise
def testMultipleManagers(self): """Test the secrets managers in their natural habitat: automatic selection of user vs shared based on flags.""" # these may not be None if we've been running other tests using run-tests. secrets._user_secrets_manager = None secrets._shared_secrets_manager = None # Devbox mode: init user secrets, and lazily init shared secrets is requesting a secret on in user secrets. options.options.devbox = True secrets.InitSecrets() self.assertIsNotNone(secrets._user_secrets_manager) self.assertIsNone(secrets._shared_secrets_manager) # Request a secret contained in user secrets: shared secrets remain uninitialized. secrets._user_secrets_manager.PutSecret('foo', 'codeforfoo') self.assertEqual(secrets.GetSecret('foo'), 'codeforfoo') self.assertIsNotNone(secrets._user_secrets_manager) self.assertIsNone(secrets._shared_secrets_manager) # Request a secret not contained anywhere. As soon as we notice that it's not in user secrets, we initialize # the shared secrets and look there, which fails. self.assertRaises(KeyError, secrets.GetSecret, 'bar') self.assertIsNotNone(secrets._user_secrets_manager) self.assertIsNotNone(secrets._shared_secrets_manager) # Non-devbox mode: user secrets are never used. shared secrets are initialized right away. options.options.devbox = False secrets._user_secrets_manager = None secrets._shared_secrets_manager = None secrets.InitSecrets() self.assertIsNone(secrets._user_secrets_manager) self.assertIsNotNone(secrets._shared_secrets_manager) # Lookup whatever we want, we still won't use the user secrets.:w secrets._shared_secrets_manager.PutSecret('foo', 'codeforfoo') self.assertEqual(secrets.GetSecret('foo'), 'codeforfoo') self.assertRaises(KeyError, secrets.GetSecret, 'bar') self.assertIsNone(secrets._user_secrets_manager) self.assertIsNotNone(secrets._shared_secrets_manager)
def VerifyReceipt(self, receipt_data, callback): """Verifies a receipt. Callback receives a VerifyResponse.""" def _OnFetch(response): response.rethrow() callback(VerifyResponse(receipt_data, response.body)) request = { 'receipt-data': base64.b64encode(receipt_data), 'password': secrets.GetSecret('itunes_subscription_secret'), } self.http_client.fetch(self._settings['verify_url'], method='POST', body=json.dumps(request), callback=_OnFetch)
def CreateTestUser(self, name): print 'creating user %s' % name url = (_FACEBOOK_CREATE_TEST_USER_URL % secrets.GetSecret('facebook_api_key')) + '?' + \ urllib.urlencode({'installed': 'true', 'name': name, 'permissions': _FACEBOOK_PERMISSIONS, 'method': 'post', 'access_token': self._access_token}) http_client = httpclient.HTTPClient() response = http_client.fetch(url, request_timeout=100) try: return json.loads(response.body) except: logging.error('unable to parse user data from response body: %s' % response.body) raise
def GetUserCode(self, callback): """Retrieve a user code from google's device login API. The given callback will be invoked with the user code and a URL where the user code can be used to authenticate a google account. """ def _OnGetDeviceCode(response): response_dict = www_util.ParseJSONResponse(response) self._device_code = response_dict.get('device_code') callback(response_dict.get('user_code'), response_dict.get('verification_url')) # Get a device code from google's API request_args = {'client_id': secrets.GetSecret('google_client_mobile_id'), 'scope': _GOOGLE_OAUTH2_SCOPES} url = _GOOGLE_OAUTH2_DEVICECODE_URL http_client = httpclient.AsyncHTTPClient() http_client.fetch(url, method='POST', body=urllib.urlencode(request_args), callback=_OnGetDeviceCode)
def _TestMakeRequestError(self): def _OnErrorRetry(response): self.assertTrue(response.error) self.assertLogMatches('(Retrying function after){1}', 'Retry should have occurred once') self.stop() def _OnErrorNoRetry(response): self.assertTrue(response.error) self.assertNotLogMatches('Retrying function after', 'Retry should not happen on HTTP 403 error') asyncS3 = AsyncS3Connection(host='unknown', aws_access_key_id=secrets.GetSecret('aws_access_key_id'), aws_secret_access_key=secrets.GetSecret('aws_secret_access_key'), retry_policy=S3RetryPolicy(max_tries=2, min_delay=0)) asyncS3.make_request('GET', self.bucket, self.key, callback=_OnErrorRetry) asyncS3 = AsyncS3Connection(aws_access_key_id='unknown', aws_secret_access_key=secrets.GetSecret('aws_secret_access_key')) asyncS3.make_request('GET', self.bucket, self.key, callback=_OnErrorNoRetry) self.wait(timeout=30)
def setUp(self): if not hasattr(self, '_enable_xsrf'): self._enable_xsrf = True if not hasattr(self, '_url_host') or self._url_host is None: self._url_host = 'www.goviewfinder.com' super(ServiceBaseTestCase, self).setUp() # TODO(spencer): remove this when switched to AsyncHTTPSTestCase. basic_auth.BasicAuthHandler._HTTP_TEST_CASE = True db_client.DBClient.SetInstance(local_client.LocalClient(vf_schema.SCHEMA)) self._client = db_client.DBClient.Instance() email_mgr.EmailManager.SetInstance(email_mgr.TestEmailManager()) sms_mgr.SMSManager.SetInstance(sms_mgr.TestSMSManager()) self._backup_dir = tempfile.mkdtemp() server_log.LogBatchPersistor.SetInstance(server_log.LogBatchPersistor(backup_dir=self._backup_dir)) APNS.SetInstance('test', APNS(environment='test', feedback_handler=Device.FeedbackHandler(self._client))) self._apns = TestService.Instance() IdAllocator.ResetState() # Do not freeze new account creation during testing (dy default). options.options.freeze_new_accounts = False # Set deterministic testing timestamp used in place of time.time() in server code. util._TEST_TIME = time.time() # Create validator and create some users and devices for testing convenience. self._validator = DBValidator(self._client, self.stop, self.wait) self._tester = ServiceTester(self.get_url(''), self.http_client, self._validator, secrets.GetSecret('cookie_secret'), self.stop, self.wait) self._test_id = 1 self._validate = True # Skip model_db validation for specified tables. Ignored if _validate==False. self._skip_validation_for = [] self._RunAsync(vf_schema.SCHEMA.VerifyOrCreate, self._client) OpManager.SetInstance(OpManager(op_map=DB_OPERATION_MAP, client=self._client, scan_ops=True)) # Ensure that test users are created. self._CreateTestUsers() # Remove limit of number of auth messages that can be sent to a particular identity key. auth_viewfinder.VerifyIdBaseHandler._MAX_MESSAGES_PER_MIN = 10000 auth_viewfinder.VerifyIdBaseHandler._MAX_MESSAGES_PER_DAY = 10000
def _Start(callback): """Grab a lock on job:itunes_trends and call RunOnce. If we get a return value, write it to the job summary.""" assert options.options.user is not None and options.options.vendor_id is not None apple_id = '*****@*****.**' % options.options.user # Attempt to lookup iTunes Connect password from secrets. password = secrets.GetSecret('itunes_connect_%s' % options.options.user) assert password client = db_client.DBClient.Instance() job = Job(client, 'itunes_trends') if options.options.require_lock: got_lock = yield gen.Task(job.AcquireLock) if got_lock == False: logging.warning('Failed to acquire job lock: exiting.') callback() return result = None job.Start() try: result = yield gen.Task(RunOnce, client, job, apple_id, password) except: # Failure: log run summary with trace. msg = traceback.format_exc() logging.info('Registering failed run with message: %s' % msg) yield gen.Task(job.RegisterRun, Job.STATUS_FAILURE, failure_msg=msg) else: if result is not None and not options.options.dry_run: # Successful run with data processed and not in dry-run mode: write run summary. stats = DotDict() stats['last_day'] = result logging.info('Registering successful run with stats: %r' % stats) yield gen.Task(job.RegisterRun, Job.STATUS_SUCCESS, stats=stats) finally: yield gen.Task(job.ReleaseLock) callback()
def __init__(self): self._api_user = secrets.GetSecret('sendgrid_api_user') self._api_key = secrets.GetSecret('sendgrid_api_key')
def __init__(self): self._api_account_sid = secrets.GetSecret('twilio_account_sid') self._api_token = secrets.GetSecret('twilio_auth_token') self._api_number = secrets.GetSecret('twilio_api_number') self._sms_gateway_api_url = 'https://api.twilio.com/2010-04-01/Accounts/%s/SMS/Messages.json' % \ self._api_account_sid
def __init__(self): self._api_user = secrets.GetSecret('clickatell_api_user') self._api_password = secrets.GetSecret('clickatell_api_password') self._api_id = secrets.GetSecret('clickatell_api_id') self._api_number = secrets.GetSecret('clickatell_api_number')
def StartServer(serve_webapp=True, serve_static_web=True, serve_admin=True): """Initialize the datastore and operation manager with the viewfinder schema. This typically verifies the schema. If the schema does not yet exist, it is created. Defines settings dictionary and sets up main application with list of handlers. """ client = db_client.DBClient.Instance() settings = { 'gzip': True, 'login_url': '/', 'admin_login_url': '/admin/otp', 'domain': options.options.domain, 'server_version': options.options.server_version, 'cookie_secret': secrets.GetSecret('cookie_secret'), 'facebook_api_key': secrets.GetSecret('facebook_api_key'), 'facebook_secret': secrets.GetSecret('facebook_secret'), 'google_client_id': secrets.GetSecret('google_client_id'), 'google_client_secret': secrets.GetSecret('google_client_secret'), 'google_client_mobile_id': secrets.GetSecret('google_client_mobile_id'), 'google_client_mobile_secret': secrets.GetSecret('google_client_mobile_secret'), 'template_path': ResourcesManager.Instance().template_path, 'ui_modules': uimodules, 'xsrf_cookies': options.options.enable_xsrf, 'debug': options.options.server_debug, 'static_path': ResourcesManager.Instance().static_path, } if options.options.log_file_prefix: settings['logs_dir'] = os.path.dirname(options.options.log_file_prefix) # Configure metrics uploading. if options.options.upload_metrics: for interval in metric.METRIC_INTERVALS: metric.Metric.StartMetricUpload(client, metric.DEFAULT_CLUSTER_NAME, interval) # Setup application and SSL HTTP server. handlers = deepcopy(COMMON_HANDLERS) if serve_webapp: # Configure web application handlers. webapp_handlers = deepcopy(WEBAPP_HANDLERS) # Initialize the file object store if specified. obj_store = ObjectStore.GetInstance(ObjectStore.PHOTO) settings['obj_store'] = obj_store if options.options.fileobjstore: for store_name, content_type in ((ObjectStore.PHOTO, r'image/jpeg'), (ObjectStore.USER_LOG, r'text/plain'), (ObjectStore.USER_ZIPS, r'application/zip')): webapp_handlers.append( (r'/fileobjstore/%s/(.*)' % store_name, file_object_store.FileObjectStoreHandler, { 'storename': store_name, 'contenttype': content_type })) if ServerEnvironment.IsDevBox(): webapp_handlers.append((r'/(link|login|register)/fakeviewfinder', auth_viewfinder.FakeAuthViewfinderHandler)) # Set the testing directories. if options.options.testing_path is not None: webapp_handlers.append( (r'/testing/hook/(.*)', test_hook.TestHookHandler)) webapp_handlers.append( (r'/testing/static/(.*)', web.StaticFileHandler, { 'path': '%s' % options.options.testing_path })) handlers.extend(webapp_handlers) if serve_static_web: # Configure static web handlers. static_web_handlers = deepcopy(STATIC_WEB_HANDLERS) handlers.extend(static_web_handlers) if serve_admin: # Configure and verify admin handlers. admin_handlers = deepcopy(ADMIN_HANDLERS) for path, handler in admin_handlers: if not issubclass(handler, basic_auth.BasicAuthHandler): raise TypeError('Administration handlers must ' 'subclass BasicAuthHandler') handlers.extend(admin_handlers) # Catch-all handler for 404 pages. handlers.extend([(r'/.*', base.PageNotFoundHandler)]) # Create application and separately add handlers for the short domain and the # regular domain. # # Note that, although the short-domain handlers are added after the initial construction # of the Application, those routes will take priority over the routes in the handlers # array. application = web.Application(handlers, **settings) application.add_handlers(re.escape(options.options.short_domain), SHORT_DOMAIN_HANDLERS) # Start the HTTP server. http_server = httpserver.HTTPServer( application, xheaders=options.options.xheaders, ssl_options={ 'certfile': secrets.GetSecretFile('%s.crt' % settings['domain']), 'keyfile': secrets.GetSecretFile('%s.key' % settings['domain']), } if options.options.ssl else None) with stack_context.NullContext(): http_server.listen(options.options.port) # Setup redirect server for HTTP -> HTTPS. if options.options.ssl: http_settings = { 'host': ServerEnvironment.GetHost(), 'redirect_port': options.options.redirect_port, 'xheaders': options.options.xheaders, } redirect_handlers = [ (r'/(.*)', index.RedirectHandler), ] redirect_server = httpserver.HTTPServer( web.Application(redirect_handlers, **http_settings)) with stack_context.NullContext(): redirect_server.listen(options.options.insecure_port) # Ensure that system users have been created if running with a local db (needs server to be running). if options.options.localdb: yield CreateSystemUsers(client) # Run the server until it hits an exception or stop signal. yield gen.Task(lambda callback: None)
def CreateUnsubscribeCookie(cls, user_id, email_type): """Create a user unsubscribe cookie that is passed as an argument to the unsubscribe handler, and which proves control of the given user id. """ unsubscribe_dict = {'user_id': user_id, 'email_type': email_type} return web.create_signed_value(secrets.GetSecret('invite_signing'), 'unsubscribe', json.dumps(unsubscribe_dict))
def initialize(self): self.conn = boto.connect_s3( aws_access_key_id=secrets.GetSecret('aws_access_key_id').strip(), aws_secret_access_key=secrets.GetSecret( 'aws_secret_access_key').strip()) self.bucket = self.conn.get_bucket('staging.goviewfinder.com')