def main():
parse_command_line()
assert options.user_id, '--user_id must be set'
ServerEnvironment.InitServerEnvironment()
yield gen.Task(InitSecrets)
yield gen.Task(InitDB, vf_schema.SCHEMA, verify_or_create=False)
yield Terminate(DBClient.Instance(),
user_id=options.user_id,
base_url='https://%s' % ServerEnvironment.GetHost())
def __init__(self, name):
self.name = name
self._svc_url = 'https://%s:%d/' % (ServerEnvironment.GetHost(), options.options.port)
self._user_cookie = None
if options.options.watchdog_auth_reset:
self._ClearAuthentication()
else:
self._LoadAuthentication()
def post(self, action):
"""POST is used when authenticating via the mobile application."""
if not ServerEnvironment.IsDevBox():
raise web.HTTPError(403, _FAKE_AUTHORIZATION_FORBIDDEN)
user_dict, ident_dict, device_dict = yield gen.Task(
self._StartAuthViewfinder, action)
# Finish user authentication.
self._AuthUser(user_dict, ident_dict, device_dict)
def _ResetArchiveDir(self):
"""Get our temp directory into a known clean state."""
# Make sure certain directories already exists.
if not os.path.exists(ServerEnvironment.GetViewfinderTempDirPath()):
os.mkdir(ServerEnvironment.GetViewfinderTempDirPath())
if not os.path.exists(self._temp_dir_path):
os.mkdir(self._temp_dir_path)
# Blow away any previously existing content.
if os.path.exists(self._content_dir_path):
shutil.rmtree(self._content_dir_path)
assert not os.path.exists(self._content_dir_path)
# Blow away any previous zip file.
if os.path.exists(self._zip_file_path):
os.remove(self._zip_file_path)
assert not os.path.exists(self._zip_file_path)
# Recreate the content directory.
os.mkdir(self._content_dir_path)
os.mkdir(self._data_dir_path)
def _GetAuthEmail(cls, client, action, use_short_token, user_name,
identity, short_url):
"""Returns a dict of parameters that will be passed to EmailManager.SendEmail in order to
email an access token to a user who is verifying his/her account.
"""
action_info = VerifyIdBaseHandler.ACTION_MAP[action]
identity_type, identity_value = Identity.SplitKey(identity.key)
# Create arguments for the email.
args = {
'from': EmailManager.Instance().GetInfoAddress(),
'fromname': 'Viewfinder',
'to': identity_value
}
util.SetIfNotNone(args, 'toname', user_name)
# Create arguments for the email template.
fmt_args = {
'user_name':
user_name or identity_value,
'user_email':
identity_value,
'url':
'https://%s/%s%s' % (ServerEnvironment.GetHost(),
short_url.group_id, short_url.random_key),
'title':
action_info.title,
'use_short_token':
use_short_token,
'access_token':
identity.access_token
}
# The email html format is designed to meet these requirements:
# 1. It must be viewable on even the most primitive email html viewer. Avoid fancy CSS.
# 2. It cannot contain any images. Some email systems (like Gmail) do not show images by default.
# 3. It must be short and look good on an IPhone 4S screen. The action button should be visible
# without any scrolling necessary.
resources_mgr = ResourcesManager.Instance()
if use_short_token:
args['subject'] = 'Viewfinder Code: %s' % identity.access_token
else:
args['subject'] = action_info.title
args['html'] = resources_mgr.GenerateTemplate(
action_info.email_template, is_html=True, **fmt_args)
args['text'] = resources_mgr.GenerateTemplate(
action_info.email_template, is_html=False, **fmt_args)
# Remove extra whitespace in the HTML (seems to help it avoid Gmail spam filter).
args['html'] = escape.squeeze(args['html'])
return args
def _OnPollTokenEndpoint(response):
json_response = www_util.ParseJSONResponse(response)
if 'error' in json_response:
callback(False)
else:
refresh_token = json_response.get('refresh_token')
url = 'https://%s:%d/auth/google?refresh_token=%s' % \
(ServerEnvironment.GetHost(), options.options.port, refresh_token)
http_client.fetch(url, method='POST',
callback=_OnLogin,
body=json.dumps({}),
validate_cert=False, follow_redirects=False,
headers={'Content-Type': 'application/json'})
def _StartWWW(run_callback, scan_ops):
"""Starts services necessary for operating in the Viewfinder WWW server environment. Invokes
'run_callback' asynchronously.
"""
client = db_client.DBClient.Instance()
# Log emails and texts to the console in local mode.
if options.options.local_services:
EmailManager.SetInstance(LoggingEmailManager())
SMSManager.SetInstance(LoggingSMSManager())
else:
EmailManager.SetInstance(SendGridEmailManager())
SMSManager.SetInstance(TwilioSMSManager())
# Set URL for local fileobjstores.
if options.options.fileobjstore:
# Import server for ssl and port options.
from viewfinder.backend.www import server
url_fmt_string = '%s://%s:%d/fileobjstore/' % (
'https' if options.options.ssl else 'http',
ServerEnvironment.GetHost(), options.options.port)
url_fmt_string += '%s/%%s'
for store_name in (ObjectStore.PHOTO, ObjectStore.USER_LOG,
ObjectStore.USER_ZIPS):
ObjectStore.GetInstance(store_name).SetUrlFmtString(
url_fmt_string % store_name)
OpManager.SetInstance(
OpManager(op_map=DB_OPERATION_MAP, client=client, scan_ops=scan_ops))
apns_feedback_handler = Device.FeedbackHandler(client)
APNS.SetInstance(
'dev', APNS(environment='dev', feedback_handler=apns_feedback_handler))
APNS.SetInstance(
'ent', APNS(environment='ent', feedback_handler=apns_feedback_handler))
APNS.SetInstance(
'prod', APNS(environment='prod',
feedback_handler=apns_feedback_handler))
http_client = AsyncHTTPClient()
ITunesStoreClient.SetInstance(
'dev', ITunesStoreClient(environment='dev', http_client=http_client))
ITunesStoreClient.SetInstance(
'prod', ITunesStoreClient(environment='prod', http_client=http_client))
# Ensure that system users are loaded.
yield LoadSystemUsers(client)
yield gen.Task(run_callback)
def _Init(init_db=True, server_logging=True):
"""Completes Viewfinder initialization, such as secrets, DB client, AMI metadata, and the
object store.
"""
# Configure the default http client to use pycurl.
httpclient.AsyncHTTPClient.configure(
'tornado.curl_httpclient.CurlAsyncHTTPClient', max_clients=100)
# Retrieve AMI metadata before initializing secrets. Don't try to get metadata on devbox.
if options.options.devbox:
metadata = ami_metadata.Metadata()
else:
metadata = yield gen.Task(ami_metadata.Metadata)
if metadata is None:
raise Exception(
'failed to fetch AWS instance metadata; if running on dev box, '
'use the --devbox option')
ami_metadata.SetAMIMetadata(metadata)
logging.info('AMI metadata initialized')
# Initialize server environment.
ServerEnvironment.InitServerEnvironment()
logging.info('server environment initialized')
# Initialize secrets.
yield gen.Task(secrets.InitSecrets, can_prompt=sys.stderr.isatty())
logging.info('secrets initialized')
# Initialize database.
if init_db:
yield gen.Task(db_client.InitDB, vf_schema.SCHEMA)
logging.info('DB client initialized')
# Initialize object store.
object_store.InitObjectStore(temporary=False)
logging.info('object store initialized')
# Initialize the server log now that the object store is initialized.
if server_logging:
server_log.InitServerLog()
logging.info('main.py initialization complete')
def __init__(self, client, user_id, email):
super(BuildArchiveOperation, self).__init__(client)
self._user_id = user_id
self._email = email
self._notify_timestamp = self._op.timestamp
self._photo_obj_store = ObjectStore.GetInstance(ObjectStore.PHOTO)
self._user_zips_obj_store = ObjectStore.GetInstance(
ObjectStore.USER_ZIPS)
self._offboarding_assets_dir_path = ResourcesManager.Instance(
).GetOffboardingPath()
self._temp_dir_path = os.path.join(
ServerEnvironment.GetViewfinderTempDirPath(),
BuildArchiveOperation._OFFBOARDING_DIR_NAME)
self._zip_file_path = os.path.join(
self._temp_dir_path, BuildArchiveOperation._ZIP_FILE_NAME)
self._content_dir_path = os.path.join(
self._temp_dir_path, BuildArchiveOperation._CONTENT_DIR_NAME)
self._data_dir_path = os.path.join(self._content_dir_path,
CONVO_FOLDER_NAME)
def StartServer(serve_webapp=True, serve_static_web=True, serve_admin=True):
"""Initialize the datastore and operation manager with the viewfinder schema. This typically
verifies the schema. If the schema does not yet exist, it is created.
Defines settings dictionary and sets up main application with list of handlers.
"""
client = db_client.DBClient.Instance()
settings = {
'gzip':
True,
'login_url':
'/',
'admin_login_url':
'/admin/otp',
'domain':
options.options.domain,
'server_version':
options.options.server_version,
'cookie_secret':
secrets.GetSecret('cookie_secret'),
'facebook_api_key':
secrets.GetSecret('facebook_api_key'),
'facebook_secret':
secrets.GetSecret('facebook_secret'),
'google_client_id':
secrets.GetSecret('google_client_id'),
'google_client_secret':
secrets.GetSecret('google_client_secret'),
'google_client_mobile_id':
secrets.GetSecret('google_client_mobile_id'),
'google_client_mobile_secret':
secrets.GetSecret('google_client_mobile_secret'),
'template_path':
ResourcesManager.Instance().template_path,
'ui_modules':
uimodules,
'xsrf_cookies':
options.options.enable_xsrf,
'debug':
options.options.server_debug,
'static_path':
ResourcesManager.Instance().static_path,
}
if options.options.log_file_prefix:
settings['logs_dir'] = os.path.dirname(options.options.log_file_prefix)
# Configure metrics uploading.
if options.options.upload_metrics:
for interval in metric.METRIC_INTERVALS:
metric.Metric.StartMetricUpload(client,
metric.DEFAULT_CLUSTER_NAME,
interval)
# Setup application and SSL HTTP server.
handlers = deepcopy(COMMON_HANDLERS)
if serve_webapp:
# Configure web application handlers.
webapp_handlers = deepcopy(WEBAPP_HANDLERS)
# Initialize the file object store if specified.
obj_store = ObjectStore.GetInstance(ObjectStore.PHOTO)
settings['obj_store'] = obj_store
if options.options.fileobjstore:
for store_name, content_type in ((ObjectStore.PHOTO,
r'image/jpeg'),
(ObjectStore.USER_LOG,
r'text/plain'),
(ObjectStore.USER_ZIPS,
r'application/zip')):
webapp_handlers.append(
(r'/fileobjstore/%s/(.*)' % store_name,
file_object_store.FileObjectStoreHandler, {
'storename': store_name,
'contenttype': content_type
}))
if ServerEnvironment.IsDevBox():
webapp_handlers.append((r'/(link|login|register)/fakeviewfinder',
auth_viewfinder.FakeAuthViewfinderHandler))
# Set the testing directories.
if options.options.testing_path is not None:
webapp_handlers.append(
(r'/testing/hook/(.*)', test_hook.TestHookHandler))
webapp_handlers.append(
(r'/testing/static/(.*)', web.StaticFileHandler, {
'path': '%s' % options.options.testing_path
}))
handlers.extend(webapp_handlers)
if serve_static_web:
# Configure static web handlers.
static_web_handlers = deepcopy(STATIC_WEB_HANDLERS)
handlers.extend(static_web_handlers)
if serve_admin:
# Configure and verify admin handlers.
admin_handlers = deepcopy(ADMIN_HANDLERS)
for path, handler in admin_handlers:
if not issubclass(handler, basic_auth.BasicAuthHandler):
raise TypeError('Administration handlers must '
'subclass BasicAuthHandler')
handlers.extend(admin_handlers)
# Catch-all handler for 404 pages.
handlers.extend([(r'/.*', base.PageNotFoundHandler)])
# Create application and separately add handlers for the short domain and the
# regular domain.
#
# Note that, although the short-domain handlers are added after the initial construction
# of the Application, those routes will take priority over the routes in the handlers
# array.
application = web.Application(handlers, **settings)
application.add_handlers(re.escape(options.options.short_domain),
SHORT_DOMAIN_HANDLERS)
# Start the HTTP server.
http_server = httpserver.HTTPServer(
application,
xheaders=options.options.xheaders,
ssl_options={
'certfile': secrets.GetSecretFile('%s.crt' % settings['domain']),
'keyfile': secrets.GetSecretFile('%s.key' % settings['domain']),
} if options.options.ssl else None)
with stack_context.NullContext():
http_server.listen(options.options.port)
# Setup redirect server for HTTP -> HTTPS.
if options.options.ssl:
http_settings = {
'host': ServerEnvironment.GetHost(),
'redirect_port': options.options.redirect_port,
'xheaders': options.options.xheaders,
}
redirect_handlers = [
(r'/(.*)', index.RedirectHandler),
]
redirect_server = httpserver.HTTPServer(
web.Application(redirect_handlers, **http_settings))
with stack_context.NullContext():
redirect_server.listen(options.options.insecure_port)
# Ensure that system users have been created if running with a local db (needs server to be running).
if options.options.localdb:
yield CreateSystemUsers(client)
# Run the server until it hits an exception or stop signal.
yield gen.Task(lambda callback: None)
def post(self, action):
if not ServerEnvironment.IsDevBox():
raise web.HTTPError(403, _TEST_HOOKS_NOT_SUPPORTED)
from PIL import Image, ImageChops
if action == 'copy':
logging.info('Updating baseline image')
urls = {}
body = json.loads(self.request.body)
testname = body['testname']
imagename = body['imagename']
scheme = body['scheme']
_FULL_RESULTS_BASELINE = '%s/results/baseline/%s' % (options.options.testing_path, scheme)
_FULL_RESULTS_CURRENT = '%s/results/current/%s' % (options.options.testing_path, scheme)
# Overwrite the 'baseline' image for the test with the 'current' image.
baseline_image = r'%s/%s/%s' % (_FULL_RESULTS_BASELINE, testname, imagename)
current_image = r'%s/%s/Run 1/%s' % (_FULL_RESULTS_CURRENT, testname, imagename)
yield self._UpdateImageMaskConfig(testname, imagename, scheme)
if os.path.exists(current_image):
shutil.copy(current_image, baseline_image)
logging.info('Updated baseline image for %s' % testname)
baseline_web_image = r'%s/%s/%s/%s' % (_STATIC_RESULTS_BASELINE, scheme, testname, imagename)
current_web_image = r'%s/%s/%s/Run 1/%s' % (_STATIC_RESULTS_CURRENT, scheme, testname, imagename)
urls['baseline'] = baseline_web_image
urls['current'] = current_web_image
# Return JSON result.
self.write(urls)
self.finish()
return
if action == 'delete':
body = json.loads(self.request.body)
testname = body['testname']
imagename = body['imagename']
current_image = r'%s/%s/Run 1/%s' % (_FULL_RESULTS_CURRENT, testname, imagename)
if os.path.exists(current_image) is True:
os.remove(current_image)
logging.info('Deleted current capture image for %s' % testname)
self.finish()
return
if action == 'token':
body = json.loads(self.request.body)
identity_key = body['auth_info']['identity'];
identity = yield gen.Task(Identity.Query, self._client, identity_key, None, must_exist=False)
if identity is None:
raise web.HTTPError(400, 'Identity does not exist.')
self.write(identity.access_token)
self.finish()
return
if action == 'image':
body = json.loads(self.request.body)
test_name = body['testname']
image_name = body['imagename']
scheme = body['scheme']
_FULL_RESULTS_BASELINE = '%s/results/baseline/%s' % (options.options.testing_path, scheme)
_FULL_RESULTS_CURRENT = '%s/results/current/%s' % (options.options.testing_path, scheme)
# get image base name
tmp = image_name[:-4]
base, num = tmp.split('|', 1)
image_base_name = '%s.png' % base
image1 = r'%s/%s/%s' % (_FULL_RESULTS_BASELINE, test_name, image_base_name)
image2 = r'%s/%s/Run 1/%s' % (_FULL_RESULTS_CURRENT, test_name, image_name)
if os.path.exists(image1) and os.path.exists(image2):
self.set_header('Content-Type', 'application/json; charset=UTF-8')
im1 = Image.open(image1)
im2 = Image.open(image2)
diff = ImageChops.difference(im2, im1)
result = diff.getbbox() is None
response = { 'response': result, 'bbox': diff.getbbox() }
self.write(response)
self.finish()
return
raise web.HTTPError(400, _TEST_ACTION_NOT_SUPPORTED)
def InitAndRun():
"""Start the watchdog process. This will enter a loop until the process is terminate via an appropriate
OS signal.
"""
options.parse_command_line()
# Set up process signal handlers.
def _OnSignal(signum, frame):
logging.info('process stopped with signal %d' % signum)
Shutdown()
signal.signal(signal.SIGHUP, _OnSignal)
signal.signal(signal.SIGINT, _OnSignal)
signal.signal(signal.SIGQUIT, _OnSignal)
signal.signal(signal.SIGTERM, _OnSignal)
# Configure the default logger.
logging.getLogger().setLevel(logging.INFO)
logging.getLogger().handlers[0].setLevel(logging.INFO)
# Initialize the server environment in order to derive the host name.
ServerEnvironment.InitServerEnvironment()
# Initialize the Watchdog object.
watchdog = Watchdog(ScenarioDevice('user1'), SCENARIO_LIST)
# Initialize Daemon manager.
# The lockfile is stored in the current user's home directory in order
# to avoid running with root permissions, which should be unnecessary.
lock_file = os.path.expanduser(LOCK_FILE_NAME)
if not os.path.exists(os.path.dirname(lock_file)):
os.makedirs(os.path.dirname(lock_file))
dm = daemon.DaemonManager(lock_file)
_setup_error = [False]
def _OnInitException(t, v, tb):
logging.error('Exception during watchdog initialization.',
exc_info=(t, v, tb))
_setup_error[0] = True
Shutdown()
def _OnInitIOLoop(shutdown_callback):
SMSManager.SetInstance(SMSManager())
watchdog.Run(shutdown_callback)
def _OnInitDaemon(shutdown_callback):
_StartIOLoop(partial(_OnInitIOLoop, shutdown_callback))
def _OnAuthenticated(auth_complete):
if not auth_complete:
print 'Waiting for user to authorize...'
cb = partial(watchdog.device.PollForAuthentication,
_OnAuthenticated)
ioloop.IOLoop.current().add_timeout(time.time() + 15, cb)
else:
# Close the IOLoop, which will proceed to daemon setup.
io_loop = ioloop.IOLoop.current()
io_loop.stop()
io_loop.close()
def _OnGetUserCode(user_code, verification_url):
print 'Please visit url:\n %s\n and input user code:\n %s\n to authorize scenario login.' \
% (verification_url, user_code)
_OnAuthenticated(False)
def _InitWatchdog():
if watchdog.device.IsAuthenticated():
# Auth credentials were loaded from file.
_OnAuthenticated(True)
else:
# Get a user code and begin the login process.
watchdog.device.GetUserCode(_OnGetUserCode)
def _InitSecrets():
with util.ExceptionBarrier(_OnInitException):
secrets.InitSecrets(_InitWatchdog, can_prompt=sys.stderr.isatty())
if options.options.daemon.lower() == 'stop':
# Short circuit the program if the daemon is being stopped.
dm.SetupFromCommandLine(util.NoCallback, util.NoCallback)
else:
# The IOLoop will be stopped before entering daemon context.
# This is because the file descriptors for kqueue cannot be easily preserved.
_StartIOLoop(_InitSecrets)
if not _setup_error[0]:
dm.SetupFromCommandLine(_OnInitDaemon, Shutdown)