def RunOnce(callback):
"""Get list of files and call processing function."""
dry_run = options.options.dry_run
logs_paths = logs_util.ServerLogsPaths('viewfinder', 'full')
if dry_run:
logging.warning('dry_run=True: will not upload processed logs files or update registry')
logs_store = ObjectStore.GetInstance(logs_paths.SOURCE_LOGS_BUCKET)
merged_store = ObjectStore.GetInstance(logs_paths.MERGED_LOGS_BUCKET)
# Fetch list of raw logs files.
files = yield gen.Task(GetRawLogsFileList, logs_store, logs_paths, options.options.start_date)
# Fetch list of processed logs files from registry.
processed_files = yield gen.Task(logs_util.GetRegistry, merged_store, logs_paths.ProcessedRegistryPath())
if processed_files is None:
# None means: registry does not exist. All other errors throw exceptions.
processed_files = []
# Compute list of raw files to process (and sort by filename -> sort by date).
files_set = set(files)
processed_set = set(processed_files)
missing_files = list(files_set.difference(processed_set))
missing_files.sort()
to_process = missing_files
if options.options.max_files_to_process is not None:
to_process = missing_files[0:options.options.max_files_to_process]
logging.info('found %d raw files and %d processed files, %d missing. Will process %d.' %
(len(files), len(processed_files), len(missing_files), len(to_process)))
if len(missing_files) == 0:
logging.info('No raw logs files to process.')
callback()
return
merged_files = yield gen.Task(ProcessFiles, logs_store, merged_store, logs_paths, to_process, dry_run)
logging.info('found %d raw files and %d processed files, %d missing, successfully processed %d' %
(len(files), len(processed_files), len(missing_files), len(merged_files)))
# Add processed files to registry and write to S3.
# TODO(marc): any failure in merged log upload or registry upload will cause us to get out of sync. To fix this,
# we should also have a list of properly applied processed logs.
processed_files.extend(merged_files)
processed_files.sort()
if not dry_run:
yield gen.Task(retry.CallWithRetryAsync, kS3UploadRetryPolicy,
logs_util.WriteRegistry, merged_store, logs_paths.ProcessedRegistryPath(), processed_files)
callback()
def RunOnce(client, job, callback):
"""Get list of files and call processing function."""
logs_paths = logs_util.ServerLogsPaths('viewfinder', 'full')
merged_store = ObjectStore.GetInstance(logs_paths.MERGED_LOGS_BUCKET)
start_date = options.options.start_date
if options.options.smart_scan:
# Search for successful full-scan run in the last week.
last_run = yield gen.Task(job.FindLastSuccess, with_payload_key='stats.last_day')
if last_run is None:
logging.info('No previous successful scan found, rerun with --start_date')
callback(None)
return
last_run_start = last_run['start_time']
if util.HoursSince(last_run_start) < options.options.hours_between_runs:
logging.info('Last successful run started at %s, less than %d hours ago; skipping.' %
(time.asctime(time.localtime(last_run_start)), options.options.hours_between_runs))
callback(None)
return
last_day = last_run['stats.last_day']
# Set scan_start to start of previous run - 1d. The extra 1d is in case some logs were pushed to S3 late.
# This really recomputes two days (the last day that was successfully processed and the one prior).
start_time = util.ISO8601ToUTCTimestamp(last_day, hour=12) - constants.SECONDS_PER_DAY
start_date = util.TimestampUTCToISO8601(start_time)
logging.info('Last successful analyze_logs run (%s) scanned up to %s, setting analysis start date to %s' %
(time.asctime(time.localtime(last_run_start)), last_day, start_date))
# Fetch list of merged logs.
files = yield gen.Task(GetMergedLogsFileList, merged_store, logs_paths, start_date)
last_day = yield gen.Task(ProcessFiles, merged_store, logs_paths, files)
callback(last_day)
return
def _UploadWelcomePhotos(http_client, client, user, upload_request):
"""Uploads a set of photos that will be used in the new user welcome conversation. These
photos are uploaded to the given user account. "upload_request" is in the UPLOAD_EPISODE_REQUEST
format in json_schema.py, except:
1. Activity, episode, and photo ids are added by this method.
2. Each photo dict must contain an additional "name" field which gives the start of the
filename of a jpg file in the backend/resources/welcome directory. Three files must
exist there, in this format: <name>_full.jpg, <name>_med.jpg, <name>_tn.jpg.
"""
obj_store = ObjectStore.GetInstance(ObjectStore.PHOTO)
welcome_path = os.path.join(ResourcesManager.Instance().resources_path,
'welcome')
# Set the ids of all activities, episodes, and photos in the welcome conversation.
yield _SetWelcomeIds(user, upload_request)
# Get copy and strip out names, which UploadEpisode chokes on.
upload_request = deepcopy(upload_request)
# Directly call the service API in order to upload the photo.
upload_request_copy = deepcopy(upload_request)
[ph_dict.pop('name') for ph_dict in upload_request_copy['photos']]
upload_response = yield UploadEpisode(client, obj_store, user.user_id,
user.webapp_dev_id,
upload_request_copy)
# Upload photo to blob store (in various formats).
for request_ph_dict, response_ph_dict in zip(upload_request['photos'],
upload_response['photos']):
for format in ('full', 'med', 'tn'):
# Get the photo bits from disk.
f = open(
os.path.join(welcome_path,
'%s_%s.jpg' % (request_ph_dict['name'], format)),
'r')
image_data = f.read()
f.close()
photo_url = response_ph_dict[format + '_put_url']
content_md5 = base64.b64encode(
request_ph_dict[format + '_md5'].decode('hex'))
headers = {
'Content-Type': 'image/jpeg',
'Content-MD5': content_md5
}
validate_cert = not options.options.fileobjstore
response = yield gen.Task(http_client.fetch,
photo_url,
method='PUT',
body=image_data,
follow_redirects=False,
validate_cert=validate_cert,
headers=headers)
if response.code != 200:
raise Exception(
'Cannot upload photo "%s". HTTP error code %d. Is server running and accessible?'
% (request_ph_dict['photo_id'], response.code))
def __init__(self, client, user_id, email):
super(BuildArchiveOperation, self).__init__(client)
self._user_id = user_id
self._email = email
self._notify_timestamp = self._op.timestamp
self._photo_obj_store = ObjectStore.GetInstance(ObjectStore.PHOTO)
self._user_zips_obj_store = ObjectStore.GetInstance(
ObjectStore.USER_ZIPS)
self._offboarding_assets_dir_path = ResourcesManager.Instance(
).GetOffboardingPath()
self._temp_dir_path = os.path.join(
ServerEnvironment.GetViewfinderTempDirPath(),
BuildArchiveOperation._OFFBOARDING_DIR_NAME)
self._zip_file_path = os.path.join(
self._temp_dir_path, BuildArchiveOperation._ZIP_FILE_NAME)
self._content_dir_path = os.path.join(
self._temp_dir_path, BuildArchiveOperation._CONTENT_DIR_NAME)
self._data_dir_path = os.path.join(self._content_dir_path,
CONVO_FOLDER_NAME)
def RunOnce(client, callback):
object_store = ObjectStore.GetInstance(ObjectStore.SERVER_DATA)
filenames = {}
for num_days in options.options.analysis_intervals_days:
filename = yield gen.Task(ProcessOneInterval, client, num_days)
filenames[num_days] = filename
yield gen.Task(UploadFiles, object_store, filenames.values())
yield gen.Task(SendReport, object_store, filenames)
callback()
def _CreateFormats():
"""Used to set up initial photos."""
obj_store = ObjectStore.GetInstance(ObjectStore.PHOTO)
client = DBClient.Instance()
http_client = AsyncHTTPClient()
for photo_id, name in [('pgAZn77bJ-Kc', 'beach_c4'),
('pgAzpz7bJ-Mc', 'beach_a1'),
('pgB-Fh7bJ-Mg', 'beach_a2'),
('pgAzo67bJ-MV', 'beach_a3'),
('pgB-pj7bJ-Mo', 'beach_a4'),
('pgAvIa7bJ-MN', 'beach_b1'),
('pgAuoQ7bJ-MF', 'beach_b2'),
('pgAtwd7bJ-M7', 'beach_b3'),
('pgAaOJ7bJ-Kw', 'beach_c1'),
('pgA_vm7bJ-Ko', 'beach_c2'),
('pgAZna7bJ-Kk', 'beach_c3'),
('pgAW0x7bJ-KV', 'beach_d1'),
('pgAUMm7bJ-KN', 'beach_d2'),
('pfYwYR7bJ-KJ', 'party_1'),
('pfYwTk7bJ-KF', 'party_2'),
('pfYwSo7bJ-K7', 'party_3'),
('pfYw0g7bJ-K-', 'party_4'),
('pfYvoK7bJ-Jw', 'party_5'),
('pfYvhI7bJ-Jo', 'party_6'),
('prHKwa7bJ-N30', 'gone_fishing_1'),
('prBUtl7bJ-Mw', 'gone_fishing_2'),
('pfSb0S7bJ-Jk', 'street_art_1'),
('pfSasJ7bJ-Jc', 'street_art_2')]:
photo = yield Photo.Query(client, photo_id, None)
photo_dict = photo._asdict()
photo_dict['name'] = name
del photo_dict['photo_id']
del photo_dict['user_id']
del photo_dict['_version']
del photo_dict['episode_id']
print json.dumps(photo_dict, indent=True)
for suffix, format in [('.f', 'full'), ('.m', 'med'),
('.t', 'tn')]:
url = obj_store.GenerateUrl('%s%s' % (photo_id, suffix))
response = yield http_client.fetch(url, method='GET')
welcome_path = os.path.join(
ResourcesManager.Instance().resources_path, 'welcome')
f = open(
os.path.join(welcome_path, '%s_%s.jpg' % (name, format)),
'w')
f.write(response.body)
f.close()
def _RunService(callback):
"""Invokes user account merge utility."""
assert options.options.method, 'must specify a service method (--method)'
assert options.options.user_id, 'must specify a user id (--user_id)'
# Read request body from standard in.
if sys.stdin.isatty():
print 'Enter JSON-encoded service request:'
request_body = sys.stdin.read()
# If version was not specified, add it now (use max supported version).
request_dict = json.loads(request_body)
if not request_dict.has_key('headers'):
request_dict['headers'] = dict()
if not request_dict['headers'].has_key('version'):
request_dict['headers']['version'] = MAX_SUPPORTED_MESSAGE_VERSION
client = DBClient.Instance()
obj_store = ObjectStore.GetInstance(ObjectStore.PHOTO)
def _OnService(response_dict):
logging.info('result: %s' %
util.ToCanonicalJSON(response_dict, indent=2))
callback()
def _OnQueryUser(user):
context = base.ViewfinderContext(None)
context.user = user
context.device_id = user.webapp_dev_id if options.options.device_id is None else options.options.device_id
with stack_context.StackContext(context):
ServiceHandler.InvokeService(client,
obj_store,
options.options.method,
context.user.user_id,
context.device_id,
request_dict,
callback=_OnService)
User.Query(client, options.options.user_id, None, _OnQueryUser)
def RunOnce(client, job, callback):
"""Get list of files and call processing function."""
merged_store = ObjectStore.GetInstance(logs_util.UserAnalyticsLogsPaths.MERGED_LOGS_BUCKET)
start_date = options.options.start_date
if options.options.smart_scan:
# Search for successful full-scan run in the last week.
last_run = yield gen.Task(job.FindLastSuccess, with_payload_key='stats.last_day')
if last_run is None:
logging.info('No previous successful scan found, rerun with --start_date')
callback(None)
return
last_run_start = last_run['start_time']
if util.HoursSince(last_run_start) < options.options.hours_between_runs:
logging.info('Last successful run started at %s, less than %d hours ago; skipping.' %
(time.asctime(time.localtime(last_run_start)), options.options.hours_between_runs))
callback(None)
return
last_day = last_run['stats.last_day']
# Set scan_start to start of previous run - 30d (we need 30 days' worth of data to properly compute
# 30-day active users. Add an extra 3 days just in case we had some missing logs during the last run.
start_time = util.ISO8601ToUTCTimestamp(last_day, hour=12) - constants.SECONDS_PER_WEEK
start_date = util.TimestampUTCToISO8601(start_time)
logging.info('Last successful analyze_analytics run (%s) scanned up to %s, setting analysis start date to %s' %
(time.asctime(time.localtime(last_run_start)), last_day, start_date))
# Fetch list of merged logs.
files = yield gen.Task(GetMergedLogsFileList, merged_store, start_date)
day_stats = yield gen.Task(ProcessFiles, merged_store, files)
# Write per-day stats to dynamodb.
if len(day_stats) > 0:
hms = logs_util.kDailyMetricsTimeByLogType['analytics_logs']
yield gen.Task(logs_util.UpdateMetrics, client, day_stats, dry_run=options.options.dry_run, hms_tuple=hms)
last_day = sorted(day_stats.keys())[-1]
callback(last_day)
else:
callback(None)
def RunOnce(callback):
"""Get list of files and call processing function."""
dry_run = options.options.dry_run
client_store = ObjectStore.GetInstance(logs_util.UserAnalyticsLogsPaths.SOURCE_LOGS_BUCKET)
if options.options.user:
users = [options.options.user]
else:
users = yield gen.Task(logs_util.ListClientLogUsers, client_store)
examined = 0
for u in users:
# Running all users in parallel can get us to exceed the open FD limit.
if options.options.start_user is not None and u < options.options.start_user:
continue
if options.options.max_users is not None and examined > options.options.max_users:
break
examined += 1
yield gen.Task(HandleOneUser, client_store, u)
if dry_run:
logging.warning('dry_run=True: will not upload processed logs files or update registry')
callback()