def get_counter_data(self, counter_names, start_date=None, end_date=None):
"""Get a date range of stored counter data.
Args:
counter_names: An iterable of counter names.
start_date: A datetime.date object. Defaults to the current day.
end_date: A datetime.date object. Defaults to current day.
Raises:
ValueError: if end time is greater than start time.
Returns:
A dictionary mapping counter_names to a list of counter data. For example:
{
'page/view': [(<window>, <value>), (<window>, <value>), ...],
}
"""
if end_date and end_date < start_date:
raise ValueError('End time %s must be greater than start time %s' %
(end_date, start_date))
now = datetime.datetime.now()
if not start_date:
start_date = now
if not end_date:
end_date = now
# Convert to datetime objects for the queries:
start_date = datetime.datetime(start_date.year, start_date.month,
start_date.day)
end_date = datetime.datetime(end_date.year, end_date.month,
end_date.day)
# Get all files within the range.
final_counter_data = {}
for counter_name in counter_names:
filters = [
files.FileProperty('stats_counter_name') == counter_name,
files.FileProperty('stats_date') >= start_date,
files.FileProperty('stats_date') <= end_date,
]
titan_files = files.Files.list(BASE_DIR,
recursive=True,
filters=filters,
_internal=True)
for titan_file in titan_files.itervalues():
# Since JSON only represents lists, convert each inner-list back
# to a two-tuple with the proper types.
raw_data = json.loads(titan_file.content)
counter_data = []
for data in raw_data:
counter_data.append(tuple(data))
if not counter_name in final_counter_data:
final_counter_data[counter_name] = []
final_counter_data[counter_name].extend(counter_data)
# Keep the counter data sorted by window.
for counter_data in final_counter_data.itervalues():
counter_data.sort(key=lambda tup: tup[0])
return final_counter_data
def get_activities(self,
keys,
user=None,
start=None,
end=None,
filters=None,
limit=None,
offset=None,
order=None):
"""Query for a set of stored activities.
Args:
keys: A list of keys to filter on.
user: A user to filter on.
start: A datetime.datetime object. Defaults to the previous hour.
end: A datetime.datetime object. Defaults to current day.
filters: A list of filters to apply to the activity retrieval.
order: An iterable of ActivityProperty objects to sort the result set.
Raises:
ValueError: if end time is greater than start time.
Returns:
An array of matched activities.
"""
if end and end < start:
raise ValueError(
'End time {} must be greater than start time {}'.format(
end, start))
now = datetime.datetime.now()
if not start:
start = now - datetime.timedelta(hours=1)
if not end:
end = now
# Get all activities within the range.
titan_files = files.OrderedFiles([])
for key in keys:
filters = filters or []
filters += [
files.FileProperty('activity_keys') == key,
files.FileProperty('created') >= start,
files.FileProperty('created') <= end,
]
if user:
filters.append(files.FileProperty('user') == user.email)
new_titan_files = files.OrderedFiles.list(BASE_DIR,
recursive=True,
filters=filters,
order=order,
limit=limit,
offset=offset,
_internal=True)
titan_files.update(new_titan_files)
final_results = []
for titan_file in titan_files.itervalues():
final_results.append(self._loads_activity(titan_file.content))
return final_results
def failed_tasks(self):
filters = [files.FileProperty('status') == STATUS_FAILED]
titan_files = files.Files.list(self._tasks_dir_path, filters=filters)
tasks = set()
for titan_file in titan_files.itervalues():
tasks.add(Task(self, _internal_key=titan_file.name_clean))
return tasks
def num_failed(self):
filters = [files.FileProperty('status') == STATUS_FAILED]
return files.Files.count(self._tasks_dir_path, filters=filters)
def num_successful(self):
filters = [files.FileProperty('status') == STATUS_SUCCESSFUL]
return files.Files.count(self._tasks_dir_path, filters=filters)