def testNamespaces(self):
files.register_file_mixins([dirs.DirManagerMixin])
self.assertIsNone(dirs.Dir('/').namespace)
self.assertEqual('aaa', dirs.Dir('/', namespace='aaa').namespace)
files.File('/a/a/foo', namespace='aaa').write('')
files.File('/z/z/foo').write('')
self.assertEqual(['/a'], dirs.Dirs.list('/', namespace='aaa').keys())
self.assertEqual(['/z'], dirs.Dirs.list('/').keys())
def _write_microversion(changeset, file_kwargs, method_kwargs, email, action):
"""Task to enqueue for microversioning a file action."""
# Set the _internal flag for all microversion operations.
file_kwargs['_internal'] = True
file_kwargs['changeset'] = changeset
method_kwargs['_delete_old_blob'] = False
if email:
method_kwargs['created_by'] = users.TitanUser(email)
method_kwargs['modified_by'] = users.TitanUser(email)
if action == _Actions.WRITE:
files.File(**file_kwargs).write(**method_kwargs)
elif action == _Actions.DELETE:
files.File(**file_kwargs).delete(**method_kwargs)
def testStronglyConsistentCommits(self):
# Microversions uses finalize_associated_files so the commit() path should
# use the always strongly-consistent get_files(), rather than a query.
# Verify this behavior by simulating a never-consistent HR datastore.
policy = datastore_stub_util.PseudoRandomHRConsistencyPolicy(probability=0)
self.testbed.init_datastore_v3_stub(consistency_policy=policy)
files.File('/foo').write('foo')
# Also, test process_data_with_backoff while we're here.
results = microversions.process_data_with_backoff(timeout_seconds=5)
final_changeset = results[0][0]['changeset'].linked_changeset
self.assertEqual(2, final_changeset.num)
titan_file = files.File('/foo', changeset=final_changeset)
self.assertEqual('foo', titan_file.content)
def testLog(self):
activity = activities.log('titan.test')
activities.process_activity_loggers()
self.RunDeferredTasks(queue_name=activities.ACTIVITY_QUEUE)
# Verify file was written.
self.assertTrue(
files.File(activity.activity_id, _internal=True).exists)
def get(self):
"""GET handler."""
path = self.request.get('path')
try:
file_kwargs, _ = _GetExtraParams(self.request.POST)
except ValueError:
self.error(400)
return
try:
titan_file = files.File(path, **file_kwargs)
except (TypeError, ValueError):
self.error(400)
_MaybeLogException('Bad request:')
return
if not titan_file.exists:
self.error(404)
return
mime_type = self.request.get('mime_type') or titan_file.mime_type
self.response.headers['Content-Type'] = str(mime_type)
self.response.headers['Content-Disposition'] = (
'inline; filename=%s' % titan_file.name.encode('ascii', 'replace'))
if titan_file.blob:
blob_key = titan_file.blob
self.send_blob(blob_key, content_type=str(mime_type))
else:
self.response.out.write(titan_file.content)
def testRemoteVersionControlService(self):
remote_vcs = self.remote_vcs_factory.make_remote_vcs()
remote_changeset = remote_vcs.new_staging_changeset()
# Error handling: writing without a changeset.
remote_file = self.remote_file_factory.make_remote_file('/a/foo')
# TODO(user): build more specific remote objects error handling.
self.assertRaises(Exception, remote_file.write, 'foo!')
remote_file = self.remote_file_factory.make_remote_file(
'/a/foo', changeset=remote_changeset.num)
remote_file.write('foo!')
remote_changeset.associate_file(remote_file)
# Error handling: changeset's files are not finalized.
self.assertRaises(versions_client.RemoteChangesetError,
remote_vcs.commit, remote_changeset)
self.assertRaises(versions_client.RemoteChangesetError,
remote_changeset.get_files)
remote_changeset.finalize_associated_files()
remote_vcs.commit(remote_changeset)
self.assertEqual(['/a/foo'], remote_changeset.get_files().keys())
# Verify actual state.
actual_file = files.File('/a/foo')
self.assertEqual('foo!', actual_file.content)
self.assertTrue(actual_file.changeset)
# Test Commit(force=True).
remote_changeset = remote_vcs.new_staging_changeset()
self.assertRaises(Exception,
remote_vcs.commit,
remote_changeset,
force=True)
remote_file = self.remote_file_factory.make_remote_file(
'/a/foo', changeset=remote_changeset.num)
remote_file.write('bar!')
# Don't call associate_file or finalize_associated_files.
remote_vcs.commit(remote_changeset, force=True)
actual_file = files.File('/a/foo')
self.assertEqual('bar!', actual_file.content)
self.assertTrue(actual_file.changeset)
# Test MakeRemoteChangeset().
num = actual_file.changeset.num
remote_changeset = self.remote_vcs_factory.make_remote_changeset(num)
def __init__(self, key):
self.key = key
self._internal_client_ids = set()
# Use a hashed version of the user key for the filename.
self._internal_key = hashlib.md5(key).hexdigest()
self._file_path = '%s%s' % (CHANNELS_DIR, self.key)
self._file = files.File(self._file_path, _internal=True)
def testContentAndBlobsHandling(self):
files.File('/foo').write('foo')
files.File('/foo').delete()
# This will immediately go to blobstore, then the deferred task will
# have a "blob" argument:
files.File('/foo').write(LARGE_FILE_CONTENT)
process_microversions()
# After tasks run, verify correct content was saved to the versioned paths:
file_versions = self.vcs.get_file_versions('/foo')
# In reverse-chronological order:
titan_file = files.File('/foo', changeset=file_versions[0].changeset)
self.assertEqual(LARGE_FILE_CONTENT, titan_file.content)
self.assertEqual('*****@*****.**', str(titan_file.created_by))
self.assertEqual('*****@*****.**', str(titan_file.modified_by))
# Whitebox test: created_by and modified_by might be coming from the
# backwards-compatibility code in versions. Verify they are actually
# stored correctly.
self.assertEqual('*****@*****.**', str(titan_file._file.created_by))
self.assertEqual('*****@*****.**', str(titan_file._file.modified_by))
titan_file = files.File('/foo', changeset=file_versions[1].changeset)
self.assertFalse(titan_file.exists)
titan_file = files.File('/foo', changeset=file_versions[2].changeset)
self.assertEqual('foo', titan_file.content)
self.assertEqual(versions.FileStatus.created, file_versions[0].status)
self.assertEqual(versions.FileStatus.deleted, file_versions[1].status)
self.assertEqual(versions.FileStatus.created, file_versions[2].status)
# Verify that this doesn't error, the behavior should be the same as above.
files.File('/foo').write(EXPLODING_FILE_CONTENT)
process_microversions()
def testRemoteFile(self):
remote_file = self.remote_file_factory.make_remote_file('/a/foo')
actual_file = files.File('/a/foo')
# Error handling for non-existent files.
self.assertRaises(files_client.BadRemoteFileError,
lambda: remote_file.name)
self.assertEqual(actual_file.exists, remote_file.exists)
actual_file.write('foo!', meta={'flag': False})
# Test properties.
self.assertEqual(actual_file.name, remote_file.name)
self.assertEqual(actual_file.path, remote_file.path)
self.assertEqual(actual_file.real_path, remote_file.real_path)
self.assertEqual(actual_file.paths, remote_file.paths)
self.assertEqual(actual_file.mime_type, remote_file.mime_type)
self.assertEqual(actual_file.created, remote_file.created)
self.assertEqual(actual_file.modified, remote_file.modified)
self.assertEqual(actual_file.size, remote_file.size)
self.assertEqual(actual_file.content, remote_file.content)
self.assertEqual(actual_file.exists, remote_file.exists)
# Different remote API (string of blob_key rather than BlobKey):
self.assertEqual(actual_file.serialize()['blob'], remote_file.blob)
# Different remote API (string of email addresses rather than User objects).
self.assertEqual(actual_file.serialize()['created_by'],
remote_file.created_by)
self.assertEqual(actual_file.serialize()['modified_by'],
remote_file.modified_by)
self.assertEqual(actual_file.md5_hash, remote_file.md5_hash)
self.assertEqual(actual_file.meta, remote_file.meta)
# Test write().
remote_file.write(content='bar')
actual_file = files.File('/a/foo')
self.assertEqual('bar', remote_file.content)
self.assertEqual('bar', actual_file.content)
# Test delete().
remote_file.delete()
actual_file = files.File('/a/foo')
self.assertRaises(files_client.BadRemoteFileError,
lambda: remote_file.name)
self.assertFalse(actual_file.exists)
self.assertFalse(remote_file.exists)
def _file(self):
if self._internal_file is None:
# /_titan/tasks/<group>/<task_manager_key>/tasks/<task_key_hash>.json
filename = utils.safe_join(_ROOT_DIR_PATH,
self._task_manager.group,
self._task_manager.key, 'tasks',
self._internal_key + '.json')
self._internal_file = files.File(filename, _internal=True)
return self._internal_file
def testProcessActivityLoggers(self):
user = users.TitanUser('*****@*****.**')
activity = activities.Activity('key', user=user, meta='meta')
activity_logger = activities.FileActivityLogger(activity)
activity_logger.store()
# Ensure that it runs normally.
activities.process_activity_loggers()
self.RunDeferredTasks(queue_name=activities.ACTIVITY_QUEUE)
# Verify file was written.
self.assertTrue(
files.File(activity.activity_id, _internal=True).exists)
def finalize(self):
"""Log the activity in Titan Files."""
titan_file = files.File(self.activity.activity_id, _internal=True)
titan_file.write(content=protojson.encode_message(
self.activity.to_message()),
meta=self.file_meta,
created=self.activity.timestamp,
modified=self.activity.timestamp)
# Ensure that it gets written first.
super(FileActivityLogger, self).finalize()
def testJsonMixin(self):
titan_file = files.File('/foo/file.json')
# Verify that invalid JSON raises an error when written.
self.assertRaises(json_mixin.BadJsonError, titan_file.write, '')
self.assertRaises(json_mixin.BadJsonError, titan_file.write, '{a:1}')
# Verify Django templates are not validated as JSON.
titan_file.write('{% some django tag %}')
# Verify valid JSON writes successfully.
titan_file.write(json.dumps({'a': True}))
# Verify behavior of json property.
titan_file.write('{"a":"b"}')
self.assertEqual({'a': 'b'}, titan_file.json)
titan_file.json['c'] = 'd'
self.assertEqual({'a': 'b', 'c': 'd'}, titan_file.json)
titan_file.save()
# Get json out of the saved file.
titan_file = files.File('/foo/file.json')
self.assertEqual({'a': 'b', 'c': 'd'}, titan_file.json)
# Verify invalid JSON raises an error when accessed using json.
titan_file = files.File('/foo/file.json')
titan_file.write('{% some django tag %}')
self.assertRaises(json_mixin.BadJsonError, lambda: titan_file.json)
# Verify JSON is not checked for blobs (out of necessity, not correctness).
titan_file = files.File('/foo/some-blob').write(LARGE_FILE_CONTENT)
files.File('/foo/file.json').write(blob=titan_file.blob.key())
# Verify ability to set json on a new file.
titan_file = files.File('/foo/new_file.json')
titan_file.json = None
self.assertIsNone(titan_file.json)
titan_file.save()
titan_file.json = {}
self.assertEqual({}, titan_file.json)
titan_file.save()
titan_file.json = {'foo': 'bar'}
titan_file.save()
titan_file = files.File('/foo/new_file.json')
self.assertEqual({'foo': 'bar'}, titan_file.json)
# Error handling.
self.assertRaises(
files.BadFileError, lambda: files.File('/fake.json').json)
def testStripPrefix(self):
files.register_file_mixins([dirs.DirManagerMixin])
titan_dir = dirs.Dir('/a/b', strip_prefix='/a')
self.assertEqual('/b', titan_dir.path)
titan_dir = dirs.Dir('/a/b', strip_prefix='/a/')
self.assertEqual('/b', titan_dir.path)
self.assertFalse(titan_dir.exists)
files.File('/a/b/foo').write('')
self.assertTrue(titan_dir.exists)
titan_dirs = dirs.Dirs.list('/a/', strip_prefix='/a/')
# Recreate object to strip paths:
titan_dirs = dirs.Dirs(dirs=titan_dirs.values())
self.assertEqual('/b', titan_dirs['/b'].path)
def testApplyPatch(self):
titan_file = files.File('/foo/file.json')
titan_file.json = {
'foo': ['bar', 'baz']
}
titan_file.apply_patch([{
'op': 'add',
'path': '/foo/1',
'value': 'qux'
}])
expected = {
'foo': ['bar', 'qux', 'baz']
}
self.assertEqual(expected, titan_file.json)
def testRemoteFiles(self):
actual_file = files.File('/a/foo')
actual_file.write('foo!', meta={'flag': False})
remote_file = self.remote_file_factory.make_remote_file('/a/foo')
remote_files = self.remote_file_factory.make_remote_files()
self.assertEqual([], remote_files.keys())
remote_files = self.remote_file_factory.make_remote_files([])
self.assertEqual([], remote_files.keys())
remote_files = self.remote_file_factory.make_remote_files(['/a/foo'])
self.assertEqual(['/a/foo'], remote_files.keys())
remote_files = self.remote_file_factory.make_remote_files(
files=[remote_file])
self.assertEqual(['/a/foo'], remote_files.keys())
# Test mapping properties.
remote_files['/a/bar'] = self.remote_file_factory.make_remote_file(
'/a/bar')
self.assertSameElements(['/a/foo', '/a/bar'], remote_files.keys())
del remote_files['/a/bar']
self.assertEqual(['/a/foo'], remote_files.keys())
self.assertIn('/a/foo', remote_files)
self.assertEqual(1, len(remote_files))
remote_files.clear()
self.assertEqual(0, len(remote_files))
# Test List().
# Different remote API: List is not a class method due to authentication.
remote_files = self.remote_file_factory.make_remote_files()
remote_files.list('/', recursive=True)
self.assertEqual(['/a/foo'], remote_files.keys())
# Test Delete().
remote_files.delete()
actual_file = files.File('/a/foo')
self.assertFalse(actual_file.exists)
def delete(self):
"""DELETE handler."""
path = self.request.get('path')
try:
file_kwargs, method_kwargs = _GetExtraParams(self.request.POST)
except ValueError:
self.error(400)
return
try:
files.File(path, **file_kwargs).delete(**method_kwargs)
except files.BadFileError:
self.error(404)
except (TypeError, ValueError):
self.error(400)
_MaybeLogException('Bad request:')
def get_activity(self, activity_id):
"""Query for a stored activity.
Args:
activity_id: The full file name of the activity.
Raises:
InvalidActivityIdError: When path is not prefixed correctly.
ActivityNotFoundError: When activity is not found in system.
Returns:
The activity matching the activity_id.
"""
if not activity_id or not activity_id.startswith(ACTIVITY_DIR):
raise InvalidActivityIdError()
titan_file = files.File(activity_id, _internal=True)
if not titan_file.exists:
raise ActivityNotFoundError()
return self._loads_activity(titan_file.content)
def testEndToEnd(self):
files.register_file_mixins([dirs.DirManagerMixin])
files.File('/a/b/foo').write('')
files.File('/a/b/bar').write('')
files.File('/a/d/foo').write('')
# List root dir.
self.assertEqual(dirs.Dirs(['/a']), dirs.Dirs.list('/'))
# List /a.
self.assertEqual(dirs.Dirs(['/a/b', '/a/d']), dirs.Dirs.list('/a/'))
# List /a/b/.
self.assertEqual(dirs.Dirs([]), dirs.Dirs.list('/a/b/'))
# List /fake/dir.
self.assertEqual(dirs.Dirs([]), dirs.Dirs.list('/fake/dir'))
# Test deleting directories.
self.assertTrue(dirs.Dir('/a/d').exists)
files.File('/a/d/foo').delete()
self.assertFalse(dirs.Dir('/a/d').exists)
# List /a.
self.assertEqual(dirs.Dirs(['/a/b']), dirs.Dirs.list('/a/'))
self.assertEqual(dirs.Dirs(['/a']), dirs.Dirs.list('/'))
# Delete the remaining files and list again.
files.File('/a/b/foo').delete()
files.File('/a/b/bar').delete()
self.assertEqual(dirs.Dirs([]), dirs.Dirs.list('/'))
# Verify behavior of set_meta and meta.
self.assertRaises(dirs.InvalidDirectoryError,
lambda: dirs.Dir('/a/b').set_meta({'flag': True}))
self.assertRaises(dirs.InvalidMetaError,
lambda: dirs.Dir('/a/b').set_meta({'name': 'foo'}))
files.File('/a/b/foo').write('')
# Also, weakly test execution path of process_windows_with_backoff.
dir_task_consumer = dirs.DirTaskConsumer()
dir_task_consumer.process_windows_with_backoff(runtime=2)
titan_dir = dirs.Dir('/a/b')
self.assertRaises(AttributeError, lambda: titan_dir.meta.flag)
titan_dir.set_meta(meta={'flag': True})
titan_dir = dirs.Dir('/a/b')
self.assertTrue(titan_dir.meta.flag)
# Verify properties.
self.assertEqual('b', titan_dir.name)
self.assertEqual('/a/b', titan_dir.path)
def testMicroversions(self):
# write.
files.File('/foo').write('foo', encoding='utf-8')
self.logout() # Mimic the cron job.
results = process_microversions()
self.login()
final_changeset = results[0]['changeset'].linked_changeset
self.assertEqual(2, final_changeset.num)
titan_file = files.File('/foo', changeset=final_changeset)
self.assertEqual(u'foo', titan_file.content)
# The final changeset's created_by should be None, because it's created
# internally in a cron job and shares multiple user writes.
self.assertEqual(None, final_changeset.created_by)
# write with an existing root file (which should be copied to the version).
files.File('/foo', _no_mixins=True).write('new foo')
files.File('/foo').write(meta={'color': 'blue'})
results = process_microversions()
final_changeset = results[0]['changeset'].linked_changeset
self.assertEqual(4, final_changeset.num)
titan_file = files.File('/foo', changeset=final_changeset)
self.assertEqual('new foo', titan_file.content)
self.assertEqual('blue', titan_file.meta.color)
# Delete. Also, this verifies that delete doesn't rely on the presence
# of the root file.
files.File('/foo').delete()
results = process_microversions()
final_changeset = results[0]['changeset'].linked_changeset
self.assertEqual(6, final_changeset.num)
titan_file = files.File('/foo', changeset=final_changeset)
self.assertFalse(titan_file.exists)
# Check file versions.
file_versions = self.vcs.get_file_versions('/foo')
self.assertEqual(6, file_versions[0].changeset.num)
self.assertEqual(4, file_versions[1].changeset.num)
self.assertEqual(2, file_versions[2].changeset.num)
self.assertEqual(versions.FileStatus.deleted, file_versions[0].status)
self.assertEqual(versions.FileStatus.edited, file_versions[1].status)
self.assertEqual(versions.FileStatus.created, file_versions[2].status)
def post(self):
"""POST handler."""
path = self.request.get('path')
# Must use str_POST here to preserve the original encoding of the string.
content = self.request.str_POST.get('content')
blob = self.request.get('blob', None)
try:
file_kwargs, method_kwargs = _GetExtraParams(self.request.POST)
except ValueError:
self.error(400)
return
if blob is not None:
# Convert any string keys to BlobKey instances.
if isinstance(blob, basestring):
blob = blobstore.BlobKey(blob)
meta = self.request.get('meta', None)
if meta:
meta = json.loads(meta)
mime_type = self.request.get('mime_type', None)
try:
files.File(path, **file_kwargs).write(content,
blob=blob,
mime_type=mime_type,
meta=meta,
**method_kwargs)
self.response.set_status(201)
# Headers must be byte-strings, not unicode strings.
# Since this is a URI, follow RFC 3986 and encode it using UTF-8.
location_header = ('/_titan/file/?path=%s' % path).encode('utf-8')
self.response.headers['Location'] = location_header
except files.BadFileError:
self.error(404)
except (TypeError, ValueError):
self.error(400)
_MaybeLogException('Bad request:')
def post(self):
"""POST handler."""
try:
namespace = self.request.get('namespace', None)
save_manifest = self.request.get('save_manifest', 'true') == 'true'
vcs = versions.VersionControlService()
staging_changeset = versions.Changeset(int(
self.request.get('changeset')),
namespace=namespace)
force = bool(self.request.get('force', False))
manifest = self.request.POST.get('manifest', None)
if not force and not manifest or force and manifest:
self.error(400)
logging.error(
'Exactly one of "manifest" or "force" params is required')
return
# If a client has full knowledge of the files uploaded to a changeset,
# the "manifest" param may be given to ensure a strongly consistent
# commit. If given, associate the files to the changeset and finalize it.
if manifest:
manifest = json.loads(manifest)
for path in manifest:
titan_file = files.File(path,
changeset=staging_changeset,
namespace=namespace,
_internal=True)
staging_changeset.associate_file(titan_file)
staging_changeset.finalize_associated_files()
final_changeset = vcs.commit(staging_changeset,
force=force,
save_manifest=save_manifest)
self.write_json_response(final_changeset)
self.response.set_status(201)
except (TypeError, ValueError):
self.error(400)
logging.exception('Bad request:')
def get(self):
"""GET handler."""
path = self.request.get('path')
full = bool(self.request.get('full'))
try:
file_kwargs, _ = _GetExtraParams(self.request.GET)
except ValueError:
self.error(400)
return
try:
titan_file = files.File(path, **file_kwargs)
except (TypeError, ValueError):
self.error(400)
_MaybeLogException('Bad request:')
return
if not titan_file.exists:
self.error(404)
return
# TODO(user): when full=True, this may fail for files with byte-string
# content.
self.write_json_response(titan_file, full=full)
def testKeepOldBlobs(self):
# Create a blob and blob_reader for testing.
filename = blobstore_files.blobstore.create(
mime_type='application/octet-stream')
with blobstore_files.open(filename, 'a') as fp:
fp.write('Blobstore!')
blobstore_files.finalize(filename)
blob_key = blobstore_files.blobstore.get_blob_key(filename)
# Verify that the blob is not deleted when microversioned content resizes.
files.File('/foo').write(blob=blob_key)
process_microversions()
titan_file = files.File('/foo')
self.assertTrue(titan_file.blob)
self.assertEqual('Blobstore!', titan_file.content)
process_microversions()
# Resize as smaller (shouldn't delete the old blob).
files.File('/foo').write('foo')
files.File('/foo').write(blob=blob_key) # Resize back to large size.
# Delete file (shouldn't delete the old blob).
files.File('/foo').delete()
process_microversions()
file_versions = self.vcs.get_file_versions('/foo')
# Deleted file (blob should be None).
changeset = file_versions[0].changeset
titan_file = files.File(
'/foo', changeset=changeset, _allow_deleted_files=True)
self.assertIsNone(titan_file.blob)
# Created file (blob key and blob content should still exist).
changeset = file_versions[-1].changeset
titan_file = files.File(
'/foo', changeset=changeset, _allow_deleted_files=True)
self.assertTrue(titan_file.blob)
self.assertEqual('Blobstore!', titan_file.content)
def testRecordStats(self):
# The following numbers should not be over-analyzed if broken, as long
# as they stay relatively low. Feel free to update if internals change.
# +2 counters (files/File/load and files/File/load/latency).
_ = files.File('/foo').exists
self.assertEqual(2, self.CountLoggers())
# +6 (internal of write() + plus load counters).
files.File('/foo').write('bar')
self.assertEqual(8, self.CountLoggers())
# +4.
files.File('/foo').serialize()
self.assertEqual(12, self.CountLoggers())
# +12.
files.File('/foo').copy_to(files.File('/bar'))
self.assertEqual(24, self.CountLoggers())
# +4.
files.File('/foo').delete()
self.assertEqual(28, self.CountLoggers())
def testRootTreeHandling(self):
# Actions should check root tree files, not versioning _FilePointers.
self.assertFalse(files.File('/foo').exists)
files.File('/foo', _no_mixins=True).write('')
self.assertTrue(files.File('/foo').exists)
files.File('/foo', _no_mixins=True).delete()
self.assertFalse(files.File('/foo').exists)
# write(), and delete() should modify root tree files and add pull tasks.
files.File('/foo').write('foo')
self.assertEqual(1, len(self.taskqueue_stub.get_filtered_tasks()))
self.assertEqual('foo', files.File('/foo', _no_mixins=True).content)
files.File('/foo').delete()
self.assertEqual(2, len(self.taskqueue_stub.get_filtered_tasks()))
self.assertFalse(files.File('/foo', _no_mixins=True).exists)
# Verify large RPC deferred task handling.
files.File('/foo').write(LARGE_FILE_CONTENT)
self.assertEqual(3, len(self.taskqueue_stub.get_filtered_tasks()))
self.assertEqual(
LARGE_FILE_CONTENT, files.File('/foo', _no_mixins=True).content)
def _data_file(self):
if self.__data_file is None:
self.__data_file = files.File('/{}.json'.format(self.key))
return self.__data_file
def _file(self):
if self._internal_file is None:
self._internal_file = files.File(self._dir_path + '.json',
_internal=True)
return self._internal_file
def _save_aggregate_data(self, final_aggregate_data):
"""Permanently store aggregate data to Titan Files."""
# Combine all data before writing files to minimize same file writes.
window_files = {}
for aggregate_data in final_aggregate_data:
window = aggregate_data['window']
window_datetime = datetime.datetime.utcfromtimestamp(window)
for counter_name, counter in aggregate_data['counters'].iteritems(
):
path = _make_log_path(window_datetime, counter)
if path not in window_files:
titan_file = files.File(path, _internal=True)
content = []
if titan_file.exists:
content = json.loads(titan_file.content)
window_files[path] = {
'file': titan_file,
'path': path,
'content': content,
'counter_name': counter_name,
'window_datetime': window_datetime,
}
# Add the counter data if it doesn't exist or is different.
old_content = window_files[path]['content']
window_files[path]['content'] = []
window_exists = False
for old_window, old_value in old_content:
# If we didn't find the window add it as a new counter.
if old_window > window and not window_exists:
window_exists = True
window_files[path]['content'].append(
(window, counter.finalize()))
# If the data is the same ignore, otherwise add old data to new.
if old_window == window:
window_exists = True
if old_value != counter.finalize():
if not counter.overwrite:
counter.aggregate(old_value)
old_value = counter.finalize()
window_files[path]['content'].append(
(old_window, old_value))
if not window_exists:
window_files[path]['content'].append(
(window, counter.finalize()))
# Keep the data sorted for update efficiency.
window_files[path]['content'].sort(key=lambda tup: tup[0])
# Write the changed window files.
for file_item in window_files.itervalues():
# Strip hours/minutes/seconds from date since the datastore can only
# store datetime objects, but we only need the date itself.
window_datetime = file_item['window_datetime']
date = datetime.datetime(window_datetime.year,
window_datetime.month,
window_datetime.day)
meta = {
'stats_counter_name': file_item['counter_name'],
'stats_date': date,
}
file_item['file'].write(content=json.dumps(file_item['content']),
meta=meta)
