def testCreateNewOoid(self):
ooid = oo.createNewOoid()
ndate = oo.dateFromOoid(ooid)
ndepth = oo.depthFromOoid(ooid)
assert self.nowstamp == ndate, 'Expect date of %s, got %s' % (
self.nowstamp, ndate)
assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (
oo.defaultDepth, ndepth)
ooid = oo.createNewOoid(timestamp=self.xmas05)
ndate = oo.dateFromOoid(ooid)
ndepth = oo.depthFromOoid(ooid)
assert self.xmas05 == ndate, 'Expect date of %s, got %s' % (
self.xmas05, ndate)
assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (
oo.defaultDepth, ndepth)
for d in range(1, 5):
ooid0 = oo.createNewOoid(depth=d)
ooid1 = oo.createNewOoid(timestamp=self.xmas05, depth=d)
ndate0 = oo.dateFromOoid(ooid0)
ndepth0 = oo.depthFromOoid(ooid0)
ndate1 = oo.dateFromOoid(ooid1)
ndepth1 = oo.depthFromOoid(ooid1)
assert self.nowstamp == ndate0, 'Expect date of %s, got %s' % (
self.nowstamp, ndate0)
assert self.xmas05 == ndate1, 'Expect date of %s, got %s' % (
self.xmas05, ndate1)
assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' % (
ndepth0, ndepth1)
assert d == ndepth0, 'Expect depth %d, got %d' % (d, ndepth0)
assert oo.depthFromOoid(self.badooid0) is None
assert oo.depthFromOoid(self.badooid1) is None
def testCreateNewOoid(self):
ooid = oo.createNewOoid()
ndate = oo.dateFromOoid(ooid)
ndepth = oo.depthFromOoid(ooid)
assert self.nowstamp == ndate, 'Expect date of %s, got %s' %(self.nowstamp,ndate)
assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)
ooid = oo.createNewOoid(timestamp=self.xmas05)
ndate = oo.dateFromOoid(ooid)
ndepth = oo.depthFromOoid(ooid)
assert self.xmas05 == ndate, 'Expect date of %s, got %s' %(self.xmas05,ndate)
assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)
for d in range(1,5):
ooid0 = oo.createNewOoid(depth=d)
ooid1 = oo.createNewOoid(timestamp=self.xmas05,depth=d)
ndate0 = oo.dateFromOoid(ooid0)
ndepth0 = oo.depthFromOoid(ooid0)
ndate1 = oo.dateFromOoid(ooid1)
ndepth1 = oo.depthFromOoid(ooid1)
assert self.nowstamp == ndate0, 'Expect date of %s, got %s' %(self.nowstamp,ndate0)
assert self.xmas05 == ndate1, 'Expect date of %s, got %s' %(self.xmas05,ndate1)
assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' %(ndepth0,ndepth1)
assert d == ndepth0, 'Expect depth %d, got %d' % (d,ndepth0)
assert oo.depthFromOoid(self.badooid0) is None
assert oo.depthFromOoid(self.badooid1) is None
def testGetDate(self):
for ooid in self.yyyyoids:
assert self.baseDate == oo.dateFromOoid(
ooid), 'Expected %s got %s' % (self.baseDate,
oo.dateFromOoid(ooid))
assert 4 == oo.depthFromOoid(
ooid), 'Expected %d, got %d' % (4, oo.depthFromOoid(ooid))
assert oo.dateFromOoid(self.badooid0) is None
assert oo.dateFromOoid(self.badooid1) is None
def build_keys(self, prefix, name_of_thing, id):
"""Use S3 pseudo-directories to make it easier to list/expire.
For "raw_crash" things, the id is an ooid/crash_id. This uses the
first three characters of the ooid for entropy and extracts the
crash submission date from the last 6 charactes. Then it mooshes
that all together into this structure::
{prefix}/v2/{name_of_thing}/{entropy}/{date}/{id}
This makes it possible to list all the raw_crashes submitted on a
given date.
It also returns the keys that KeyBuilderBase builds so that we can
fetch objects that were saved with older keys.
"""
keys = []
if name_of_thing == "raw_crash":
datestamp = dateFromOoid(id)
if datestamp is not None:
# We insert the first 3 chars of the ooid/crash_id providing
# some entropy earlier in the key so that consecutive s3
# requests get distributed across multiple s3 partitions.
first_chars = id[:3]
date = datestamp.strftime("%Y%m%d")
keys.append("%s/v2/%s/%s/%s/%s" % (prefix, name_of_thing, first_chars, date, id))
keys.extend(super(DatePrefixKeyBuilder, self).build_keys(prefix, name_of_thing, id))
return keys
def build_keys(self, prefix, name_of_thing, id):
"""Use S3 pseudo-directories to make it easier to list/expire.
For "raw_crash" things, the id is an ooid/crash_id. This uses the
first three characters of the ooid for entropy and extracts the
crash submission date from the last 6 charactes. Then it mooshes
that all together into this structure::
{prefix}/v2/{name_of_thing}/{entropy}/{date}/{id}
This makes it possible to list all the raw_crashes submitted on a
given date.
It also returns the keys that KeyBuilderBase builds so that we can
fetch objects that were saved with older keys.
"""
keys = []
if name_of_thing == 'raw_crash':
datestamp = dateFromOoid(id)
if datestamp is not None:
# We insert the first 3 chars of the ooid/crash_id providing
# some entropy earlier in the key so that consecutive s3
# requests get distributed across multiple s3 partitions.
first_chars = id[:3]
date = datestamp.strftime('%Y%m%d')
keys.append('%s/v2/%s/%s/%s/%s' %
(prefix, name_of_thing, first_chars, date, id))
keys.extend(
super(DatePrefixKeyBuilder,
self).build_keys(prefix, name_of_thing, id))
return keys
def _get_base(self, crash_id):
"""this method overrides the base method to define the daily file
system root directory name. While the default class is to use a
YYYYMMDD form, this class substitutes a simple DD form. This is the
mechanism of directory recycling as at the first day of a new month
we return to the same directiory structures that were created on the
first day of the previous month"""
date = dateFromOoid(crash_id)
if not date:
date = utc_now()
date_formatted = "%02d" % (date.day,)
return [self.config.fs_root, date_formatted]
def _get_base(self, crash_id):
"""Overrides the base method to define the daily file system root directory
name.
While the default class uses a YYYYMMDD form, this class substitutes a
simple DD form. This is the mechanism of directory recycling as at the
first day of a new month we return to the same directiory structures
that were created on the first day of the previous month.
"""
date = dateFromOoid(crash_id)
if not date:
date = utc_now()
date_formatted = "%02d" % (date.day, )
return [self.config.fs_root, date_formatted]
def _get_base(self, crash_id):
date = dateFromOoid(crash_id)
if not date:
date = utc_now()
date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day)
return [self.config.fs_root, date_formatted]
def testGetDate(self):
for ooid in self.yyyyoids:
assert self.baseDate == oo.dateFromOoid(ooid), 'Expected %s got %s' %(self.baseDate, oo.dateFromOoid(ooid))
assert 4 == oo.depthFromOoid(ooid), 'Expected %d, got %d' %(4, oo.depthFromOoid(ooid))
assert oo.dateFromOoid(self.badooid0) is None
assert oo.dateFromOoid(self.badooid1) is None
def _get_base(self, crash_id):
date = dateFromOoid(crash_id)
if not date:
date = utc_now()
date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day)
return [self.config.fs_root, date_formatted]