def testCreateNewOoid(self): ooid = oo.createNewOoid() ndate = oo.dateFromOoid(ooid) ndepth = oo.depthFromOoid(ooid) assert self.nowstamp == ndate, 'Expect date of %s, got %s' % ( self.nowstamp, ndate) assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % ( oo.defaultDepth, ndepth) ooid = oo.createNewOoid(timestamp=self.xmas05) ndate = oo.dateFromOoid(ooid) ndepth = oo.depthFromOoid(ooid) assert self.xmas05 == ndate, 'Expect date of %s, got %s' % ( self.xmas05, ndate) assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % ( oo.defaultDepth, ndepth) for d in range(1, 5): ooid0 = oo.createNewOoid(depth=d) ooid1 = oo.createNewOoid(timestamp=self.xmas05, depth=d) ndate0 = oo.dateFromOoid(ooid0) ndepth0 = oo.depthFromOoid(ooid0) ndate1 = oo.dateFromOoid(ooid1) ndepth1 = oo.depthFromOoid(ooid1) assert self.nowstamp == ndate0, 'Expect date of %s, got %s' % ( self.nowstamp, ndate0) assert self.xmas05 == ndate1, 'Expect date of %s, got %s' % ( self.xmas05, ndate1) assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' % ( ndepth0, ndepth1) assert d == ndepth0, 'Expect depth %d, got %d' % (d, ndepth0) assert oo.depthFromOoid(self.badooid0) is None assert oo.depthFromOoid(self.badooid1) is None
def testCreateNewOoid(self): ooid = oo.createNewOoid() ndate = oo.dateFromOoid(ooid) ndepth = oo.depthFromOoid(ooid) assert self.nowstamp == ndate, 'Expect date of %s, got %s' %(self.nowstamp,ndate) assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth) ooid = oo.createNewOoid(timestamp=self.xmas05) ndate = oo.dateFromOoid(ooid) ndepth = oo.depthFromOoid(ooid) assert self.xmas05 == ndate, 'Expect date of %s, got %s' %(self.xmas05,ndate) assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth) for d in range(1,5): ooid0 = oo.createNewOoid(depth=d) ooid1 = oo.createNewOoid(timestamp=self.xmas05,depth=d) ndate0 = oo.dateFromOoid(ooid0) ndepth0 = oo.depthFromOoid(ooid0) ndate1 = oo.dateFromOoid(ooid1) ndepth1 = oo.depthFromOoid(ooid1) assert self.nowstamp == ndate0, 'Expect date of %s, got %s' %(self.nowstamp,ndate0) assert self.xmas05 == ndate1, 'Expect date of %s, got %s' %(self.xmas05,ndate1) assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' %(ndepth0,ndepth1) assert d == ndepth0, 'Expect depth %d, got %d' % (d,ndepth0) assert oo.depthFromOoid(self.badooid0) is None assert oo.depthFromOoid(self.badooid1) is None
def testGetDate(self): for ooid in self.yyyyoids: assert self.baseDate == oo.dateFromOoid( ooid), 'Expected %s got %s' % (self.baseDate, oo.dateFromOoid(ooid)) assert 4 == oo.depthFromOoid( ooid), 'Expected %d, got %d' % (4, oo.depthFromOoid(ooid)) assert oo.dateFromOoid(self.badooid0) is None assert oo.dateFromOoid(self.badooid1) is None
def build_keys(self, prefix, name_of_thing, id): """Use S3 pseudo-directories to make it easier to list/expire. For "raw_crash" things, the id is an ooid/crash_id. This uses the first three characters of the ooid for entropy and extracts the crash submission date from the last 6 charactes. Then it mooshes that all together into this structure:: {prefix}/v2/{name_of_thing}/{entropy}/{date}/{id} This makes it possible to list all the raw_crashes submitted on a given date. It also returns the keys that KeyBuilderBase builds so that we can fetch objects that were saved with older keys. """ keys = [] if name_of_thing == "raw_crash": datestamp = dateFromOoid(id) if datestamp is not None: # We insert the first 3 chars of the ooid/crash_id providing # some entropy earlier in the key so that consecutive s3 # requests get distributed across multiple s3 partitions. first_chars = id[:3] date = datestamp.strftime("%Y%m%d") keys.append("%s/v2/%s/%s/%s/%s" % (prefix, name_of_thing, first_chars, date, id)) keys.extend(super(DatePrefixKeyBuilder, self).build_keys(prefix, name_of_thing, id)) return keys
def build_keys(self, prefix, name_of_thing, id): """Use S3 pseudo-directories to make it easier to list/expire. For "raw_crash" things, the id is an ooid/crash_id. This uses the first three characters of the ooid for entropy and extracts the crash submission date from the last 6 charactes. Then it mooshes that all together into this structure:: {prefix}/v2/{name_of_thing}/{entropy}/{date}/{id} This makes it possible to list all the raw_crashes submitted on a given date. It also returns the keys that KeyBuilderBase builds so that we can fetch objects that were saved with older keys. """ keys = [] if name_of_thing == 'raw_crash': datestamp = dateFromOoid(id) if datestamp is not None: # We insert the first 3 chars of the ooid/crash_id providing # some entropy earlier in the key so that consecutive s3 # requests get distributed across multiple s3 partitions. first_chars = id[:3] date = datestamp.strftime('%Y%m%d') keys.append('%s/v2/%s/%s/%s/%s' % (prefix, name_of_thing, first_chars, date, id)) keys.extend( super(DatePrefixKeyBuilder, self).build_keys(prefix, name_of_thing, id)) return keys
def _get_base(self, crash_id): """this method overrides the base method to define the daily file system root directory name. While the default class is to use a YYYYMMDD form, this class substitutes a simple DD form. This is the mechanism of directory recycling as at the first day of a new month we return to the same directiory structures that were created on the first day of the previous month""" date = dateFromOoid(crash_id) if not date: date = utc_now() date_formatted = "%02d" % (date.day,) return [self.config.fs_root, date_formatted]
def _get_base(self, crash_id): """Overrides the base method to define the daily file system root directory name. While the default class uses a YYYYMMDD form, this class substitutes a simple DD form. This is the mechanism of directory recycling as at the first day of a new month we return to the same directiory structures that were created on the first day of the previous month. """ date = dateFromOoid(crash_id) if not date: date = utc_now() date_formatted = "%02d" % (date.day, ) return [self.config.fs_root, date_formatted]
def _get_base(self, crash_id): date = dateFromOoid(crash_id) if not date: date = utc_now() date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day) return [self.config.fs_root, date_formatted]
def testGetDate(self): for ooid in self.yyyyoids: assert self.baseDate == oo.dateFromOoid(ooid), 'Expected %s got %s' %(self.baseDate, oo.dateFromOoid(ooid)) assert 4 == oo.depthFromOoid(ooid), 'Expected %d, got %d' %(4, oo.depthFromOoid(ooid)) assert oo.dateFromOoid(self.badooid0) is None assert oo.dateFromOoid(self.badooid1) is None