Exemplo n.º 1
0
    def testCreateNewOoid(self):
        ooid = oo.createNewOoid()
        ndate = oo.dateFromOoid(ooid)
        ndepth = oo.depthFromOoid(ooid)
        assert self.nowstamp == ndate, 'Expect date of %s, got %s' % (
            self.nowstamp, ndate)
        assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (
            oo.defaultDepth, ndepth)

        ooid = oo.createNewOoid(timestamp=self.xmas05)
        ndate = oo.dateFromOoid(ooid)
        ndepth = oo.depthFromOoid(ooid)
        assert self.xmas05 == ndate, 'Expect date of %s, got %s' % (
            self.xmas05, ndate)
        assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (
            oo.defaultDepth, ndepth)

        for d in range(1, 5):
            ooid0 = oo.createNewOoid(depth=d)
            ooid1 = oo.createNewOoid(timestamp=self.xmas05, depth=d)
            ndate0 = oo.dateFromOoid(ooid0)
            ndepth0 = oo.depthFromOoid(ooid0)
            ndate1 = oo.dateFromOoid(ooid1)
            ndepth1 = oo.depthFromOoid(ooid1)
            assert self.nowstamp == ndate0, 'Expect date of %s, got %s' % (
                self.nowstamp, ndate0)
            assert self.xmas05 == ndate1, 'Expect date of %s, got %s' % (
                self.xmas05, ndate1)
            assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' % (
                ndepth0, ndepth1)
            assert d == ndepth0, 'Expect depth %d, got %d' % (d, ndepth0)
        assert oo.depthFromOoid(self.badooid0) is None
        assert oo.depthFromOoid(self.badooid1) is None
Exemplo n.º 2
0
  def testCreateNewOoid(self):
    ooid = oo.createNewOoid()
    ndate = oo.dateFromOoid(ooid)
    ndepth = oo.depthFromOoid(ooid)
    assert self.nowstamp == ndate, 'Expect date of %s, got %s' %(self.nowstamp,ndate)
    assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)

    ooid = oo.createNewOoid(timestamp=self.xmas05)
    ndate = oo.dateFromOoid(ooid)
    ndepth = oo.depthFromOoid(ooid)
    assert self.xmas05 == ndate, 'Expect date of %s, got %s' %(self.xmas05,ndate)
    assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)

    for d in range(1,5):
      ooid0 = oo.createNewOoid(depth=d)
      ooid1 = oo.createNewOoid(timestamp=self.xmas05,depth=d)
      ndate0 = oo.dateFromOoid(ooid0)
      ndepth0 = oo.depthFromOoid(ooid0)
      ndate1 = oo.dateFromOoid(ooid1)
      ndepth1 = oo.depthFromOoid(ooid1)
      assert self.nowstamp == ndate0, 'Expect date of %s, got %s' %(self.nowstamp,ndate0)
      assert self.xmas05 == ndate1, 'Expect date of %s, got %s' %(self.xmas05,ndate1)
      assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' %(ndepth0,ndepth1)
      assert d == ndepth0, 'Expect depth %d, got %d' % (d,ndepth0)
    assert oo.depthFromOoid(self.badooid0) is None
    assert oo.depthFromOoid(self.badooid1) is None
Exemplo n.º 3
0
 def testGetDate(self):
     for ooid in self.yyyyoids:
         assert self.baseDate == oo.dateFromOoid(
             ooid), 'Expected %s got %s' % (self.baseDate,
                                            oo.dateFromOoid(ooid))
         assert 4 == oo.depthFromOoid(
             ooid), 'Expected %d, got %d' % (4, oo.depthFromOoid(ooid))
     assert oo.dateFromOoid(self.badooid0) is None
     assert oo.dateFromOoid(self.badooid1) is None
    def build_keys(self, prefix, name_of_thing, id):
        """Use S3 pseudo-directories to make it easier to list/expire.

        For "raw_crash" things, the id is an ooid/crash_id. This uses the
        first three characters of the ooid for entropy and extracts the
        crash submission date from the last 6 charactes. Then it mooshes
        that all together into this structure::

            {prefix}/v2/{name_of_thing}/{entropy}/{date}/{id}

        This makes it possible to list all the raw_crashes submitted on a
        given date.

        It also returns the keys that KeyBuilderBase builds so that we can
        fetch objects that were saved with older keys.

        """
        keys = []

        if name_of_thing == "raw_crash":
            datestamp = dateFromOoid(id)

            if datestamp is not None:
                # We insert the first 3 chars of the ooid/crash_id providing
                # some entropy earlier in the key so that consecutive s3
                # requests get distributed across multiple s3 partitions.
                first_chars = id[:3]
                date = datestamp.strftime("%Y%m%d")
                keys.append("%s/v2/%s/%s/%s/%s" % (prefix, name_of_thing, first_chars, date, id))

        keys.extend(super(DatePrefixKeyBuilder, self).build_keys(prefix, name_of_thing, id))
        return keys
    def build_keys(self, prefix, name_of_thing, id):
        """Use S3 pseudo-directories to make it easier to list/expire.

        For "raw_crash" things, the id is an ooid/crash_id. This uses the
        first three characters of the ooid for entropy and extracts the
        crash submission date from the last 6 charactes. Then it mooshes
        that all together into this structure::

            {prefix}/v2/{name_of_thing}/{entropy}/{date}/{id}

        This makes it possible to list all the raw_crashes submitted on a
        given date.

        It also returns the keys that KeyBuilderBase builds so that we can
        fetch objects that were saved with older keys.

        """
        keys = []

        if name_of_thing == 'raw_crash':
            datestamp = dateFromOoid(id)

            if datestamp is not None:
                # We insert the first 3 chars of the ooid/crash_id providing
                # some entropy earlier in the key so that consecutive s3
                # requests get distributed across multiple s3 partitions.
                first_chars = id[:3]
                date = datestamp.strftime('%Y%m%d')
                keys.append('%s/v2/%s/%s/%s/%s' %
                            (prefix, name_of_thing, first_chars, date, id))

        keys.extend(
            super(DatePrefixKeyBuilder,
                  self).build_keys(prefix, name_of_thing, id))
        return keys
Exemplo n.º 6
0
 def _get_base(self, crash_id):
     """this method overrides the base method to define the daily file
     system root directory name.  While the default class is to use a
     YYYYMMDD form, this class substitutes a simple DD form.  This is the
     mechanism of directory recycling as at the first day of a new month
     we return to the same directiory structures that were created on the
     first day of the previous month"""
     date = dateFromOoid(crash_id)
     if not date:
         date = utc_now()
     date_formatted = "%02d" % (date.day,)
     return [self.config.fs_root, date_formatted]
Exemplo n.º 7
0
    def _get_base(self, crash_id):
        """Overrides the base method to define the daily file system root directory
        name.

        While the default class uses a YYYYMMDD form, this class substitutes a
        simple DD form. This is the mechanism of directory recycling as at the
        first day of a new month we return to the same directiory structures
        that were created on the first day of the previous month.

        """
        date = dateFromOoid(crash_id)
        if not date:
            date = utc_now()
        date_formatted = "%02d" % (date.day, )
        return [self.config.fs_root, date_formatted]
Exemplo n.º 8
0
 def _get_base(self, crash_id):
     date = dateFromOoid(crash_id)
     if not date:
         date = utc_now()
     date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day)
     return [self.config.fs_root, date_formatted]
Exemplo n.º 9
0
 def testGetDate(self):
   for ooid in self.yyyyoids:
     assert self.baseDate == oo.dateFromOoid(ooid), 'Expected %s got %s' %(self.baseDate, oo.dateFromOoid(ooid))
     assert 4 == oo.depthFromOoid(ooid), 'Expected %d, got %d' %(4, oo.depthFromOoid(ooid))
   assert oo.dateFromOoid(self.badooid0) is None
   assert oo.dateFromOoid(self.badooid1) is None
Exemplo n.º 10
0
 def _get_base(self, crash_id):
     date = dateFromOoid(crash_id)
     if not date:
         date = utc_now()
     date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day)
     return [self.config.fs_root, date_formatted]