def testCreateNewOoid(self):
    ooid = oo.createNewOoid()
    ndate = oo.dateFromOoid(ooid)
    ndepth = oo.depthFromOoid(ooid)
    assert self.nowstamp == ndate, 'Expect date of %s, got %s' %(self.nowstamp,ndate)
    assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)

    ooid = oo.createNewOoid(timestamp=self.xmas05)
    ndate = oo.dateFromOoid(ooid)
    ndepth = oo.depthFromOoid(ooid)
    assert self.xmas05 == ndate, 'Expect date of %s, got %s' %(self.xmas05,ndate)
    assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth)

    for d in range(1,5):
      ooid0 = oo.createNewOoid(depth=d)
      ooid1 = oo.createNewOoid(timestamp=self.xmas05,depth=d)
      ndate0 = oo.dateFromOoid(ooid0)
      ndepth0 = oo.depthFromOoid(ooid0)
      ndate1 = oo.dateFromOoid(ooid1)
      ndepth1 = oo.depthFromOoid(ooid1)
      assert self.nowstamp == ndate0, 'Expect date of %s, got %s' %(self.nowstamp,ndate0)
      assert self.xmas05 == ndate1, 'Expect date of %s, got %s' %(self.xmas05,ndate1)
      assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' %(ndepth0,ndepth1)
      assert d == ndepth0, 'Expect depth %d, got %d' % (d,ndepth0)
    assert None == oo.depthFromOoid(self.badooid0)
    assert None == oo.depthFromOoid(self.badooid1)
Exemple #2
0
    def build_keys(self, prefix, name_of_thing, id):
        """Use S3 pseudo-directories to make it easier to list/expire.

        For "raw_crash" things, the id is an ooid/crash_id. This uses the
        first three characters of the ooid for entropy and extracts the
        crash submission date from the last 6 charactes. Then it mooshes
        that all together into this structure::

            {prefix}/v2/{name_of_thing}/{entropy}/{date}/{id}

        This makes it possible to list all the raw_crashes submitted on a
        given date.

        It also returns the keys that KeyBuilderBase builds so that we can
        fetch objects that were saved with older keys.

        """
        keys = []

        if name_of_thing == 'raw_crash':
            datestamp = dateFromOoid(id)

            if datestamp is not None:
                # We insert the first 3 chars of the ooid/crash_id providing
                # some entropy earlier in the key so that consecutive s3
                # requests get distributed across multiple s3 partitions.
                first_chars = id[:3]
                date = datestamp.strftime('%Y%m%d')
                keys.append('%s/v2/%s/%s/%s/%s' %
                            (prefix, name_of_thing, first_chars, date, id))

        keys.extend(
            super(DatePrefixKeyBuilder,
                  self).build_keys(prefix, name_of_thing, id))
        return keys
Exemple #3
0
 def build_keys(self, prefix, name_of_thing, id):
     """return a list of one key. The reason for not returning more than
     one is that this key builder class is going to be used for
     something new so it has no legacy."""
     datestamp = dateFromOoid(id)
     if datestamp is None:
         # the id did not have a date component in it
         datestamp = datetime.datetime.utcnow()
     date = datestamp.strftime('%Y%m%d')
     keys = ['%s/v1/%s/%s/%s' % (prefix, name_of_thing, date, id)]
     return keys
Exemple #4
0
 def _get_base(self, crash_id):
     """this method overrides the base method to define the daily file
     system root directory name.  While the default class is to use a
     YYYYMMDD form, this class substitutes a simple DD form.  This is the
     mechanism of directory recycling as at the first day of a new month
     we return to the same directiory structures that were created on the
     first day of the previous month"""
     date = dateFromOoid(crash_id)
     if not date:
         date = utc_now()
     date_formatted = "%02d" % (date.day,)
     return [self.config.fs_root, date_formatted]
Exemple #5
0
 def _get_base(self, crash_id):
     """this method overrides the base method to define the daily file
     system root directory name.  While the default class is to use a
     YYYYMMDD form, this class substitutes a simple DD form.  This is the
     mechanism of directory recycling as at the first day of a new month
     we return to the same directiory structures that were created on the
     first day of the previous month"""
     date = dateFromOoid(crash_id)
     if not date:
         date = utc_now()
     date_formatted = "%02d" % (date.day, )
     return [self.config.fs_root, date_formatted]
    def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta):

        processor_notes = processor_meta.processor_notes

        processed_crash.submitted_timestamp = raw_crash.get("submitted_timestamp", dateFromOoid(raw_crash.uuid))
        if isinstance(processed_crash.submitted_timestamp, basestring):
            processed_crash.submitted_timestamp = datetimeFromISOdateString(processed_crash.submitted_timestamp)
        processed_crash.date_processed = processed_crash.submitted_timestamp
        # defaultCrashTime: must have crashed before date processed
        submitted_timestamp_as_epoch = int(time.mktime(processed_crash.submitted_timestamp.timetuple()))
        try:
            timestampTime = int(raw_crash.get("timestamp", submitted_timestamp_as_epoch))  # the old name for crash time
        except ValueError:
            timestampTime = 0
            processor_notes.append('non-integer value of "timestamp"')
        try:
            crash_time = int(self._get_truncate_or_warn(raw_crash, "CrashTime", processor_notes, timestampTime, 10))
        except ValueError:
            crash_time = 0
            processor_notes.append('non-integer value of "CrashTime" (%s)' % raw_crash.CrashTime)

        processed_crash.crash_time = crash_time
        if crash_time == submitted_timestamp_as_epoch:
            processor_notes.append("client_crash_date is unknown")
        # StartupTime: must have started up some time before crash
        try:
            startupTime = int(raw_crash.get("StartupTime", crash_time))
        except ValueError:
            startupTime = 0
            processor_notes.append('non-integer value of "StartupTime"')
        # InstallTime: must have installed some time before startup
        try:
            installTime = int(raw_crash.get("InstallTime", startupTime))
        except ValueError:
            installTime = 0
            processor_notes.append('non-integer value of "InstallTime"')
        processed_crash.client_crash_date = datetime.datetime.fromtimestamp(crash_time, UTC)
        processed_crash.install_age = crash_time - installTime
        processed_crash.uptime = max(0, crash_time - startupTime)
        try:
            last_crash = int(raw_crash.SecondsSinceLastCrash)
        except (KeyError, TypeError, ValueError):
            last_crash = None
            processor_notes.append('non-integer value of "SecondsSinceLastCrash"')
        if last_crash > maxint:
            last_crash = None
            processor_notes.append('"SecondsSinceLastCrash" larger than MAXINT - set to NULL')
        processed_crash.last_crash = last_crash

        return True
 def build_keys(self, prefix, name_of_thing, id):
     """return a list of one key. The reason for not returning more than
     one is that this key builder class is going to be used for
     something new so it has no legacy."""
     datestamp = dateFromOoid(id)
     if datestamp is None:
         # the id did not have a date component in it
         datestamp = datetime.datetime.utcnow()
     date = datestamp.strftime('%Y%m%d')
     keys = [
         '%s/v1/%s/%s/%s' % (
             prefix, name_of_thing, date, id
         )
     ]
     return keys
    def build_keys(self, prefix, name_of_thing, id):
        """Use S3 pseudo-directories to make it easier to list/expire.

        For "raw_crash" things, the id is an ooid/crash_id. This uses the
        first three characters of the ooid for entropy and extracts the
        crash submission date from the last 6 charactes. Then it mooshes
        that all together into this structure::

            {prefix}/v2/{name_of_thing}/{entropy}/{date}/{id}

        This makes it possible to list all the raw_crashes submitted on a
        given date.

        It also returns the keys that KeyBuilderBase builds so that we can
        fetch objects that were saved with older keys.

        """
        keys = []

        if name_of_thing == 'raw_crash':
            datestamp = dateFromOoid(id)

            if datestamp is not None:
                # We insert the first 3 chars of the ooid/crash_id providing
                # some entropy earlier in the key so that consecutive s3
                # requests get distributed across multiple s3 partitions.
                first_chars = id[:3]
                date = datestamp.strftime('%Y%m%d')
                keys.append(
                    '%s/v2/%s/%s/%s/%s' % (
                        prefix, name_of_thing, first_chars, date, id
                    )
                )

        keys.extend(
            super(DatePrefixKeyBuilder, self).build_keys(
                prefix, name_of_thing, id
            )
        )
        return keys
Exemple #9
0
 def _get_base(self, crash_id):
     date = dateFromOoid(crash_id)
     if not date:
         date = utc_now()
     date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day)
     return [self.config.fs_root, date_formatted]
 def testGetDate(self):
   for ooid in self.yyyyoids:
     assert self.baseDate == oo.dateFromOoid(ooid), 'Expected %s got %s' %(self.baseDate, oo.dateFromOoid(ooid))
     assert 4 == oo.depthFromOoid(ooid), 'Expected %d, got %d' %(4, oo.depthFromOoid(ooid))
   assert None == oo.dateFromOoid(self.badooid0)
   assert None == oo.dateFromOoid(self.badooid1)
Exemple #11
0
    def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta):

        processor_notes = processor_meta.processor_notes

        processed_crash.submitted_timestamp = raw_crash.get(
            'submitted_timestamp', dateFromOoid(raw_crash.uuid))
        if isinstance(processed_crash.submitted_timestamp, basestring):
            processed_crash.submitted_timestamp = datetimeFromISOdateString(
                processed_crash.submitted_timestamp)
        processed_crash.date_processed = processed_crash.submitted_timestamp
        # defaultCrashTime: must have crashed before date processed
        submitted_timestamp_as_epoch = int(
            time.mktime(processed_crash.submitted_timestamp.timetuple()))
        try:
            timestampTime = int(
                raw_crash.get('timestamp', submitted_timestamp_as_epoch)
            )  # the old name for crash time
        except ValueError:
            timestampTime = 0
            processor_notes.append('non-integer value of "timestamp"')
        try:
            crash_time = int(
                self._get_truncate_or_warn(raw_crash, 'CrashTime',
                                           processor_notes, timestampTime, 10))
        except ValueError:
            crash_time = 0
            processor_notes.append('non-integer value of "CrashTime" (%s)' %
                                   raw_crash.CrashTime)

        processed_crash.crash_time = crash_time
        if crash_time == submitted_timestamp_as_epoch:
            processor_notes.append("client_crash_date is unknown")
        # StartupTime: must have started up some time before crash
        try:
            startupTime = int(raw_crash.get('StartupTime', crash_time))
        except ValueError:
            startupTime = 0
            processor_notes.append('non-integer value of "StartupTime"')
        # InstallTime: must have installed some time before startup
        try:
            installTime = int(raw_crash.get('InstallTime', startupTime))
        except ValueError:
            installTime = 0
            processor_notes.append('non-integer value of "InstallTime"')
        processed_crash.client_crash_date = datetime.datetime.fromtimestamp(
            crash_time, UTC)
        processed_crash.install_age = crash_time - installTime
        processed_crash.uptime = max(0, crash_time - startupTime)
        try:
            last_crash = int(raw_crash.SecondsSinceLastCrash)
        except (KeyError, TypeError, ValueError):
            last_crash = None
            processor_notes.append(
                'non-integer value of "SecondsSinceLastCrash"')
        if last_crash > maxint:
            last_crash = None
            processor_notes.append(
                '"SecondsSinceLastCrash" larger than MAXINT - set to NULL')
        processed_crash.last_crash = last_crash

        return True
Exemple #12
0
 def _get_base(self, crash_id):
     date = dateFromOoid(crash_id)
     if not date:
         date = utc_now()
     date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day)
     return [self.config.fs_root, date_formatted]