def testCreateNewOoid(self): ooid = oo.createNewOoid() ndate = oo.dateFromOoid(ooid) ndepth = oo.depthFromOoid(ooid) assert self.nowstamp == ndate, 'Expect date of %s, got %s' %(self.nowstamp,ndate) assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth) ooid = oo.createNewOoid(timestamp=self.xmas05) ndate = oo.dateFromOoid(ooid) ndepth = oo.depthFromOoid(ooid) assert self.xmas05 == ndate, 'Expect date of %s, got %s' %(self.xmas05,ndate) assert oo.defaultDepth == ndepth, 'Expect default depth (%d) got %d' % (oo.defaultDepth,ndepth) for d in range(1,5): ooid0 = oo.createNewOoid(depth=d) ooid1 = oo.createNewOoid(timestamp=self.xmas05,depth=d) ndate0 = oo.dateFromOoid(ooid0) ndepth0 = oo.depthFromOoid(ooid0) ndate1 = oo.dateFromOoid(ooid1) ndepth1 = oo.depthFromOoid(ooid1) assert self.nowstamp == ndate0, 'Expect date of %s, got %s' %(self.nowstamp,ndate0) assert self.xmas05 == ndate1, 'Expect date of %s, got %s' %(self.xmas05,ndate1) assert ndepth0 == ndepth1, 'Expect depth0(%d) == depth1(%d)' %(ndepth0,ndepth1) assert d == ndepth0, 'Expect depth %d, got %d' % (d,ndepth0) assert None == oo.depthFromOoid(self.badooid0) assert None == oo.depthFromOoid(self.badooid1)
def build_keys(self, prefix, name_of_thing, id): """Use S3 pseudo-directories to make it easier to list/expire. For "raw_crash" things, the id is an ooid/crash_id. This uses the first three characters of the ooid for entropy and extracts the crash submission date from the last 6 charactes. Then it mooshes that all together into this structure:: {prefix}/v2/{name_of_thing}/{entropy}/{date}/{id} This makes it possible to list all the raw_crashes submitted on a given date. It also returns the keys that KeyBuilderBase builds so that we can fetch objects that were saved with older keys. """ keys = [] if name_of_thing == 'raw_crash': datestamp = dateFromOoid(id) if datestamp is not None: # We insert the first 3 chars of the ooid/crash_id providing # some entropy earlier in the key so that consecutive s3 # requests get distributed across multiple s3 partitions. first_chars = id[:3] date = datestamp.strftime('%Y%m%d') keys.append('%s/v2/%s/%s/%s/%s' % (prefix, name_of_thing, first_chars, date, id)) keys.extend( super(DatePrefixKeyBuilder, self).build_keys(prefix, name_of_thing, id)) return keys
def build_keys(self, prefix, name_of_thing, id): """return a list of one key. The reason for not returning more than one is that this key builder class is going to be used for something new so it has no legacy.""" datestamp = dateFromOoid(id) if datestamp is None: # the id did not have a date component in it datestamp = datetime.datetime.utcnow() date = datestamp.strftime('%Y%m%d') keys = ['%s/v1/%s/%s/%s' % (prefix, name_of_thing, date, id)] return keys
def _get_base(self, crash_id): """this method overrides the base method to define the daily file system root directory name. While the default class is to use a YYYYMMDD form, this class substitutes a simple DD form. This is the mechanism of directory recycling as at the first day of a new month we return to the same directiory structures that were created on the first day of the previous month""" date = dateFromOoid(crash_id) if not date: date = utc_now() date_formatted = "%02d" % (date.day,) return [self.config.fs_root, date_formatted]
def _get_base(self, crash_id): """this method overrides the base method to define the daily file system root directory name. While the default class is to use a YYYYMMDD form, this class substitutes a simple DD form. This is the mechanism of directory recycling as at the first day of a new month we return to the same directiory structures that were created on the first day of the previous month""" date = dateFromOoid(crash_id) if not date: date = utc_now() date_formatted = "%02d" % (date.day, ) return [self.config.fs_root, date_formatted]
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta): processor_notes = processor_meta.processor_notes processed_crash.submitted_timestamp = raw_crash.get("submitted_timestamp", dateFromOoid(raw_crash.uuid)) if isinstance(processed_crash.submitted_timestamp, basestring): processed_crash.submitted_timestamp = datetimeFromISOdateString(processed_crash.submitted_timestamp) processed_crash.date_processed = processed_crash.submitted_timestamp # defaultCrashTime: must have crashed before date processed submitted_timestamp_as_epoch = int(time.mktime(processed_crash.submitted_timestamp.timetuple())) try: timestampTime = int(raw_crash.get("timestamp", submitted_timestamp_as_epoch)) # the old name for crash time except ValueError: timestampTime = 0 processor_notes.append('non-integer value of "timestamp"') try: crash_time = int(self._get_truncate_or_warn(raw_crash, "CrashTime", processor_notes, timestampTime, 10)) except ValueError: crash_time = 0 processor_notes.append('non-integer value of "CrashTime" (%s)' % raw_crash.CrashTime) processed_crash.crash_time = crash_time if crash_time == submitted_timestamp_as_epoch: processor_notes.append("client_crash_date is unknown") # StartupTime: must have started up some time before crash try: startupTime = int(raw_crash.get("StartupTime", crash_time)) except ValueError: startupTime = 0 processor_notes.append('non-integer value of "StartupTime"') # InstallTime: must have installed some time before startup try: installTime = int(raw_crash.get("InstallTime", startupTime)) except ValueError: installTime = 0 processor_notes.append('non-integer value of "InstallTime"') processed_crash.client_crash_date = datetime.datetime.fromtimestamp(crash_time, UTC) processed_crash.install_age = crash_time - installTime processed_crash.uptime = max(0, crash_time - startupTime) try: last_crash = int(raw_crash.SecondsSinceLastCrash) except (KeyError, TypeError, ValueError): last_crash = None processor_notes.append('non-integer value of "SecondsSinceLastCrash"') if last_crash > maxint: last_crash = None processor_notes.append('"SecondsSinceLastCrash" larger than MAXINT - set to NULL') processed_crash.last_crash = last_crash return True
def build_keys(self, prefix, name_of_thing, id): """return a list of one key. The reason for not returning more than one is that this key builder class is going to be used for something new so it has no legacy.""" datestamp = dateFromOoid(id) if datestamp is None: # the id did not have a date component in it datestamp = datetime.datetime.utcnow() date = datestamp.strftime('%Y%m%d') keys = [ '%s/v1/%s/%s/%s' % ( prefix, name_of_thing, date, id ) ] return keys
def build_keys(self, prefix, name_of_thing, id): """Use S3 pseudo-directories to make it easier to list/expire. For "raw_crash" things, the id is an ooid/crash_id. This uses the first three characters of the ooid for entropy and extracts the crash submission date from the last 6 charactes. Then it mooshes that all together into this structure:: {prefix}/v2/{name_of_thing}/{entropy}/{date}/{id} This makes it possible to list all the raw_crashes submitted on a given date. It also returns the keys that KeyBuilderBase builds so that we can fetch objects that were saved with older keys. """ keys = [] if name_of_thing == 'raw_crash': datestamp = dateFromOoid(id) if datestamp is not None: # We insert the first 3 chars of the ooid/crash_id providing # some entropy earlier in the key so that consecutive s3 # requests get distributed across multiple s3 partitions. first_chars = id[:3] date = datestamp.strftime('%Y%m%d') keys.append( '%s/v2/%s/%s/%s/%s' % ( prefix, name_of_thing, first_chars, date, id ) ) keys.extend( super(DatePrefixKeyBuilder, self).build_keys( prefix, name_of_thing, id ) ) return keys
def _get_base(self, crash_id): date = dateFromOoid(crash_id) if not date: date = utc_now() date_formatted = "%4d%02d%02d" % (date.year, date.month, date.day) return [self.config.fs_root, date_formatted]
def testGetDate(self): for ooid in self.yyyyoids: assert self.baseDate == oo.dateFromOoid(ooid), 'Expected %s got %s' %(self.baseDate, oo.dateFromOoid(ooid)) assert 4 == oo.depthFromOoid(ooid), 'Expected %d, got %d' %(4, oo.depthFromOoid(ooid)) assert None == oo.dateFromOoid(self.badooid0) assert None == oo.dateFromOoid(self.badooid1)
def _action(self, raw_crash, raw_dumps, processed_crash, processor_meta): processor_notes = processor_meta.processor_notes processed_crash.submitted_timestamp = raw_crash.get( 'submitted_timestamp', dateFromOoid(raw_crash.uuid)) if isinstance(processed_crash.submitted_timestamp, basestring): processed_crash.submitted_timestamp = datetimeFromISOdateString( processed_crash.submitted_timestamp) processed_crash.date_processed = processed_crash.submitted_timestamp # defaultCrashTime: must have crashed before date processed submitted_timestamp_as_epoch = int( time.mktime(processed_crash.submitted_timestamp.timetuple())) try: timestampTime = int( raw_crash.get('timestamp', submitted_timestamp_as_epoch) ) # the old name for crash time except ValueError: timestampTime = 0 processor_notes.append('non-integer value of "timestamp"') try: crash_time = int( self._get_truncate_or_warn(raw_crash, 'CrashTime', processor_notes, timestampTime, 10)) except ValueError: crash_time = 0 processor_notes.append('non-integer value of "CrashTime" (%s)' % raw_crash.CrashTime) processed_crash.crash_time = crash_time if crash_time == submitted_timestamp_as_epoch: processor_notes.append("client_crash_date is unknown") # StartupTime: must have started up some time before crash try: startupTime = int(raw_crash.get('StartupTime', crash_time)) except ValueError: startupTime = 0 processor_notes.append('non-integer value of "StartupTime"') # InstallTime: must have installed some time before startup try: installTime = int(raw_crash.get('InstallTime', startupTime)) except ValueError: installTime = 0 processor_notes.append('non-integer value of "InstallTime"') processed_crash.client_crash_date = datetime.datetime.fromtimestamp( crash_time, UTC) processed_crash.install_age = crash_time - installTime processed_crash.uptime = max(0, crash_time - startupTime) try: last_crash = int(raw_crash.SecondsSinceLastCrash) except (KeyError, TypeError, ValueError): last_crash = None processor_notes.append( 'non-integer value of "SecondsSinceLastCrash"') if last_crash > maxint: last_crash = None processor_notes.append( '"SecondsSinceLastCrash" larger than MAXINT - set to NULL') processed_crash.last_crash = last_crash return True