class RepositoryContentUnit(AutoRetryDocument): """ Represents the link between a repository and the units associated with it. This inherits from mongoengine.Document and defines the schema for the documents in repo_content_units collection. :ivar repo_id: string representation of the repository id :type repo_id: mongoengine.StringField :ivar unit_id: string representation of content unit id :type unit_id: mongoengine.StringField :ivar unit_type_id: string representation of content unit type :type unit_type_id: mongoengine.StringField :ivar created: ISO8601 representation of the time the association was created :type created: pulp.server.db.fields.ISO8601StringField :ivar updated: ISO8601 representation of last time a copy, sync, or upload ensured that the association existed :type updated: pulp.server.db.fields.ISO8601StringField :ivar _ns: The namespace field (Deprecated), reading :type _ns: mongoengine.StringField """ repo_id = StringField(required=True) unit_id = StringField(required=True) unit_type_id = StringField(required=True) created = ISO8601StringField( required=True, default=lambda: dateutils.format_iso8601_utc_timestamp( dateutils.now_utc_timestamp())) updated = ISO8601StringField( required=True, default=lambda: dateutils.format_iso8601_utc_timestamp( dateutils.now_utc_timestamp())) # For backward compatibility _ns = StringField(default='repo_content_units') meta = { 'collection': 'repo_content_units', 'allow_inheritance': False, 'indexes': [ { 'fields': ['repo_id', 'unit_type_id', 'unit_id'], 'unique': True }, { # Used for reverse lookup of units to repositories 'fields': ['unit_id'] } ] }
def calculate_next_run(self): """ This algorithm starts by determining when the first call was or should have been. If that is in the future, it just returns that time. If not, it adds as many intervals as it can without exceeding the current time, adds one more interval, and returns the result. For a schedule with no historically-recorded or scheduled start time, it will run immediately. :return: ISO8601 string representing the next time this call should run. :rtype: str """ now_s, first_run_s, since_first_s, run_every_s, \ last_scheduled_run_s, expected_runs = self._calculate_times() # if first run is in the future, return that time if first_run_s > now_s: next_run_s = first_run_s # if I've never run before and my first run is not in the future, run now! elif self.total_run_count == 0: next_run_s = now_s else: next_run_s = last_scheduled_run_s + run_every_s return dateutils.format_iso8601_utc_timestamp(next_run_s)
def calculate_next_run(self): """ This algorithm starts by determining when the first call was or should have been. If that is in the future, it just returns that time. If not, it adds as many intervals as it can without exceeding the current time, adds one more interval, and returns the result. For a schedule with no historically-recorded or scheduled start time, it will run immediately. :return: ISO8601 string representing the next time this call should run. :rtype: str """ now_s, first_run_s, since_first_s, run_every_s, \ last_scheduled_run_s, expected_runs = self._calculate_times() # if first run is in the future, return that time if first_run_s > now_s: next_run_s = first_run_s # if I've never run before and my first run is not in the future, run now! elif self.total_run_count == 0: next_run_s = now_s else: next_run_s = last_scheduled_run_s + run_every_s return dateutils.format_iso8601_utc_timestamp(next_run_s)
def content_unit_obj(content_unit): """ Serialize a content unit. """ serial = db.scrub_mongo_fields(content_unit) last_updated = content_unit.get('_last_updated') if last_updated: content_unit['_last_updated'] = dateutils.format_iso8601_utc_timestamp(last_updated) return serial
def content_unit_obj(content_unit): """ Serialize a content unit. """ serial = db.scrub_mongo_fields(content_unit) serialize_unit_with_serializer(content_unit) last_updated = content_unit.get('_last_updated') if last_updated: content_unit['_last_updated'] = dateutils.format_iso8601_utc_timestamp(last_updated) return serial
def __init__(self, repo_id, unit_id, unit_type_id): super(RepoContentUnit, self).__init__() # Mapping Identity Information self.repo_id = repo_id self.unit_id = unit_id self.unit_type_id = unit_type_id # store time in UTC utc_timestamp = dateutils.now_utc_timestamp() self.created = dateutils.format_iso8601_utc_timestamp(utc_timestamp) self.updated = self.created
def __init__(self, repo_id, unit_id, unit_type_id): super(RepoContentUnit, self).__init__() # Mapping Identity Information self.repo_id = repo_id self.unit_id = unit_id self.unit_type_id = unit_type_id # store time in UTC utc_timestamp = dateutils.now_utc_timestamp() self.created = dateutils.format_iso8601_utc_timestamp(utc_timestamp) self.updated = self.created
def associate_single_unit(repository, unit): """ Associate a single unit to a repository. :param repository: The repository to update. :type repository: pulp.server.db.model.Repository :param unit: The unit to associate to the repository. :type unit: pulp.server.db.model.ContentUnit """ current_timestamp = dateutils.now_utc_timestamp() formatted_datetime = dateutils.format_iso8601_utc_timestamp( current_timestamp) qs = model.RepositoryContentUnit.objects(repo_id=repository.repo_id, unit_id=unit.id, unit_type_id=unit.unit_type_id) qs.update_one(set_on_insert__created=formatted_datetime, set__updated=formatted_datetime, upsert=True)
def _next_instance(self, last_run_at=None): """ Returns an instance of this class with the appropriate fields incremented and updated to reflect that its task has been queued. The parent ScheduledCall gets saved to the database with these updated values. :param last_run_at: not used here, but it is part of the superclass function signature :return: a new instance of the same class, but with its date and count fields updated. :rtype: pulp.server.db.model.dispatch.ScheduleEntry """ self._scheduled_call.last_run_at = dateutils.format_iso8601_utc_timestamp(time.time()) self._scheduled_call.total_run_count += 1 if self._scheduled_call.remaining_runs: self._scheduled_call.remaining_runs -= 1 if self._scheduled_call.remaining_runs == 0: logger.info('disabling schedule with 0 remaining runs: %s' % self._scheduled_call.id) self._scheduled_call.enabled = False self._scheduled_call.save() return self._scheduled_call.as_schedule_entry()
def test_formatting_utc_timestamp(self): dt = datetime.datetime(2012, 10, 24, 10, 20, tzinfo=dateutils.utc_tz()) ts = dateutils.datetime_to_utc_timestamp(dt) formatted = dateutils.format_iso8601_utc_timestamp(ts) self.assertEqual(formatted, '2012-10-24T10:20:00Z')
def setUp(self): super(RemoveRepoDuplicateNevra, self).setUp() # repo_a is based on the test repo defined in TestPurgeBase self.repo_a = platform_model.Repository(repo_id=self.repo.id) self.repo_a.save() # repo_b is a control repo, that should be untouched by purge functions self.repo_b = platform_model.Repository(repo_id='b') self.repo_b.save() # create units unit_key_base = { 'epoch': '0', 'version': '0', 'release': '23', 'arch': 'noarch', 'checksumtype': 'sha256', '_last_updated': 0, } units = [] self.duplicate_unit_ids = set() for unit_type in self.UNIT_TYPES: unit_key_dupe = unit_key_base.copy() unit_key_uniq = unit_key_base.copy() # account for slightly different unit key field on drpm if unit_type is models.DRPM: unit_key_dupe['filename'] = 'dupe' unit_key_uniq['filename'] = 'uniq' else: unit_key_dupe['name'] = 'dupe' unit_key_uniq['name'] = 'uniq' # create units with duplicate nevra for this type # after purging, only one of the three should remain for i in range(3): unit_dupe = unit_type(**unit_key_dupe) # use the unit's python id to guarantee a unique "checksum" unit_dupe.checksum = str(id(unit_dupe)) unit_dupe.save() units.append(unit_dupe) if i != 0: # after the first unit, stash the "extra" duplicates to make it easier # to modify the unit association updated timestamps for predictable sorting self.duplicate_unit_ids.add(unit_dupe.id) # use the incrementing unit count to make the uniq unit's nevra unique unit_key_uniq['version'] = str(len(units)) # create a unit with unique nevra unit_uniq = unit_type(**unit_key_uniq) unit_uniq.checksum = str(hash(unit_uniq)) unit_uniq.save() units.append(unit_uniq) # associate each unit with each repo for repo in self.repo_a, self.repo_b: for i, unit in enumerate(units): repo_controller.associate_single_unit(repo, unit) # Sanity check: 3 dupe units and 1 uniq unit for n unit types, for each repo expected_rcu_count = 4 * len(self.UNIT_TYPES) for repo_id in self.repo_a.repo_id, self.repo_b.repo_id: self.assertEqual( platform_model.RepositoryContentUnit.objects.filter( repo_id=repo_id).count(), expected_rcu_count) # To ensure the purge mechanism behavior is predictable for testing, # go through the duplicate unit IDs and set their updated time to be in the past, # since unit associations were all just created at the same time. # The older associations are the ones that should be purged. earlier_timestamp = dateutils.now_utc_timestamp() - 3600 formatted_timestamp = dateutils.format_iso8601_utc_timestamp( earlier_timestamp) platform_model.RepositoryContentUnit.objects.filter(unit_id__in=self.duplicate_unit_ids)\ .update(set__updated=formatted_timestamp)