def get(self, schedule_id): """ Get the call request and the schedule for the given schedule id @param schedule_id: id of the schedule for the call request @type schedule_id: str @return: scheduled call report dictionary @rtype: dict """ if isinstance(schedule_id, basestring): schedule_id = ObjectId(schedule_id) scheduled_call = self.scheduled_call_collection.find_one({'_id': schedule_id}) if scheduled_call is None: raise pulp_exceptions.MissingResource(schedule=str(schedule_id)) report = scheduled_call_to_report_dict(scheduled_call) return report
def test_expired_object_id(self): """ Make sure that _create_expired_object_id() generates correct ObjectIds. """ expired_oid = _create_expired_object_id(timedelta(seconds=1)) # The oid should be about a second, but since we didn't create the now_oid until after we # ran the _create_expired_object_id() function, we can't assert that the timedelta is # exactly one second. It should definitely be less than two seconds of difference between # them, however (unless Nose is really struggling to run the tests), so we can make sure # it fits inside the window of 1 - 2 seconds. now_oid = ObjectId() self.assertTrue((now_oid.generation_time - expired_oid.generation_time) >= timedelta(seconds=1)) self.assertTrue((now_oid.generation_time - expired_oid.generation_time) < timedelta(seconds=2)) # Also, let's make sure the type is correct self.assertTrue(isinstance(expired_oid, ObjectId))
def test_update_only_config(self): # Setup manager = manager_factory.event_listener_manager() created = manager.create(http.TYPE_ID, {'a' : 'a', 'b' : 'b'}, [event_data.TYPE_REPO_SYNC_STARTED]) # Test new_config = {'a' : 'x', 'c' : 'c'} body = { 'notifier_config' : new_config, } status, body = self.put('/v2/events/%s/' % created['id'], body) # Verify self.assertEqual(200, status) updated = EventListener.get_collection().find_one({'_id' : ObjectId(created['_id'])}) expected_config = {'a' : 'x', 'b' : 'b', 'c' : 'c'} self.assertEqual(updated['notifier_config'], expected_config)
def _repos(v1_database, v2_database): v1_coll = v1_database.repos v2_coll = v2_database.repos # Idempotency: By repo_id v2_repo_ids = [x['id'] for x in v2_coll.find({}, {'id': 1})] spec = { '$and': [ { 'id': { '$nin': v2_repo_ids } }, { 'content_types': V1_ISO_REPO }, ] } missing_v1_repos = v1_coll.find(spec) new_repos = [] for v1_repo in missing_v1_repos: id = ObjectId() # Identifying tag for the CLI v2_notes = v1_repo.get('notes', {}) v2_notes[REPO_NOTE_KEY] = REPO_NOTE_ISO v2_repo = { '_id': id, # technically not needed but added for clarity 'id': v1_repo['id'], 'display_name': v1_repo['name'], 'description': None, 'notes': v2_notes, 'scratchpad': {}, 'content_unit_count': 0 } new_repos.append(v2_repo) if new_repos: v2_coll.insert(new_repos, safe=True) return True
def update(self, schedule_id, **schedule_updates): """ Update a scheduled call request Valid schedule updates: * call_request * schedule * failure_threshold * remaining_runs * enabled @param schedule_id: id of the schedule for the call request @type schedule_id: str @param schedule_updates: updates for scheduled call @type schedule_updates: dict """ if isinstance(schedule_id, basestring): schedule_id = ObjectId(schedule_id) if self.scheduled_call_collection.find_one(schedule_id) is None: raise pulp_exceptions.MissingResource(schedule=str(schedule_id)) validate_schedule_updates(schedule_updates) call_request = schedule_updates.pop('call_request', None) if call_request is not None: schedule_updates['serialized_call_request'] = call_request.serialize() schedule = schedule_updates.get('schedule', None) if schedule is not None: interval, start, runs = dateutils.parse_iso8601_interval(schedule) schedule_updates.setdefault('remaining_runs', runs) # honor explicit update # XXX (jconnor) it'd be nice to update the next_run if the schedule # has changed, but it requires mucking with the internals of the # of the scheduled call instance, which is all encapsulated in the # ScheduledCall constructor # the next_run field will be correctly updated after the next run self.scheduled_call_collection.update({'_id': schedule_id}, {'$set': schedule_updates}, safe=True)
def _associate_distribution(v1_distribution, v2_distribution, v2_ass_coll): # This functions just like errata associations so check _errata for comments # on this approach. new_associations = [] for repo_id in v1_distribution['repoids']: new_association = { '_id': ObjectId(), 'repo_id': repo_id, 'unit_id': v2_distribution['_id'], 'unit_type_id': 'distribution', 'owner_type': DEFAULT_OWNER_TYPE, 'owner_id': DEFAULT_OWNER_ID, 'created': DEFAULT_CREATED, 'updated': DEFAULT_UPDATED, } new_associations.append(new_association) if new_associations: v2_ass_coll.insert(new_associations, safe=True)
def _consumer_bindings(v2_database, v1_consumer): v2_coll = v2_database.consumer_bindings consumer_id = v1_consumer['id'] repo_ids = v1_consumer['repoids'] # Idempotency: Uniqueness is determined by the tuple of consumer and repo ID bound_repo_ids = [x['repo_id'] for x in v2_coll.find({'consumer_id' : consumer_id})] unbound_repo_ids = set(repo_ids) - set(bound_repo_ids) new_bindings = [] for repo_id in unbound_repo_ids: binding = { '_id' : ObjectId(), 'consumer_id' : consumer_id, 'repo_id' : repo_id, 'distributor_id' : YUM_DISTRIBUTOR_ID, 'deleted' : False, } new_bindings.append(binding) if new_bindings: v2_coll.insert(new_bindings, safe=True)
def _consumer_groups(v1_database, v2_database): v1_coll = v1_database.consumergroups # seriously, no underscore v2_coll = v2_database.consumer_groups # Idempotency: By consumer group ID v1_group_ids = [x['id'] for x in v1_coll.find({}, {'id' : 1})] v2_existing_ids = [x['id'] for x in v2_coll.find({}, {'id' : 1})] ids_to_add = set(v1_group_ids) - set(v2_existing_ids) for group_id in ids_to_add: v1_group = v1_coll.find_one({'id' : group_id}) v2_group = { '_id' : ObjectId(), 'id' : v1_group['id'], 'display_name' : v1_group['id'], # no display name in v1 'description' : v1_group['description'], 'consumer_ids' : v1_group['consumerids'], # again, no underscores 'notes' : v1_group['key_value_pairs'], 'scratchpad' : None, # no idea why this is in the model, this doesn't make any sense } # Likely a small amount of groups, not worried about batching the inserts v2_coll.insert(v2_group, safe=True)
def _package_group_categories(v1_database, v2_database, report): # These act nearly identically to groups, so see the comments in there # for more information. # Idempotency: As with groups, pre-load the tuples of repo ID to category # ID into memory and use that to check for the category's existed before # inserting. v2_coll = v2_database.units_package_category v2_ass_coll = v2_database.repo_content_units # Tuple of repo ID and group ID already_added_tuples = [(x['repo_id'], x['id']) for x in v2_coll.find({}, { 'repo_id': 1, 'id': 1 })] v1_repos = v1_database.repos.find({}, { 'id': 1, 'packagegroupcategories': 1 }) for v1_repo in v1_repos: for category_id in v1_repo.get('packagegroupcategories', {}).keys(): # Idempotency check if (v1_repo['id'], category_id) in already_added_tuples: continue v1_category = v1_repo['packagegroupcategories'][category_id] category_id = str(uuid.uuid4()) new_category = { '_id': category_id, '_storage_path': None, '_content_type_id': 'package_category', 'description': v1_category['description'], 'display_order': v1_category['display_order'], 'id': v1_category['id'], 'name': v1_category['name'], 'packagegroupids': v1_category['packagegroupids'], 'repo_id': v1_repo['id'], 'translated_description': v1_category['translated_description'], 'translated_name': v1_category['translated_name'], } v2_coll.insert(new_category, safe=True) new_association = { '_id': ObjectId(), 'repo_id': v1_repo['id'], 'unit_id': category_id, 'unit_type_id': 'package_category', 'owner_type': DEFAULT_OWNER_TYPE, 'owner_id': DEFAULT_OWNER_ID, 'created': DEFAULT_CREATED, 'updated': DEFAULT_UPDATED, } v2_ass_coll.insert(new_association, safe=True) return True
def _package_groups(v1_database, v2_database, report): # In v2, the unique identifier for a package group is the # pairing of the group ID and the repository it's in. In v1, the package # group is embedded in the repo document itself, which is where we get # that information from. # In v1 the repository owns the relationship to a package group. Don't look # at the model class itself, it wasn't added there, but the code will still # stuff it in under the key "packagegroups". The value at that key is a dict # of group ID to a PackageGroup instance (v1 model). # Idempotency: The simplest way to handle this is to pre-load the set of # repo ID/group ID tuples into memory and verify each group found in each # v1 repo against that to determine if it has successfully been added or # not. The nice part about this over letting the uniqueness checks in mongo # itself is that we can batch the group inserts. v2_coll = v2_database.units_package_group v2_ass_coll = v2_database.repo_content_units # Tuple of repo ID and group ID already_added_tuples = [(x['repo_id'], x['id']) for x in v2_coll.find({}, { 'repo_id': 1, 'id': 1 })] v1_repos = v1_database.repos.find({}, {'id': 1, 'packagegroups': 1}) for v1_repo in v1_repos: for group_id in v1_repo.get('packagegroups', {}).keys(): # Idempotency check if (v1_repo['id'], group_id) in already_added_tuples: continue v1_group = v1_repo['packagegroups'][group_id] v2_group_id = str(uuid.uuid4()) new_group = { '_id': v2_group_id, '_storage_path': None, '_content_type_id': 'package_group', 'conditional_package_names': v1_group['conditional_package_names'], 'default': v1_group['default'], 'default_package_names': v1_group['default_package_names'], 'description': v1_group['description'], 'display_order': v1_group['display_order'], 'id': v1_group['id'], 'langonly': v1_group['langonly'], 'mandatory_package_names': v1_group['mandatory_package_names'], 'name': v1_group['name'], 'optional_package_names': v1_group['optional_package_names'], 'repo_id': v1_repo['id'], 'translated_description': v1_group['translated_description'], 'translated_name': v1_group['translated_name'], 'user_visible': v1_group['user_visible'], } v2_coll.insert(new_group, safe=True) new_association = { '_id': ObjectId(), 'repo_id': v1_repo['id'], 'unit_id': v2_group_id, 'unit_type_id': 'package_group', 'owner_type': DEFAULT_OWNER_TYPE, 'owner_id': DEFAULT_OWNER_ID, 'created': DEFAULT_CREATED, 'updated': DEFAULT_UPDATED, } v2_ass_coll.insert(new_association, safe=True) return True
def _errata(v1_database, v2_database, report): v1_coll = v1_database.errata v2_coll = v2_database.units_erratum v2_ass_coll = v2_database.repo_content_units # Idempotency: We're lucky here, the uniqueness is just by ID, so we can # do a pre-fetch and determine what needs to be added. v2_errata_ids = [x['id'] for x in v2_coll.find({}, {'id': 1})] missing_v1_errata = v1_coll.find({'id': {'$nin': v2_errata_ids}}) for v1_erratum in missing_v1_errata: erratum_id = str(uuid.uuid4()) new_erratum = { '_id': erratum_id, '_storage_path': None, '_content_type_id': 'erratum', 'description': v1_erratum['description'], 'from_str': v1_erratum['from_str'], 'id': v1_erratum['id'], 'issued': v1_erratum['issued'], 'pkglist': v1_erratum.get('pkglist', []), 'pushcount': v1_erratum['pushcount'], 'reboot_suggested': v1_erratum['reboot_suggested'], 'references': v1_erratum['references'], 'release': v1_erratum['release'], 'rights': v1_erratum['rights'], 'severity': v1_erratum['severity'], 'solution': v1_erratum['solution'], 'status': v1_erratum['status'], 'summary': v1_erratum['summary'], 'title': v1_erratum['title'], 'type': v1_erratum['type'], 'updated': v1_erratum['updated'], 'version': v1_erratum['version'], } v2_coll.insert(new_erratum, safe=True) # Throughout most of the upgrade scripts, they can be cancelled and # resumed at any point and it will do the right thing. In this case, # it's a nightmare to cross-reference the v1 erratum against the v2 # _id field. So adding the association here isn't 100% safe in the event # the user ctrl+c's the upgrade (which they shouldn't do anyway) but # it's close enough. new_associations = [] for repo_id in v1_erratum['repoids']: new_association = { '_id': ObjectId(), 'repo_id': repo_id, 'unit_id': erratum_id, 'unit_type_id': 'erratum', 'owner_type': DEFAULT_OWNER_TYPE, 'owner_id': DEFAULT_OWNER_ID, 'created': DEFAULT_CREATED, 'updated': DEFAULT_UPDATED, } new_associations.append(new_association) if new_associations: v2_ass_coll.insert(new_associations, safe=True) return True
def __init__(self): self._id = ObjectId() self.id = str( self._id) # legacy behavior, would love to rid ourselves of this
def _repo_distributors(v1_database, v2_database, report): v1_coll = v1_database.repos v2_repo_coll = v2_database.repos v2_dist_coll = v2_database.repo_distributors # Only the yum distributor is added in this process. The export distributor # will be added to any repositories that do not have it as part of the # normal DB migration process. # Idempotency: Only one distributor is added per repo, so check for that v2_distributor_repo_ids = [x['repo_id'] for x in v2_dist_coll.find({}, {'repo_id' : 1})] spec = { '$and' : [ {'id' : {'$nin' : v2_distributor_repo_ids}}, {'content_types' : V1_YUM_REPO}, ] } missing_v1_repos = v1_coll.find(spec) new_distributors = [] for v1_repo in missing_v1_repos: # Sanity check that the repository was added to v2. This should never # happen, but we should account for it anyway. v2_repo = v2_repo_coll.find_one({'id' : v1_repo['id']}) if v2_repo is None: report.error('Repository [%s] not found in the v2 database; ' 'distributor addition being canceled' % v1_repo['id']) return False new_distributor = { '_id' : ObjectId(), 'id' : YUM_DISTRIBUTOR_ID, 'repo_id' : v1_repo['id'], 'distributor_type_id' : YUM_DISTRIBUTOR_TYPE_ID, 'auto_publish' : True, 'scratchpad' : None, 'last_publish' : v1_repo['last_sync'], # in v1 sync and publish are the same, so close enough 'scheduled_publishes' : [], # likely don't need to revisit, the sync/auto-publish will take care } config = { 'relative_url' : v1_repo['relative_path'], 'http' : False, 'https' : True, } # Load values from the static server.conf file if not SKIP_SERVER_CONF: parser = SafeConfigParser() parser.read(V1_SERVER_CONF) if parser.has_option('security', 'ssl_ca_certificate'): # Read in the contents of the certificate and store in the # configuration in the DB. ca_filename = parser.get('security', 'ssl_ca_certificate') try: # Not bothering with a existence check, let any problems # trigger the warning in the except f = open(ca_filename, 'r') ca_cert_contents = f.read() f.close() config['https_ca'] = ca_cert_contents except: report.warning('Could not read SSL CA certificate at [%s] for ' 'repository [%s]' % (ca_filename, v1_repo['id'])) # Load the GPG keys from disk if present repo_key_dir = os.path.join(GPG_KEY_ROOT, v1_repo['relative_path']) if os.path.exists(repo_key_dir) and not SKIP_GPG_KEYS: key_filenames = os.listdir(repo_key_dir) if len(key_filenames) > 0: filename = os.path.join(repo_key_dir, key_filenames[0]) try: f = open(filename, 'r') key_contents = f.read() f.close() config['gpgkey'] = key_contents except: report.warning('Could not read GPG key at [%s] for ' 'repository [%s]' % (filename, v1_repo['id'])) new_distributor['config'] = config new_distributors.append(new_distributor) if new_distributors: v2_dist_coll.insert(new_distributors, safe=True) return True
def _repo_importers(v1_database, v2_database, report): v1_coll = v1_database.repos v2_repo_coll = v2_database.repos v2_imp_coll = v2_database.repo_importers # Idempotency: There is a single importer per repo, so we can simply check # for an importer with the given repo ID v2_importer_repo_ids = [x['repo_id'] for x in v2_imp_coll.find({}, {'repo_id' : 1})] spec = { '$and' : [ {'id' : {'$nin' : v2_importer_repo_ids}}, {'content_types' : V1_YUM_REPO}, ] } missing_v1_repos = v1_coll.find(spec) new_importers = [] for v1_repo in missing_v1_repos: # Sanity check that the repository was added to v2. This should never # happen, but we should account for it anyway. v2_repo = v2_repo_coll.find_one({'id' : v1_repo['id']}) if v2_repo is None: report.error('Repository [%s] not found in the v2 database; ' 'importer addition being canceled' % v1_repo['id']) return False new_importer = { '_id' : ObjectId(), 'id' : YUM_IMPORTER_ID, 'repo_id' : v1_repo['id'], 'importer_type_id' : YUM_IMPORTER_TYPE_ID, 'scratchpad' : None, 'last_sync' : v1_repo['last_sync'], 'scheduled_syncs' : [], } # The configuration intentionally omits the importer configuration # values: num_threads, num_old_packages, remove_old, verify_checksum, # verify_size, max_speed # Being omitted will cause the yum importer to use the default, which # is the desired behavior of the upgrade. # All are set below. To keep consistent with a fresh install, the fields aren't # defaulted to None but rather omitted entirely. config = {} if v1_repo['source']: # will be None for a feedless repo config['feed_url'] = v1_repo['source']['url'] # Load the certificate content into the database. It needs to be written to the # working directory as well, but that will be done in the filesystem scripts. if v1_repo['feed_ca']: if not os.path.exists(v1_repo['feed_ca']): continue f = open(v1_repo['feed_ca'], 'r') cert = f.read() f.close() config['ssl_ca_cert'] = cert if v1_repo['feed_cert']: if not os.path.exists(v1_repo['feed_cert']): continue f = open(v1_repo['feed_cert'], 'r') cert = f.read() f.close() config['ssl_client_cert'] = cert # Load values from the static server.conf file if not SKIP_SERVER_CONF: parser = SafeConfigParser() parser.read(V1_SERVER_CONF) for o in ('proxy_url', 'proxy_port', 'proxy_user', 'proxy_pass'): if parser.has_option('yum', o): config[o] = parser.get('yum', o) new_importer['config'] = config new_importers.append(new_importer) if new_importers: v2_imp_coll.insert(new_importers, safe=True) return True
def _sync_schedules(v1_database, v2_database, report): v1_repo_collection = v1_database.repos v2_repo_importer_collection = v2_database.repo_importers v2_scheduled_call_collection = v2_database.scheduled_calls # ugly hack to find out which repos have already been scheduled # necessary because $size is not a meta-query and doesn't support $gt, etc repos_without_schedules = v2_repo_importer_collection.find( {'scheduled_syncs': { '$size': 0 }}, fields=['repo_id']) repo_ids_without_schedules = [ r['repo_id'] for r in repos_without_schedules ] repos_with_schedules = v2_repo_importer_collection.find( {'repo_id': { '$nin': repo_ids_without_schedules }}, fields=['repo_id']) repo_ids_with_schedules = [r['repo_id'] for r in repos_with_schedules] repos_to_schedule = v1_repo_collection.find( { 'id': { '$nin': repo_ids_with_schedules }, 'sync_schedule': { '$ne': None } }, fields=['id', 'sync_schedule', 'sync_options', 'last_sync']) for repo in repos_to_schedule: if repo['id'] not in repo_ids_without_schedules: report.error('Repository [%s] not found in the v2 database.' 'sync scheduling being canceled.' % repo['id']) return False args = [repo['id']] kwargs = {'overrides': {}} call_request = CallRequest(sync_with_auto_publish_itinerary, args, kwargs, principal=SystemUser()) scheduled_call_document = { '_id': ObjectId(), 'id': None, 'serialized_call_request': None, 'schedule': repo['sync_schedule'], 'failure_threshold': None, 'consecutive_failures': 0, 'first_run': None, 'last_run': None, 'next_run': None, 'remaining_runs': None, 'enabled': True } scheduled_call_document['id'] = str(scheduled_call_document['_id']) schedule_tag = resource_tag(dispatch_constants.RESOURCE_SCHEDULE_TYPE, scheduled_call_document['id']) call_request.tags.append(schedule_tag) scheduled_call_document[ 'serialized_call_request'] = call_request.serialize() if isinstance(repo['sync_options'], dict): scheduled_call_document['failure_threshold'] = repo[ 'sync_options'].get('failure_threshold', None) interval, start, recurrences = dateutils.parse_iso8601_interval( scheduled_call_document['schedule']) scheduled_call_document['first_run'] = start or datetime.utcnow() scheduled_call_document['remaining_runs'] = recurrences scheduled_call_document['next_run'] = _calculate_next_run( scheduled_call_document) if repo['last_sync'] is not None: scheduled_call_document[ 'last_run'] = dateutils.to_naive_utc_datetime( dateutils.parse_iso8601_datetime(repo['last_sync'])) v2_scheduled_call_collection.insert(scheduled_call_document, safe=True) v2_repo_importer_collection.update( {'repo_id': repo['id']}, {'$push': { 'scheduled_syncs': scheduled_call_document['id'] }}, safe=True) return True
def test_expired_object_id(self): expired_oid = self.reaper._create_expired_object_id( timedelta(seconds=1)) self.assertTrue(isinstance(expired_oid, ObjectId)) now_oid = ObjectId() self.assertTrue(now_oid > expired_oid)