def set_progress(self, status): """ Informs the server of the current state of the publish operation. The contents of the status is dependent on how the distributor implementation chooses to divide up the publish process. @param status: contains arbitrary data to describe the state of the publish; the contents may contain whatever information is relevant to the distributor implementation so long as it is serializable """ try: self.progress_report[self.report_id] = status context = dispatch_factory.context() context.report_progress(self.progress_report) except Exception, e: _LOG.exception( 'Exception from server setting progress for report [%s]' % self.report_id) try: _LOG.error('Progress value: %s' % str(status)) except Exception: # Best effort to print this, but if its that grossly unserializable # the log will tank and we don't want that exception to bubble up pass raise self.exception_class(e), None, sys.exc_info()[2]
def set_progress(self, status): """ Informs the server of the current state of the publish operation. The contents of the status is dependent on how the distributor implementation chooses to divide up the publish process. @param status: contains arbitrary data to describe the state of the publish; the contents may contain whatever information is relevant to the distributor implementation so long as it is serializable """ try: self.progress_report[self.report_id] = status context = dispatch_factory.context() delta = {'progress_report': self.progress_report} TaskStatusManager.update_task_status(context.call_request_id, delta) context.report_progress(self.progress_report) except Exception, e: _LOG.exception('Exception from server setting progress for report [%s]' % self.report_id) try: _LOG.error('Progress value: %s' % str(status)) except Exception: # Best effort to print this, but if its that grossly unserializable # the log will tank and we don't want that exception to bubble up pass raise self.exception_class(e), None, sys.exc_info()[2]
def __init__(self, consumer): self.uuid = consumer['id'] certificate = consumer.get('certificate') hash = hashlib.sha256() hash.update(certificate.strip()) self.secret = hash.hexdigest() self.taskid = factory.context().task_id
def bind(self, consumer_id, repo_id, distributor_id, options): """ Request the agent to perform the specified bind. This method will be called after the server-side representation of the binding has been created. :param consumer_id: The consumer ID. :type consumer_id: str :param repo_id: A repository ID. :type repo_id: str :param distributor_id: A distributor ID. :type distributor_id: str :param options: The options are handler specific. :type options: dict """ # agent request consumer_manager = managers.consumer_manager() binding_manager = managers.consumer_bind_manager() consumer = consumer_manager.get_consumer(consumer_id) binding = binding_manager.get_bind(consumer_id, repo_id, distributor_id) agent_bindings = self.__bindings([binding]) agent = PulpAgent(consumer) agent.consumer.bind(agent_bindings, options) # request tracking action_id = factory.context().call_request_id consumer_manager = managers.consumer_bind_manager() consumer_manager.action_pending(consumer_id, repo_id, distributor_id, Bind.Action.BIND, action_id)
def __call__(self, *args, **kwargs): """ This overrides CeleryTask's __call__() method. We use this method for task state tracking of Pulp tasks. """ # Add task_id to the task context, so that agent and plugins have access to the task id. # There are a few other attributes in the context as defined by old dispatch system. # These are unused right now. These should be removed when we cleanup the dispatch folder # after the migration to celery is complete. task_context = dispatch_factory.context() task_context.call_request_id = self.request.id # Check task status and skip running the task if task state is 'canceled'. task_status = TaskStatusManager.find_by_task_id(task_id=self.request.id) if task_status and task_status['state'] == dispatch_constants.CALL_CANCELED_STATE: logger.debug("Task cancel received for task-id : [%s]" % self.request.id) return # Update start_time and set the task state to 'running' for asynchronous tasks. # Skip updating status for eagerly executed tasks, since we don't want to track # synchronous tasks in our database. if not self.request.called_directly: # Using 'upsert' to avoid a possible race condition described in the apply_async method # above. TaskStatus.get_collection().update( {'task_id': self.request.id}, {'$set': {'state': dispatch_constants.CALL_RUNNING_STATE, 'start_time': dateutils.now_utc_timestamp()}}, upsert=True) # Run the actual task logger.debug("Running task : [%s]" % self.request.id) return super(Task, self).__call__(*args, **kwargs)
def sync(self, repo_id, sync_config_override=None): """ Performs a synchronize operation on the given repository. The given repo must have an importer configured. The identity of the importer is not a parameter to this call; if multiple importers are eventually supported this will have to change to indicate which importer to use. This method is intentionally limited to synchronizing a single repo. Performing multiple repository syncs concurrently will require a more global view of the server and must be handled outside the scope of this class. @param repo_id: identifies the repo to sync @type repo_id: str @param sync_config_override: optional config containing values to use for this sync only @type sync_config_override: dict @raise MissingResource: if repo_id does not refer to a valid repo @raise OperationFailed: if the given repo does not have an importer set """ repo_coll = Repo.get_collection() # Validation repo = repo_coll.find_one({'id' : repo_id}) if repo is None: raise MissingResource(repo_id) importer_instance, importer_config = self._get_importer_instance_and_config(repo_id) if importer_instance is None: raise MissingResource(repo_id) dispatch_context = dispatch_factory.context() dispatch_context.set_cancel_control_hook(importer_instance.cancel_sync_repo) importer_manager = manager_factory.repo_importer_manager() repo_importer = importer_manager.get_importer(repo_id) # Assemble the data needed for the sync conduit = RepoSyncConduit(repo_id, repo_importer['id'], RepoContentUnit.OWNER_TYPE_IMPORTER, repo_importer['id']) call_config = PluginCallConfiguration(importer_config, repo_importer['config'], sync_config_override) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir(repo_importer['importer_type_id'], repo_id, mkdir=True) # Fire an events around the call fire_manager = manager_factory.event_fire_manager() fire_manager.fire_repo_sync_started(repo_id) sync_result = self._do_sync(repo, importer_instance, transfer_repo, conduit, call_config) fire_manager.fire_repo_sync_finished(sync_result) dispatch_context.clear_cancel_control_hook() if sync_result['result'] == RepoSyncResult.RESULT_FAILED: raise PulpExecutionException(_('Importer indicated a failed response'))
def unbind(self, consumer_id, repo_id, distributor_id, options): """ Request the agent to perform the specified unbind. @param consumer_id: The consumer ID. @type consumer_id: str @param repo_id: A repository ID. @type repo_id: str @param distributor_id: A distributor ID. @type distributor_id: str @param options: The options are handler specific. @type options: dict """ # agent request manager = managers.consumer_manager() consumer = manager.get_consumer(consumer_id) binding = dict(repo_id=repo_id, distributor_id=distributor_id) bindings = self.__unbindings([binding]) agent = PulpAgent(consumer) agent.consumer.unbind(bindings, options) # request tracking action_id = factory.context().call_request_id manager = managers.consumer_bind_manager() manager.action_pending( consumer_id, repo_id, distributor_id, Bind.Action.UNBIND, action_id)
def bind(self, consumer_id, repo_id, distributor_id, options): """ Request the agent to perform the specified bind. This method will be called after the server-side representation of the binding has been created. :param consumer_id: The consumer ID. :type consumer_id: str :param repo_id: A repository ID. :type repo_id: str :param distributor_id: A distributor ID. :type distributor_id: str :param options: The options are handler specific. :type options: dict """ # agent request consumer_manager = managers.consumer_manager() binding_manager = managers.consumer_bind_manager() consumer = consumer_manager.get_consumer(consumer_id) binding = binding_manager.get_bind(consumer_id, repo_id, distributor_id) agent_bindings = self.__bindings([binding]) agent = PulpAgent(consumer) agent.consumer.bind(agent_bindings, options) # request tracking action_id = factory.context().call_request_id consumer_manager = managers.consumer_bind_manager() consumer_manager.action_pending( consumer_id, repo_id, distributor_id, Bind.Action.BIND, action_id)
def _get_call_report(): context = dispatch_factory.context() coordinator = dispatch_factory.coordinator() call_report_list = coordinator.find_call_reports(call_request_id=context.call_request_id) if not call_report_list: return None return call_report_list[0].serialize()
def publish(self, repo_id, distributor_id, publish_config_override=None): """ Requests the given distributor publish the repository it is configured on. The publish operation is executed synchronously in the caller's thread and will block until it is completed. The caller must take the necessary steps to address the fact that a publish call may be time intensive. @param repo_id: identifies the repo being published @type repo_id: str @param distributor_id: identifies the repo's distributor to publish @type distributor_id: str @param publish_config_override: optional config values to use for this publish call only @type publish_config_override: dict, None :return: report of the details of the publish :rtype: pulp.server.plugins.model.PublishReport """ repo_coll = Repo.get_collection() distributor_coll = RepoDistributor.get_collection() # Validation repo = repo_coll.find_one({'id' : repo_id}) if repo is None: raise MissingResource(repo_id) repo_distributor = distributor_coll.find_one({'repo_id' : repo_id, 'id' : distributor_id}) if repo_distributor is None: raise MissingResource(repository=repo_id, distributor=distributor_id) distributor_instance, distributor_config = self._get_distributor_instance_and_config(repo_id, distributor_id) if distributor_instance is None: raise MissingResource(repo_id), None, sys.exc_info()[2] dispatch_context = dispatch_factory.context() dispatch_context.set_cancel_control_hook(distributor_instance.cancel_publish_repo) # Assemble the data needed for the publish conduit = RepoPublishConduit(repo_id, distributor_id) call_config = PluginCallConfiguration(distributor_config, repo_distributor['config'], publish_config_override) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.distributor_working_dir(repo_distributor['distributor_type_id'], repo_id, mkdir=True) # Fire events describing the publish state fire_manager = manager_factory.event_fire_manager() fire_manager.fire_repo_publish_started(repo_id, distributor_id) result = self._do_publish(repo, distributor_id, distributor_instance, transfer_repo, conduit, call_config) fire_manager.fire_repo_publish_finished(result) dispatch_context.clear_cancel_control_hook() return result
def _get_call_report(): context = dispatch_factory.context() coordinator = dispatch_factory.coordinator() call_report_list = coordinator.find_call_reports( call_request_id=context.call_request_id) if not call_report_list: return None return call_report_list[0].serialize()
def __init__(self, consumer): self.uuid = consumer['id'] self.url = config.get('messaging', 'url') certificate = consumer.get('certificate') hash = hashlib.sha256() hash.update(certificate.strip()) self.secret = hash.hexdigest() self.call_request_id = factory.context().call_request_id
def __init__(self, consumer): self.uuid = consumer["id"] self.url = config.get("messaging", "url") certificate = consumer.get("certificate") hash = hashlib.sha256() hash.update(certificate.strip()) self.secret = hash.hexdigest() self.call_request_id = factory.context().call_request_id
def _current_task_state(): context = dispatch_factory.context() if context.call_request_id is None: return None coordinator = dispatch_factory.coordinator() tasks = coordinator._find_tasks(call_request_id=context.call_request_id) if not tasks: return None return tasks[0].state
def test_task_context(self): """ Assert that the task context has task id correctly populated in it. """ expected_task_id = 'test_task_id' @celery.task(base=tasks.Task) def fake_task(): task_context = dispatch_factory.context() self.assertEqual(task_context.call_request_id, expected_task_id) task = fake_task task.request.id = expected_task_id task.__call__() task_context = dispatch_factory.context() self.assertEqual(task_context.call_request_id, expected_task_id)
def unbind(self, consumer_id, repo_id, distributor_id, options): """ Request the agent to perform the specified unbind. :param consumer_id: The consumer ID. :type consumer_id: str :param repo_id: A repository ID. :type repo_id: str :param distributor_id: A distributor ID. :type distributor_id: str :param options: The options are handler specific. :type options: dict """ # agent request manager = managers.consumer_manager() consumer = manager.get_consumer(consumer_id) binding = dict(repo_id=repo_id, distributor_id=distributor_id) bindings = self.__unbindings([binding]) agent = PulpAgent(consumer) agent.consumer.unbind(bindings, options) # request tracking action_id = factory.context().call_request_id manager = managers.consumer_bind_manager() manager.action_pending(consumer_id, repo_id, distributor_id, Bind.Action.UNBIND, action_id)
def fake_task(): task_context = dispatch_factory.context() self.assertEqual(task_context.call_request_id, expected_task_id)
def publish(self, repo_id, distributor_id, publish_config_override=None): """ Requests the given distributor publish the repository it is configured on. The publish operation is executed synchronously in the caller's thread and will block until it is completed. The caller must take the necessary steps to address the fact that a publish call may be time intensive. @param repo_id: identifies the repo being published @type repo_id: str @param distributor_id: identifies the repo's distributor to publish @type distributor_id: str @param publish_config_override: optional config values to use for this publish call only @type publish_config_override: dict, None """ repo_coll = Repo.get_collection() distributor_coll = RepoDistributor.get_collection() # Validation repo = repo_coll.find_one({'id': repo_id}) if repo is None: raise MissingResource(repo_id) repo_distributor = distributor_coll.find_one({ 'repo_id': repo_id, 'id': distributor_id }) if repo_distributor is None: raise MissingResource(repository=repo_id, distributor=distributor_id) distributor_instance, distributor_config = self._get_distributor_instance_and_config( repo_id, distributor_id) if distributor_instance is None: raise MissingResource(repo_id), None, sys.exc_info()[2] dispatch_context = dispatch_factory.context() dispatch_context.set_cancel_control_hook( distributor_instance.cancel_publish_repo) # Assemble the data needed for the publish conduit = RepoPublishConduit(repo_id, distributor_id) call_config = PluginCallConfiguration(distributor_config, repo_distributor['config'], publish_config_override) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.distributor_working_dir( repo_distributor['distributor_type_id'], repo_id, mkdir=True) # Fire events describing the publish state fire_manager = manager_factory.event_fire_manager() fire_manager.fire_repo_publish_started(repo_id, distributor_id) result = self._do_publish(repo, distributor_id, distributor_instance, transfer_repo, conduit, call_config) fire_manager.fire_repo_publish_finished(result) dispatch_context.clear_cancel_control_hook()
def sync(self, repo_id, sync_config_override=None): """ Performs a synchronize operation on the given repository. The given repo must have an importer configured. The identity of the importer is not a parameter to this call; if multiple importers are eventually supported this will have to change to indicate which importer to use. This method is intentionally limited to synchronizing a single repo. Performing multiple repository syncs concurrently will require a more global view of the server and must be handled outside the scope of this class. @param repo_id: identifies the repo to sync @type repo_id: str @param sync_config_override: optional config containing values to use for this sync only @type sync_config_override: dict @return: The synchronization report. @rtype: L{pulp.server.plugins.model.SyncReport} @raise MissingResource: if repo_id does not refer to a valid repo @raise OperationFailed: if the given repo does not have an importer set """ repo_coll = Repo.get_collection() # Validation repo = repo_coll.find_one({'id' : repo_id}) if repo is None: raise MissingResource(repo_id) importer_instance, importer_config = self._get_importer_instance_and_config(repo_id) if importer_instance is None: raise MissingResource(repo_id) dispatch_context = dispatch_factory.context() dispatch_context.set_cancel_control_hook(importer_instance.cancel_sync_repo) importer_manager = manager_factory.repo_importer_manager() repo_importer = importer_manager.get_importer(repo_id) # Assemble the data needed for the sync conduit = RepoSyncConduit(repo_id, repo_importer['id'], RepoContentUnit.OWNER_TYPE_IMPORTER, repo_importer['id']) call_config = PluginCallConfiguration(importer_config, repo_importer['config'], sync_config_override) transfer_repo = common_utils.to_transfer_repo(repo) transfer_repo.working_dir = common_utils.importer_working_dir(repo_importer['importer_type_id'], repo_id, mkdir=True) # Fire an events around the call fire_manager = manager_factory.event_fire_manager() fire_manager.fire_repo_sync_started(repo_id) sync_result = self._do_sync(repo, importer_instance, transfer_repo, conduit, call_config) fire_manager.fire_repo_sync_finished(sync_result) dispatch_context.clear_cancel_control_hook() if sync_result['result'] == RepoSyncResult.RESULT_FAILED: raise PulpExecutionException(_('Importer indicated a failed response')) return sync_result