def set_progress(self, status):
"""
Informs the server of the current state of the publish operation. The
contents of the status is dependent on how the distributor
implementation chooses to divide up the publish process.
@param status: contains arbitrary data to describe the state of the
publish; the contents may contain whatever information is relevant
to the distributor implementation so long as it is serializable
"""
try:
self.progress_report[self.report_id] = status
context = dispatch_factory.context()
context.report_progress(self.progress_report)
except Exception, e:
_LOG.exception(
'Exception from server setting progress for report [%s]' %
self.report_id)
try:
_LOG.error('Progress value: %s' % str(status))
except Exception:
# Best effort to print this, but if its that grossly unserializable
# the log will tank and we don't want that exception to bubble up
pass
raise self.exception_class(e), None, sys.exc_info()[2]
def set_progress(self, status):
"""
Informs the server of the current state of the publish operation. The
contents of the status is dependent on how the distributor
implementation chooses to divide up the publish process.
@param status: contains arbitrary data to describe the state of the
publish; the contents may contain whatever information is relevant
to the distributor implementation so long as it is serializable
"""
try:
self.progress_report[self.report_id] = status
context = dispatch_factory.context()
delta = {'progress_report': self.progress_report}
TaskStatusManager.update_task_status(context.call_request_id, delta)
context.report_progress(self.progress_report)
except Exception, e:
_LOG.exception('Exception from server setting progress for report [%s]' % self.report_id)
try:
_LOG.error('Progress value: %s' % str(status))
except Exception:
# Best effort to print this, but if its that grossly unserializable
# the log will tank and we don't want that exception to bubble up
pass
raise self.exception_class(e), None, sys.exc_info()[2]
def __init__(self, consumer):
self.uuid = consumer['id']
certificate = consumer.get('certificate')
hash = hashlib.sha256()
hash.update(certificate.strip())
self.secret = hash.hexdigest()
self.taskid = factory.context().task_id
def bind(self, consumer_id, repo_id, distributor_id, options):
"""
Request the agent to perform the specified bind. This method will be called
after the server-side representation of the binding has been created.
:param consumer_id: The consumer ID.
:type consumer_id: str
:param repo_id: A repository ID.
:type repo_id: str
:param distributor_id: A distributor ID.
:type distributor_id: str
:param options: The options are handler specific.
:type options: dict
"""
# agent request
consumer_manager = managers.consumer_manager()
binding_manager = managers.consumer_bind_manager()
consumer = consumer_manager.get_consumer(consumer_id)
binding = binding_manager.get_bind(consumer_id, repo_id,
distributor_id)
agent_bindings = self.__bindings([binding])
agent = PulpAgent(consumer)
agent.consumer.bind(agent_bindings, options)
# request tracking
action_id = factory.context().call_request_id
consumer_manager = managers.consumer_bind_manager()
consumer_manager.action_pending(consumer_id, repo_id, distributor_id,
Bind.Action.BIND, action_id)
def __call__(self, *args, **kwargs):
"""
This overrides CeleryTask's __call__() method. We use this method
for task state tracking of Pulp tasks.
"""
# Add task_id to the task context, so that agent and plugins have access to the task id.
# There are a few other attributes in the context as defined by old dispatch system.
# These are unused right now. These should be removed when we cleanup the dispatch folder
# after the migration to celery is complete.
task_context = dispatch_factory.context()
task_context.call_request_id = self.request.id
# Check task status and skip running the task if task state is 'canceled'.
task_status = TaskStatusManager.find_by_task_id(task_id=self.request.id)
if task_status and task_status['state'] == dispatch_constants.CALL_CANCELED_STATE:
logger.debug("Task cancel received for task-id : [%s]" % self.request.id)
return
# Update start_time and set the task state to 'running' for asynchronous tasks.
# Skip updating status for eagerly executed tasks, since we don't want to track
# synchronous tasks in our database.
if not self.request.called_directly:
# Using 'upsert' to avoid a possible race condition described in the apply_async method
# above.
TaskStatus.get_collection().update(
{'task_id': self.request.id},
{'$set': {'state': dispatch_constants.CALL_RUNNING_STATE,
'start_time': dateutils.now_utc_timestamp()}},
upsert=True)
# Run the actual task
logger.debug("Running task : [%s]" % self.request.id)
return super(Task, self).__call__(*args, **kwargs)
def sync(self, repo_id, sync_config_override=None):
"""
Performs a synchronize operation on the given repository.
The given repo must have an importer configured. The identity of the
importer is not a parameter to this call; if multiple importers are
eventually supported this will have to change to indicate which
importer to use.
This method is intentionally limited to synchronizing a single repo.
Performing multiple repository syncs concurrently will require a more
global view of the server and must be handled outside the scope of this
class.
@param repo_id: identifies the repo to sync
@type repo_id: str
@param sync_config_override: optional config containing values to use
for this sync only
@type sync_config_override: dict
@raise MissingResource: if repo_id does not refer to a valid repo
@raise OperationFailed: if the given repo does not have an importer set
"""
repo_coll = Repo.get_collection()
# Validation
repo = repo_coll.find_one({'id' : repo_id})
if repo is None:
raise MissingResource(repo_id)
importer_instance, importer_config = self._get_importer_instance_and_config(repo_id)
if importer_instance is None:
raise MissingResource(repo_id)
dispatch_context = dispatch_factory.context()
dispatch_context.set_cancel_control_hook(importer_instance.cancel_sync_repo)
importer_manager = manager_factory.repo_importer_manager()
repo_importer = importer_manager.get_importer(repo_id)
# Assemble the data needed for the sync
conduit = RepoSyncConduit(repo_id, repo_importer['id'], RepoContentUnit.OWNER_TYPE_IMPORTER, repo_importer['id'])
call_config = PluginCallConfiguration(importer_config, repo_importer['config'], sync_config_override)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(repo_importer['importer_type_id'], repo_id, mkdir=True)
# Fire an events around the call
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_sync_started(repo_id)
sync_result = self._do_sync(repo, importer_instance, transfer_repo, conduit, call_config)
fire_manager.fire_repo_sync_finished(sync_result)
dispatch_context.clear_cancel_control_hook()
if sync_result['result'] == RepoSyncResult.RESULT_FAILED:
raise PulpExecutionException(_('Importer indicated a failed response'))
def unbind(self, consumer_id, repo_id, distributor_id, options):
"""
Request the agent to perform the specified unbind.
@param consumer_id: The consumer ID.
@type consumer_id: str
@param repo_id: A repository ID.
@type repo_id: str
@param distributor_id: A distributor ID.
@type distributor_id: str
@param options: The options are handler specific.
@type options: dict
"""
# agent request
manager = managers.consumer_manager()
consumer = manager.get_consumer(consumer_id)
binding = dict(repo_id=repo_id, distributor_id=distributor_id)
bindings = self.__unbindings([binding])
agent = PulpAgent(consumer)
agent.consumer.unbind(bindings, options)
# request tracking
action_id = factory.context().call_request_id
manager = managers.consumer_bind_manager()
manager.action_pending(
consumer_id,
repo_id,
distributor_id,
Bind.Action.UNBIND,
action_id)
def bind(self, consumer_id, repo_id, distributor_id, options):
"""
Request the agent to perform the specified bind. This method will be called
after the server-side representation of the binding has been created.
:param consumer_id: The consumer ID.
:type consumer_id: str
:param repo_id: A repository ID.
:type repo_id: str
:param distributor_id: A distributor ID.
:type distributor_id: str
:param options: The options are handler specific.
:type options: dict
"""
# agent request
consumer_manager = managers.consumer_manager()
binding_manager = managers.consumer_bind_manager()
consumer = consumer_manager.get_consumer(consumer_id)
binding = binding_manager.get_bind(consumer_id, repo_id, distributor_id)
agent_bindings = self.__bindings([binding])
agent = PulpAgent(consumer)
agent.consumer.bind(agent_bindings, options)
# request tracking
action_id = factory.context().call_request_id
consumer_manager = managers.consumer_bind_manager()
consumer_manager.action_pending(
consumer_id,
repo_id,
distributor_id,
Bind.Action.BIND,
action_id)
def _get_call_report():
context = dispatch_factory.context()
coordinator = dispatch_factory.coordinator()
call_report_list = coordinator.find_call_reports(call_request_id=context.call_request_id)
if not call_report_list:
return None
return call_report_list[0].serialize()
def publish(self, repo_id, distributor_id, publish_config_override=None):
"""
Requests the given distributor publish the repository it is configured
on.
The publish operation is executed synchronously in the caller's thread
and will block until it is completed. The caller must take the necessary
steps to address the fact that a publish call may be time intensive.
@param repo_id: identifies the repo being published
@type repo_id: str
@param distributor_id: identifies the repo's distributor to publish
@type distributor_id: str
@param publish_config_override: optional config values to use for this
publish call only
@type publish_config_override: dict, None
:return: report of the details of the publish
:rtype: pulp.server.plugins.model.PublishReport
"""
repo_coll = Repo.get_collection()
distributor_coll = RepoDistributor.get_collection()
# Validation
repo = repo_coll.find_one({'id' : repo_id})
if repo is None:
raise MissingResource(repo_id)
repo_distributor = distributor_coll.find_one({'repo_id' : repo_id, 'id' : distributor_id})
if repo_distributor is None:
raise MissingResource(repository=repo_id, distributor=distributor_id)
distributor_instance, distributor_config = self._get_distributor_instance_and_config(repo_id, distributor_id)
if distributor_instance is None:
raise MissingResource(repo_id), None, sys.exc_info()[2]
dispatch_context = dispatch_factory.context()
dispatch_context.set_cancel_control_hook(distributor_instance.cancel_publish_repo)
# Assemble the data needed for the publish
conduit = RepoPublishConduit(repo_id, distributor_id)
call_config = PluginCallConfiguration(distributor_config, repo_distributor['config'], publish_config_override)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.distributor_working_dir(repo_distributor['distributor_type_id'], repo_id, mkdir=True)
# Fire events describing the publish state
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_publish_started(repo_id, distributor_id)
result = self._do_publish(repo, distributor_id, distributor_instance, transfer_repo, conduit, call_config)
fire_manager.fire_repo_publish_finished(result)
dispatch_context.clear_cancel_control_hook()
return result
def _get_call_report():
context = dispatch_factory.context()
coordinator = dispatch_factory.coordinator()
call_report_list = coordinator.find_call_reports(
call_request_id=context.call_request_id)
if not call_report_list:
return None
return call_report_list[0].serialize()
def __init__(self, consumer):
self.uuid = consumer['id']
self.url = config.get('messaging', 'url')
certificate = consumer.get('certificate')
hash = hashlib.sha256()
hash.update(certificate.strip())
self.secret = hash.hexdigest()
self.call_request_id = factory.context().call_request_id
def __init__(self, consumer):
self.uuid = consumer["id"]
self.url = config.get("messaging", "url")
certificate = consumer.get("certificate")
hash = hashlib.sha256()
hash.update(certificate.strip())
self.secret = hash.hexdigest()
self.call_request_id = factory.context().call_request_id
def __init__(self, consumer):
self.uuid = consumer['id']
self.url = config.get('messaging', 'url')
certificate = consumer.get('certificate')
hash = hashlib.sha256()
hash.update(certificate.strip())
self.secret = hash.hexdigest()
self.call_request_id = factory.context().call_request_id
def _current_task_state():
context = dispatch_factory.context()
if context.call_request_id is None:
return None
coordinator = dispatch_factory.coordinator()
tasks = coordinator._find_tasks(call_request_id=context.call_request_id)
if not tasks:
return None
return tasks[0].state
def test_task_context(self):
"""
Assert that the task context has task id correctly populated in it.
"""
expected_task_id = 'test_task_id'
@celery.task(base=tasks.Task)
def fake_task():
task_context = dispatch_factory.context()
self.assertEqual(task_context.call_request_id, expected_task_id)
task = fake_task
task.request.id = expected_task_id
task.__call__()
task_context = dispatch_factory.context()
self.assertEqual(task_context.call_request_id, expected_task_id)
def unbind(self, consumer_id, repo_id, distributor_id, options):
"""
Request the agent to perform the specified unbind.
:param consumer_id: The consumer ID.
:type consumer_id: str
:param repo_id: A repository ID.
:type repo_id: str
:param distributor_id: A distributor ID.
:type distributor_id: str
:param options: The options are handler specific.
:type options: dict
"""
# agent request
manager = managers.consumer_manager()
consumer = manager.get_consumer(consumer_id)
binding = dict(repo_id=repo_id, distributor_id=distributor_id)
bindings = self.__unbindings([binding])
agent = PulpAgent(consumer)
agent.consumer.unbind(bindings, options)
# request tracking
action_id = factory.context().call_request_id
manager = managers.consumer_bind_manager()
manager.action_pending(consumer_id, repo_id, distributor_id,
Bind.Action.UNBIND, action_id)
def fake_task():
task_context = dispatch_factory.context()
self.assertEqual(task_context.call_request_id, expected_task_id)
def publish(self, repo_id, distributor_id, publish_config_override=None):
"""
Requests the given distributor publish the repository it is configured
on.
The publish operation is executed synchronously in the caller's thread
and will block until it is completed. The caller must take the necessary
steps to address the fact that a publish call may be time intensive.
@param repo_id: identifies the repo being published
@type repo_id: str
@param distributor_id: identifies the repo's distributor to publish
@type distributor_id: str
@param publish_config_override: optional config values to use for this
publish call only
@type publish_config_override: dict, None
"""
repo_coll = Repo.get_collection()
distributor_coll = RepoDistributor.get_collection()
# Validation
repo = repo_coll.find_one({'id': repo_id})
if repo is None:
raise MissingResource(repo_id)
repo_distributor = distributor_coll.find_one({
'repo_id': repo_id,
'id': distributor_id
})
if repo_distributor is None:
raise MissingResource(repository=repo_id,
distributor=distributor_id)
distributor_instance, distributor_config = self._get_distributor_instance_and_config(
repo_id, distributor_id)
if distributor_instance is None:
raise MissingResource(repo_id), None, sys.exc_info()[2]
dispatch_context = dispatch_factory.context()
dispatch_context.set_cancel_control_hook(
distributor_instance.cancel_publish_repo)
# Assemble the data needed for the publish
conduit = RepoPublishConduit(repo_id, distributor_id)
call_config = PluginCallConfiguration(distributor_config,
repo_distributor['config'],
publish_config_override)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.distributor_working_dir(
repo_distributor['distributor_type_id'], repo_id, mkdir=True)
# Fire events describing the publish state
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_publish_started(repo_id, distributor_id)
result = self._do_publish(repo, distributor_id, distributor_instance,
transfer_repo, conduit, call_config)
fire_manager.fire_repo_publish_finished(result)
dispatch_context.clear_cancel_control_hook()
def sync(self, repo_id, sync_config_override=None):
"""
Performs a synchronize operation on the given repository.
The given repo must have an importer configured. The identity of the
importer is not a parameter to this call; if multiple importers are
eventually supported this will have to change to indicate which
importer to use.
This method is intentionally limited to synchronizing a single repo.
Performing multiple repository syncs concurrently will require a more
global view of the server and must be handled outside the scope of this
class.
@param repo_id: identifies the repo to sync
@type repo_id: str
@param sync_config_override: optional config containing values to use
for this sync only
@type sync_config_override: dict
@return: The synchronization report.
@rtype: L{pulp.server.plugins.model.SyncReport}
@raise MissingResource: if repo_id does not refer to a valid repo
@raise OperationFailed: if the given repo does not have an importer set
"""
repo_coll = Repo.get_collection()
# Validation
repo = repo_coll.find_one({'id' : repo_id})
if repo is None:
raise MissingResource(repo_id)
importer_instance, importer_config = self._get_importer_instance_and_config(repo_id)
if importer_instance is None:
raise MissingResource(repo_id)
dispatch_context = dispatch_factory.context()
dispatch_context.set_cancel_control_hook(importer_instance.cancel_sync_repo)
importer_manager = manager_factory.repo_importer_manager()
repo_importer = importer_manager.get_importer(repo_id)
# Assemble the data needed for the sync
conduit = RepoSyncConduit(repo_id, repo_importer['id'], RepoContentUnit.OWNER_TYPE_IMPORTER, repo_importer['id'])
call_config = PluginCallConfiguration(importer_config, repo_importer['config'], sync_config_override)
transfer_repo = common_utils.to_transfer_repo(repo)
transfer_repo.working_dir = common_utils.importer_working_dir(repo_importer['importer_type_id'], repo_id, mkdir=True)
# Fire an events around the call
fire_manager = manager_factory.event_fire_manager()
fire_manager.fire_repo_sync_started(repo_id)
sync_result = self._do_sync(repo, importer_instance, transfer_repo, conduit, call_config)
fire_manager.fire_repo_sync_finished(sync_result)
dispatch_context.clear_cancel_control_hook()
if sync_result['result'] == RepoSyncResult.RESULT_FAILED:
raise PulpExecutionException(_('Importer indicated a failed response'))
return sync_result
