def __init__(self):
"""Constructor."""
# Do singleton checking.
if Scavenger.INSTANCE != None:
raise SingletonException('A Scavenger instance already exists.')
# Initialize the object.
super(Scavenger, self).__init__()
# Create a context monitor.
self._monitor = ContextMonitor()
# Create the schedulers.
self._schedulers = {}
self._schedulers['aprofile'] = AdaptiveProfScheduler(self._monitor._context, Scavenger)
# Load in the config.
self._config = Config.get_instance()
# Set the local activity count.
self._activity = LocalActivity()
# Assign the instance pointer.
Scavenger.INSTANCE = self
class Scavenger(object):
INSTANCE = None
@classmethod
def get_instance(cls):
if cls.INSTANCE != None:
return cls.INSTANCE
else:
cls()
if not cls.INSTANCE:
raise SingletonException('Error creating Scavenger instance.')
return cls.INSTANCE
def __init__(self):
"""Constructor."""
# Do singleton checking.
if Scavenger.INSTANCE != None:
raise SingletonException('A Scavenger instance already exists.')
# Initialize the object.
super(Scavenger, self).__init__()
# Create a context monitor.
self._monitor = ContextMonitor()
# Create the schedulers.
self._schedulers = {}
self._schedulers['aprofile'] = AdaptiveProfScheduler(self._monitor._context, Scavenger)
# Load in the config.
self._config = Config.get_instance()
# Set the local activity count.
self._activity = LocalActivity()
# Assign the instance pointer.
Scavenger.INSTANCE = self
@classmethod
def get_peers(cls):
return cls.INSTANCE._get_peers()
def _get_peers(self):
"""
Get a list of known peers offering the scavenger service.
@rtype: list
@return: A list of ScavengerPeer objects for the peers that are currently available.
"""
return self._monitor.get_peers()
@classmethod
def perform_task(cls, peer, task_name, task_input, connection=None,
timeout=ScavengerDefines.TIMEOUT, store=False):
return cls.INSTANCE._perform_task(peer, task_name, task_input,
connection, timeout, store)
def _perform_task(self, peer, task_name, task_input, connection=None,
timeout=ScavengerDefines.TIMEOUT, store=False):
"""
Try to perform a task at a remote scavenger host.
@type peer: ScavengerPeer
@param peer: The peer where the task should be performed.
@type task_name: str
@param task_name: The name of the task.
@type task_input: dict
@param task_input: The input for the task.
@type connection: SCProxy
@param connection: An initiated connection to a Scavenger peer.
@type timeout: float
@param timeout: The number of seconds to wait for the result before
quitting.
@type store: bool
@param store: Whether or not the surrogate should store the result locally.
@raise ScavengerException: If the surrogate can not be contacted, or if
an error occurs during remote execution.
"""
print peer.name, #DEBUG
# Check that the peer is still there.
if not self._monitor.has_peer(peer.name):
raise ScavengerException('No such peer is within range.')
# Fire the RPC call.
proxy = connection if connection != None else SCProxy(peer.address)
try:
return proxy.perform_task(task_name, task_input, timeout, store)
finally:
if connection == None:
proxy.close()
@classmethod
def perform_scheduled_task(cls, peer, task, connection = None):
return cls.INSTANCE._perform_scheduled_task(peer, task, connection)
def _perform_scheduled_task(self, peer, task, connection):
print peer.name, #DEBUG
# Check that the peer is still there.
if not self._monitor.has_peer(peer.name):
raise ScavengerException('No such peer is within range.')
# Fire the RPC call.
proxy = connection if connection != None else SCProxy(peer.address)
try:
if task.scheduler in ('aprofile'):
result, complexity = proxy.perform_task(task.name, task.input, ScavengerDefines.TIMEOUT, task.store, True)
if task.scheduler == 'aprofile':
self._schedulers[task.scheduler].gprofile.register(task.name, complexity, task.complexity)
self._schedulers[task.scheduler].lprofile.register((peer.name, task.name), complexity, task.complexity)
return result
else:
return proxy.perform_task(task.name, task.input, ScavengerDefines.TIMEOUT, task.store, False)
finally:
if connection == None:
proxy.close()
@classmethod
def install_task(cls, peer, task_name, task_code, connection=None):
cls.INSTANCE._install_task(peer, task_name, task_code, connection)
def _install_task(self, peer, task_name, task_code, connection=None):
"""
Installs the given task onto the given peer.
@type peer: ScavengerPeer
@param peer: The peer where the task is to be installed.
@type task_name: str
@param task_name: The name of the new task. This must be on
the form name1.name2.name3, e.g., 'daimi.imaging.scale'.
@type task_code: str
@param task_code: The source code of the task.
@type connection: SCProxy
@param connection: An initiated connection to a Scavenger peer.
@raise ScavengerException: If the peer cannot be contacted, or if an
error occurs at the remote peer during installation.
"""
# Check that the peer is still there.
if not self._monitor.has_peer(peer.name):
raise ScavengerException('No such peer is within range.')
# Fire the RPC call.
proxy = connection if connection != None else SCProxy(peer.address)
try:
proxy.install_task(task_name, task_code)
finally:
if connection == None:
proxy.close()
@classmethod
def has_task(cls, peer, task_name, connection=None):
return cls.INSTANCE._has_task(peer, task_name, connection)
def _has_task(self, peer, task_name, connection=None):
"""
Checks whether the given peer offers the named task.
@type peer: ScavengerPeer
@param peer: The peer that is to be checked.
@type task_name: str
@param task_name: The name of the task.
@type connection: SCProxy
@param connection: An initiated connection to a Scavenger peer.
@raise ScavengerException: If the peer can not be contacted, or if
an error occurs at the remote peer.
"""
# Check that the peer is still there.
if not self._monitor.has_peer(peer.name):
raise ScavengerException('No such peer is within range.')
# Fire the RPC call.
proxy = connection if connection != None else SCProxy(peer.address)
try:
return proxy.has_task(task_name)
finally:
if connection == None:
proxy.close()
def _resolve_data_handles(self, task_input):
"""Resolves any remote data handles in the input so that
local execution may be performed."""
if type(task_input) == dict:
for key, value in task_input.items():
if type(value) == RemoteDataHandle:
task_input[key] = Scavenger.fetch_data(value)
elif type(task_input) in (tuple, list):
new_task_input = []
for item in task_input:
if type(item) == RemoteDataHandle:
new_task_input.append(Scavenger.fetch_data(item))
else:
new_task_input.append(item)
task_input = new_task_input
else:
if type(task_input) == RemoteDataHandle:
task_input = Scavenger.fetch_data(task_input)
return task_input
@classmethod
def scavenge(cls, task_name, task_input, task_code=None, local_code=None):
task_invocation = AdaptiveProfTaskInvokation(task_name, task_input, task_code, output_size='0')
return cls.INSTANCE._scavenge(task_invocation, local_code)
def _scavenge(self, task, local_code=None):
"""
This method offers opportunistic use of nearby computing resources.
This is all done using the adaptive profiling scheduler.
@type task: AdaptiveProfileTaskInvokation
@param task: The task that must be invoked. This object
contains the task name, input, code and possible more information about
the task.
@type local_code: function
@param local_code: A local function that is capable of performing the
task. This may be the same code that is used when performing the task
remotely, or it may be a 'lighter' version of the code that yields a lower
quality result but is better suited for executing on a small device.
@rtype: Depends on the task being performed.
@return: The result of performing the task.
@raise ScavengerException: For lots of reasons...
"""
# Schedule the task execution.
try:
# Ask the scheduler to schedule the task.
if local_code == None:
# If we do not have local code we need to enable prefer_remote.
return self._schedulers[task.scheduler].schedule(task,
self._config.getfloat('cpu', 'strength'),
self._config.getint('network', 'speed'),
self._activity,
True)
else:
return self._schedulers[task.scheduler].schedule(task,
self._config.getfloat('cpu', 'strength'),
self._config.getint('network', 'speed'),
self._activity)
except ScheduleError:
# Remote execution was not possible. Do local execution if possible.
if local_code != None:
print 'localhost', #DEBUG
# Resolve any remote data handles.
task.input = self._resolve_data_handles(task.input)
def perform_local_function(task_input):
try:
# Perform the local function.
if type(task_input) == dict:
return local_code(**task_input)
elif type(task_input) in (tuple, list):
return local_code(*task_input)
else:
return local_code(task_input)
finally:
self._activity.decrement()
if task.scheduler in ('aprofile'):
# We need to profile this task run.
start = time()
start_activity = self._activity.value
result = perform_local_function(task.input)
stop_activity = self._activity.value + 1
stop = time()
activity_level = float(start_activity + stop_activity) / 2
complexity = ((stop-start) * self._config.getfloat('cpu', 'strength')) / activity_level
if task.scheduler == 'aprofile':
self._schedulers[task.scheduler].gprofile.register(task.name, complexity, task.complexity)
self._schedulers[task.scheduler].lprofile.register(('localhost', task.name), complexity, task.complexity)
return result
else:
return perform_local_function(task.input)
else:
raise ScavengerException('No surrogates available.')
@classmethod
def scavenge_partial(cls, task_invokation, local_function, *task_input, **kwargs):
invocation = deepcopy(task_invokation)
invocation.input = task_input
if kwargs.has_key('id'):
invocation.id = kwargs['id']
return cls.INSTANCE._scavenge(invocation, local_function)
@classmethod
def shutdown(cls):
cls.INSTANCE._shutdown()
def _shutdown(self):
"""Make a clean break from Presence."""
self._monitor.shutdown()
# Save the profiling data.
self._schedulers['aprofile'].lprofile.save()
self._schedulers['aprofile'].gprofile.save()
# self._schedulers['aprofile']._log.close()
@classmethod
def resolve(cls, peer_name):
"""Recolves a peer name (presence id) to an ip,port tuple."""
return cls.INSTANCE._monitor._context.resolve(peer_name)
@classmethod
def fetch_data(cls, rdh, connection=None):
return rdh.fetch(connection, cls.INSTANCE._monitor._context)
@classmethod
def store_data(cls, peer, data, connection=None):
# Check that the peer is still there.
if not cls.INSTANCE._monitor.has_peer(peer.name):
raise ScavengerException('No such peer is within range.')
# Fire the RPC call.
proxy = connection if connection != None else SCProxy(peer.address)
try:
return proxy.store_data(data)
finally:
if connection == None:
proxy.close()
@classmethod
def retain_data(cls, rdh, connection=None):
rdh.refresh(connection, cls.INSTANCE._monitor._context)
@classmethod
def expire_data(cls, rdh, connection=None):
rdh.expire(connection, cls.INSTANCE._monitor._context)
