def connect(self):
# register with Queue
SyncManager.register('getPipeline')
SyncManager.register('getStore')
self.qInstance= self.opts.qInstance
(self.qHost, self.qPort, self.qKey)= self.opts.queue.split(':')
queue= SyncManager(address= (self.qHost, int(self.qPort)), authkey= self.qKey)
queue.connect()
self.pipeline= queue.getPipeline()
self.store= queue.getStore()
class Manager(object):
def __init__(self, opts):
self.opts= opts
super(Manager, self).__init__()
self.sleep= self.opts.sleep
self.alive= True
self.workers= dict()
self.connect()
'''
The fully qualified domain name for the aws ec2 instance
should match what the instance private_dns_name is
'''
self.id= getfqdn()
def connect(self):
# register with queue
SyncManager.register('getPipeline')
SyncManager.register('getStore')
SyncManager.register('setFileContents')
SyncManager.register('getFileContents')
SyncManager.register('deleteFile')
(qHost, qPort, qKey)= self.opts.queue.split(':')
self.queue= SyncManager(address= (qHost, int(qPort)), authkey= qKey)
self.queue.connect()
self.pipeline= self.queue.getPipeline()
self.store= self.queue.getStore()
# register with dfs
self.dfs = None
self.instances= dict()
if self.opts.dfs != None:
SyncManager.register('getInstances')
(dHost, dPort, dKey)= self.opts.dfs.split(':')
self.dfs= SyncManager(address= (dHost, int(dPort)), authkey= dKey)
self.dfs.connect()
self.instances= self.dfs.getInstances()
def run(self):
while self.alive:
try:
# stop tracking dead workers
[self.workers.pop(pid) for (pid, worker) in self.workers.items() if not worker.is_alive()]
instanceStore= self.instances.get(self.id, dict())
# update dfs worker availability
availability= self.opts.maxProcesses - len(self.workers)
self.instances.update([(self.id, dict(
id= self.id,
status= 'running',
capacity= self.opts.maxProcesses,
availability= availability,
lastTask= instanceStore.get('lastTask', datetime.strftime(datetime.utcnow(), '%Y-%m-%dT%H:%M:%S.000Z')
)
))])
print "========================================================"
print "Queue:", self.pipeline.qsize()
print "Store:", len(self.store)
print "Capacity:", self.opts.maxProcesses
print 'Workers:', len(self.workers)
print "Availability:", self.opts.maxProcesses - len(self.workers)
print "--------------------------------------------------------"
# create workers
for i in range(min(self.pipeline.qsize(), self.opts.maxProcesses - len(self.workers))):
worker= Worker(self.opts, self.id, availability)
worker.start()
self.workers[worker.pid]= worker
except EOFError:
self.connect()
except IOError:
self.connect()
sleep(self.sleep)
# if manager is shutting down -- then wait for workers to finish
print "manager shutting down"
map(lambda (pid, worker): worker.join(), self.workers.items())
def stop(self):
print "de-registering with dfs -- all workers down"
'''
tell dfs are are shutting down and have no capacity/availabilty
if dfs doesn't know who we are then create a default stub
'''
self.instances.update([(self.id, dict(
id= self.id,
status= 'running',
capacity= 0,
availability= 0
))])
self.alive= False
class Worker(Process):
def __init__(self, opts, id, availability):
super(Worker, self).__init__()
self.opts= opts
self.id= id
self.availability= availability
self.connect()
self.alive= True
self.sleep= self.opts.sleep
def connect(self):
(qHost, qPort, qKey)= self.opts.queue.split(':')
self.queue= SyncManager(address= (qHost, int(qPort)), authkey= qKey)
self.queue.connect()
self.pipeline= self.queue.getPipeline()
self.store= self.queue.getStore()
# register with DFS
self.dfs = None
self.instances= dict()
if self.opts.dfs != None:
SyncManager.register('getInstances')
(dHost, dPort, dKey)= self.opts.dfs.split(':')
self.dfs= SyncManager(address= (dHost, int(dPort)), authkey= dKey)
self.dfs.connect()
self.instances= self.dfs.getInstances()
def handleTransport(self, processId, transport, results):
'''
Handles requested transport types
'''
if transport == 's3' and self.opts.s3 != None:
try:
(accessKey, secretKey, bucket)= self.opts.s3.split(':')
s3file= S3File(
accessKey= accessKey,
secretKey= secretKey,
bucket= bucket,
processId= processId,
mode= 'w'
)
s3file.write(results)
results= s3file.getName()
s3file.close()
transport= 's3'
except Exception, e:
print >> stderr, "s3 transport failure using data store instead: %s" % (str(e))
elif transport == 'file' and self.opts.taskDir != None:
try:
fileStore= FileStore(proxy= self.queue, processId= processId, mode= 'w')
fileStore.write(results)
results= fileStore.getName()
fileStore.close()
transport= 'file'
except Exception, e:
print >> stderr, "fileStore transport failure using data store instead: %s" % (str(e))
class Task(object):
statuses = ["waiting", "running", "ready", "error"]
# statuses= dict([(value, id) for (id, value) in enumerate(['waiting', 'running', 'ready', 'error'], start= 1))
def __init__(self, queue, taskId=str(uuid1()), s3=None, taskDir=None):
"""creates an instance of Task
:param queue: <host>:<port>:<security key> of queue instance
:param taskId: optional, auto generated guid representing this
instance of Task. all jobs forked with thecurrent instance will
assume this taskId. optionally, the developer may pass in a taskId that is
meaing full to them.
:param s3: <access key>:<secret key>:<bucket> of s3 resource to use
when s3 transport is specified during .forkTask()
:param taskDir: output directory to write task results to
:returns: instance of Task
"""
self.taskId = str(uuid1()) if taskId == None else taskId
self.s3 = s3
self.subTaskId = 0
self.subTasks = dict()
self.sleep = 0.1
self.taskDir = taskDir
self.lock = Lock()
(qHost, qPort, qKey) = queue.split(":")
SyncManager.register("getPipeline")
SyncManager.register("getStore")
SyncManager.register("getFileContents")
SyncManager.register("setFileContents")
SyncManager.register("deleteFile")
self.queue = SyncManager(address=(qHost, int(qPort)), authkey=qKey)
self.queue.connect()
self.pipeline = self.queue.getPipeline()
self.store = self.queue.getStore()
super(Task, self).__init__()
def __genProcessId__(self, subTaskId):
"""used internally by Task to create a processId for the
users task.
:param subTaskId: the subTaskId of the task to be associated with this
processId.
:returns: a valid processId representing the process for the task.
"""
return "%s.%s" % (self.getTaskId(), subTaskId)
def getTaskId(self):
"""gets the current taskID for the instance of NoddleTask
:returns: taskId of for the instance of Task
"""
return self.taskId
def getSubTaskId(self):
"""gets the last known subTaskId
:returns: the last forked subTaskId
"""
return self.subTaskId
def getSubTaskIds(self):
"""gets a list of subTaskIds associated with the current instance of
Task.
:returns: list of subTaskIds forked by the current instance of Task
"""
with self.lock:
subTasks = deepcopy(self.subTasks)
return subTasks
def handleTransport(self, transport, results, delete=False):
"""makes the results data from the optional transports accesable
through the results interface of the data store. typically this method
is used internally by Task but is exposed to the developer as
there may be stitutations where the developer may want to resolve
the transport themselves.
:param transport: transport type (eg, 's3', 'file')
:param results: the results store from the data store
:param delete: True|False delete the originating resource (s3/file) after
resolving it.
:returns: returns results store with the transport resolved
"""
# if transport is s3 then load the results file
if transport == "s3":
if self.s3 != None:
try:
(accessKey, secretKey, bucket) = self.s3.split(":")
except Exception, e:
raise ("invalid s3 transport credentials: %s" % (str(e)))
try:
s3file = S3File(accessKey, secretKey, bucket, processId=path.basename(results), mode="r")
results = s3file.read()
s3file.close(delete=delete)
except Exception, e:
raise Exception("s3 transport error: %s" % (str(e)))
else:
