def send_result(self, task_id, distribution, changes):
manager.update_status(task_id, manager.UPLOADING)
try:
fd, log = tempfile.mkstemp('-irgsh-upload')
flog = open(log, 'wb')
stdout = stderr = flog
path = 'incoming/%s/' % task_id
opts = {'user': settings.UPLOAD_USER,
'host': settings.UPLOAD_HOST,
'path': path}
dput = Dput(distribution, **opts)
dput.upload(changes, stdout, stderr)
flog.close()
except UploadFailedError, e:
flog.close()
e.log = open(log).read()
raise e
def send_result(self, task_id, distribution, changes):
manager.update_status(task_id, manager.UPLOADING)
try:
fd, log = tempfile.mkstemp('-irgsh-upload')
flog = open(log, 'wb')
stdout = stderr = flog
path = 'incoming/%s/' % task_id
opts = {
'user': settings.UPLOAD_USER,
'host': settings.UPLOAD_HOST,
'path': path
}
dput = Dput(distribution, **opts)
dput.upload(changes, stdout, stderr)
flog.close()
except UploadFailedError, e:
flog.close()
e.log = open(log).read()
raise e
def update_status(self, task_id, status):
try:
manager.update_status(task_id, status)
except IOError:
pass
def update_status(self, task_id, status):
try:
manager.update_status(task_id, status)
except IOError:
pass
def run(self):
delay = 0.1
self.log.info('Uploader started')
self.stopped = False
while not self.stopped:
time.sleep(delay)
# Get last item in the upload queue
item = self.queue.get()
if item is None:
delay = delay * 2
if delay > 5:
delay = 5
continue
delay = 0.1
self.log.debug('Got a new item to upload, id=%s' % item.id)
data = item.payload
content_type = data['content_type']
task_id = data['task_id']
path = data['path']
# Check if the file exists
taskdir = os.path.join(settings.RESULT_DIR, task_id)
fname = os.path.join(taskdir, path)
if not os.path.exists(fname):
self.log.error('File not found: %s' % fname)
continue
# Upload it
reset = False
try:
if content_type == consts.TYPE_RESULT:
distribution = Distribution(**data['distribution'])
manager.send_changes(task_id, fname)
self.send_result(task_id, distribution, fname)
elif content_type == consts.TYPE_LOG:
manager.send_log(task_id, fname)
else:
self.queue.remove(item)
self.log.error('Unrecognized content type: %s' % content_type)
continue
# Success! Remove item from the queue
self.queue.remove(item)
self.log.debug('Data uploaded')
except UploadFailedError, e:
self.log.error(str(e))
# TODO: add task log remotely
# manager.send_log(task_id, 'Failed to upload result: (%s) %s' % \
# (e.code, e.log))
manager.update_status(task_id, manager.FAILED)
except IOError, e:
reset = True
if isinstance(e, HTTPError):
if e.code == 404:
self.log.error('Task is not registered')
reset = False
path = 'incoming/%s/' % task_id
opts = {'user': settings.UPLOAD_USER,
'host': settings.UPLOAD_HOST,
'path': path}
dput = Dput(distribution, **opts)
dput.upload(changes, stdout, stderr)
flog.close()
except UploadFailedError, e:
flog.close()
e.log = open(log).read()
raise e
finally:
os.unlink(log)
manager.update_status(task_id, manager.FINISHED)
def main():
uploader = Uploader()
uploader.start()
# TODO
# - add ability to launch multiple uploaders
if __name__ == '__main__':
main()
def run(self):
delay = 0.1
self.log.info('Uploader started')
self.stopped = False
while not self.stopped:
time.sleep(delay)
# Get last item in the upload queue
item = self.queue.get()
if item is None:
delay = delay * 2
if delay > 5:
delay = 5
continue
delay = 0.1
self.log.debug('Got a new item to upload, id=%s' % item.id)
data = item.payload
content_type = data['content_type']
task_id = data['task_id']
path = data['path']
# Check if the file exists
taskdir = os.path.join(settings.RESULT_DIR, task_id)
fname = os.path.join(taskdir, path)
if not os.path.exists(fname):
self.log.error('File not found: %s' % fname)
continue
# Upload it
reset = False
try:
if content_type == consts.TYPE_RESULT:
distribution = Distribution(**data['distribution'])
manager.send_changes(task_id, fname)
self.send_result(task_id, distribution, fname)
elif content_type == consts.TYPE_LOG:
manager.send_log(task_id, fname)
else:
self.queue.remove(item)
self.log.error('Unrecognized content type: %s' %
content_type)
continue
# Success! Remove item from the queue
self.queue.remove(item)
self.log.debug('Data uploaded')
except UploadFailedError, e:
self.log.error(str(e))
# TODO: add task log remotely
# manager.send_log(task_id, 'Failed to upload result: (%s) %s' % \
# (e.code, e.log))
manager.update_status(task_id, manager.FAILED)
except IOError, e:
reset = True
if isinstance(e, HTTPError):
if e.code == 404:
self.log.error('Task is not registered')
reset = False
'host': settings.UPLOAD_HOST,
'path': path
}
dput = Dput(distribution, **opts)
dput.upload(changes, stdout, stderr)
flog.close()
except UploadFailedError, e:
flog.close()
e.log = open(log).read()
raise e
finally:
os.unlink(log)
manager.update_status(task_id, manager.FINISHED)
def main():
uploader = Uploader()
uploader.start()
# TODO
# - add ability to launch multiple uploaders
if __name__ == '__main__':
main()
