def __init__(self): super(Uploader, self).__init__() self.stopped = True self.log = logging.getLogger('irgsh_node.uploader') self.queue = Queue(settings.LOCAL_DATABASE) self.force_stop = False
def upload(self, task_id, path, content_type, extra={}): from irgsh_node.localqueue import Queue data = {} data.update(extra) data.update({'task_id': task_id, 'path': path, 'content_type': content_type}) queue = Queue(settings.LOCAL_DATABASE) queue.put(data)
def upload(self, task_id, path, content_type, extra={}): from irgsh_node.localqueue import Queue data = {} data.update(extra) data.update({ 'task_id': task_id, 'path': path, 'content_type': content_type }) queue = Queue(settings.LOCAL_DATABASE) queue.put(data)
class Uploader(object): def __init__(self): super(Uploader, self).__init__() self.stopped = True self.log = logging.getLogger('irgsh_node.uploader') self.queue = Queue(settings.LOCAL_DATABASE) self.force_stop = False def stop(self): self.stopped = True def start(self): signal.signal(signal.SIGINT, self.sigint_handler) self.force_stop = False self.run() def sigint_handler(self, sig, frame): self.stopped = True if not self.force_stop: self.log.info('Ctrl+C was pressed, finishing current job..') self.log.info('Press Ctrl+C again to force stop the uploader.') self.force_stop = True else: self.log.info('Stopping the uploader by force.') sys.exit(1) def run(self): delay = 0.1 self.log.info('Uploader started') self.stopped = False while not self.stopped: time.sleep(delay) # Get last item in the upload queue item = self.queue.get() if item is None: delay = delay * 2 if delay > 5: delay = 5 continue delay = 0.1 self.log.debug('Got a new item to upload, id=%s' % item.id) data = item.payload content_type = data['content_type'] task_id = data['task_id'] path = data['path'] # Check if the file exists taskdir = os.path.join(settings.RESULT_DIR, task_id) fname = os.path.join(taskdir, path) if not os.path.exists(fname): self.log.error('File not found: %s' % fname) continue # Upload it reset = False try: if content_type == consts.TYPE_RESULT: distribution = Distribution(**data['distribution']) manager.send_changes(task_id, fname) self.send_result(task_id, distribution, fname) elif content_type == consts.TYPE_LOG: manager.send_log(task_id, fname) else: self.queue.remove(item) self.log.error('Unrecognized content type: %s' % content_type) continue # Success! Remove item from the queue self.queue.remove(item) self.log.debug('Data uploaded') except UploadFailedError, e: self.log.error(str(e)) # TODO: add task log remotely # manager.send_log(task_id, 'Failed to upload result: (%s) %s' % \ # (e.code, e.log)) manager.update_status(task_id, manager.FAILED) except IOError, e: reset = True if isinstance(e, HTTPError): if e.code == 404: self.log.error('Task is not registered') reset = False if reset: # Fail! Reset item so it will be picked up again self.queue.reset(item) self.log.error('Failed to upload, send item back to queue: %s' % e)
class Uploader(object): def __init__(self): super(Uploader, self).__init__() self.stopped = True self.log = logging.getLogger('irgsh_node.uploader') self.queue = Queue(settings.LOCAL_DATABASE) self.force_stop = False def stop(self): self.stopped = True def start(self): signal.signal(signal.SIGINT, self.sigint_handler) self.force_stop = False self.run() def sigint_handler(self, sig, frame): self.stopped = True if not self.force_stop: self.log.info('Ctrl+C was pressed, finishing current job..') self.log.info('Press Ctrl+C again to force stop the uploader.') self.force_stop = True else: self.log.info('Stopping the uploader by force.') sys.exit(1) def run(self): delay = 0.1 self.log.info('Uploader started') self.stopped = False while not self.stopped: time.sleep(delay) # Get last item in the upload queue item = self.queue.get() if item is None: delay = delay * 2 if delay > 5: delay = 5 continue delay = 0.1 self.log.debug('Got a new item to upload, id=%s' % item.id) data = item.payload content_type = data['content_type'] task_id = data['task_id'] path = data['path'] # Check if the file exists taskdir = os.path.join(settings.RESULT_DIR, task_id) fname = os.path.join(taskdir, path) if not os.path.exists(fname): self.log.error('File not found: %s' % fname) continue # Upload it reset = False try: if content_type == consts.TYPE_RESULT: distribution = Distribution(**data['distribution']) manager.send_changes(task_id, fname) self.send_result(task_id, distribution, fname) elif content_type == consts.TYPE_LOG: manager.send_log(task_id, fname) else: self.queue.remove(item) self.log.error('Unrecognized content type: %s' % content_type) continue # Success! Remove item from the queue self.queue.remove(item) self.log.debug('Data uploaded') except UploadFailedError, e: self.log.error(str(e)) # TODO: add task log remotely # manager.send_log(task_id, 'Failed to upload result: (%s) %s' % \ # (e.code, e.log)) manager.update_status(task_id, manager.FAILED) except IOError, e: reset = True if isinstance(e, HTTPError): if e.code == 404: self.log.error('Task is not registered') reset = False if reset: # Fail! Reset item so it will be picked up again self.queue.reset(item) self.log.error( 'Failed to upload, send item back to queue: %s' % e)