def get_file(self, datatype, filename, outdir='.', chunk_size=1000*5120, check_md5=False): """Fetch a file in chunks. """ logger.info('getting %s', self.url + '/' + datatype + '/' + filename) if check_md5: md5 = hashlib.md5() localfile = os.path.join(outdir, filename) fob = open(localfile, 'w+b') offset = 0 while True: buf = self.server.get_file_chunk(datatype, filename, offset, chunk_size, timeout=1800) buf = buf.data if not buf: break if check_md5: md5.update(buf) fob.write(buf) offset += len(buf) fob.close() logger.info('saved %s (%d bytes)', localfile, offset) if check_md5: logger.info('md5 check on %s', filename) remote_md5 = self.server.get_file_md5(datatype, filename, timeout=1800) if remote_md5 != md5.hexdigest(): logger.error('md5 check failed on %s', filename) return None return localfile
def get(self, timeout=None): """Return a published message or None if a timeout has happend. """ if timeout: timeout *= 1000. self.subscriber.connect(self.destination) poller = zmq.Poller() poller.register(self.subscriber, zmq.POLLIN) try: while(True): try: ret = poller.poll(timeout=timeout) if ret: if ret[0][0] == self.subscriber: msg = Message.decode( self.subscriber.recv(zmq.NOBLOCK)) yield msg else: logger.error("WHAT THE HECK") else: # timeout yield None except zmq.ZMQError: logger.exception('recv failed') finally: poller.unregister(self.subscriber) self.subscriber.close()
def get(self, timeout=None): """Return a published message or None if a timeout has happend. """ if timeout: timeout *= 1000. self.subscriber.connect(self.destination) poller = zmq.Poller() poller.register(self.subscriber, zmq.POLLIN) try: while (True): try: ret = poller.poll(timeout=timeout) if ret: if ret[0][0] == self.subscriber: msg = Message.decode( self.subscriber.recv(zmq.NOBLOCK)) yield msg else: logger.error("WHAT THE HECK") else: # timeout yield None except zmq.ZMQError: logger.exception('recv failed') finally: poller.unregister(self.subscriber) self.subscriber.close()