def get_file(self, datatype, filename,
outdir='.',
chunk_size=1000*5120,
check_md5=False):
"""Fetch a file in chunks.
"""
logger.info('getting %s', self.url + '/' + datatype + '/' + filename)
if check_md5:
md5 = hashlib.md5()
localfile = os.path.join(outdir, filename)
fob = open(localfile, 'w+b')
offset = 0
while True:
buf = self.server.get_file_chunk(datatype, filename,
offset, chunk_size, timeout=1800)
buf = buf.data
if not buf:
break
if check_md5:
md5.update(buf)
fob.write(buf)
offset += len(buf)
fob.close()
logger.info('saved %s (%d bytes)', localfile, offset)
if check_md5:
logger.info('md5 check on %s', filename)
remote_md5 = self.server.get_file_md5(datatype, filename,
timeout=1800)
if remote_md5 != md5.hexdigest():
logger.error('md5 check failed on %s', filename)
return None
return localfile
def get(self, timeout=None):
"""Return a published message or None if a timeout has happend.
"""
if timeout:
timeout *= 1000.
self.subscriber.connect(self.destination)
poller = zmq.Poller()
poller.register(self.subscriber, zmq.POLLIN)
try:
while(True):
try:
ret = poller.poll(timeout=timeout)
if ret:
if ret[0][0] == self.subscriber:
msg = Message.decode(
self.subscriber.recv(zmq.NOBLOCK))
yield msg
else:
logger.error("WHAT THE HECK")
else:
# timeout
yield None
except zmq.ZMQError:
logger.exception('recv failed')
finally:
poller.unregister(self.subscriber)
self.subscriber.close()
def get(self, timeout=None):
"""Return a published message or None if a timeout has happend.
"""
if timeout:
timeout *= 1000.
self.subscriber.connect(self.destination)
poller = zmq.Poller()
poller.register(self.subscriber, zmq.POLLIN)
try:
while (True):
try:
ret = poller.poll(timeout=timeout)
if ret:
if ret[0][0] == self.subscriber:
msg = Message.decode(
self.subscriber.recv(zmq.NOBLOCK))
yield msg
else:
logger.error("WHAT THE HECK")
else:
# timeout
yield None
except zmq.ZMQError:
logger.exception('recv failed')
finally:
poller.unregister(self.subscriber)
self.subscriber.close()
