def rpc(self, method, *args, **kwargs):
"""
Send a command to the queue. Timeout after 10 seconds
"""
pubsub = self._redis.pubsub()
channel = "%s%s" % (platform.node(), int(time.time()))
pubsub.subscribe(channel)
self._redis.rpush(
RedisTransport.COMMAND_KEY,
cPickle.dumps(RedisMessage(channel, method, args, kwargs)))
resp = pubsub.listen()
signal.signal(signal.SIGALRM, self.shutdown)
signal.alarm(10)
resp.next() # clear subscribe message
response = resp.next()
pubsub.unsubscribe()
try:
return cPickle.loads(response['data'])
except: # pylint: disable=W0702
msg = "%s: Failed to receive response: %s" % \
(self.__class__.__name__,
traceback.format_exc().splitlines()[-1])
self.logger.error(msg)
return None
def rpc(self, method, *args, **kwargs):
"""
Send a command to the queue. Timeout after 10 seconds
"""
pubsub = self._redis.pubsub()
channel = "%s%s" % (platform.node(), int(time.time()))
pubsub.subscribe(channel)
self._redis.rpush(RedisTransport.COMMAND_KEY,
cPickle.dumps(RedisMessage(channel, method, args, kwargs)))
resp = pubsub.listen()
signal.signal(signal.SIGALRM, self.shutdown)
signal.alarm(10)
resp.next() # clear subscribe message
response = resp.next()
pubsub.unsubscribe()
try:
return cPickle.loads(response['data'])
except: # pylint: disable=W0702
msg = "%s: Failed to receive response: %s" % \
(self.__class__.__name__,
traceback.format_exc().splitlines()[-1])
self.logger.error(msg)
return None
def store(self, hostname, metadata, stats):
"""Store the file to disk"""
try:
payload = cPickle.dumps(
dict(hostname=hostname, metadata=metadata, stats=stats))
except: # pylint: disable=W0702
msg = "%s: Failed to build interaction object: %s" % \
(self.__class__.__name__,
traceback.format_exc().splitlines()[-1])
self.logger.error(msg)
raise TransportError(msg)
fname = "%s-%s" % (hostname, time.time())
save_file = os.path.join(self.work_path, fname)
tmp_file = os.path.join(self.work_path, "." + fname)
if os.path.exists(save_file):
self.logger.error("%s: Oops.. duplicate statistic in directory." %
self.__class__.__name__)
raise TransportError
# using a tmpfile to hopefully avoid the file monitor from grabbing too
# soon
saved = open(tmp_file, 'wb')
try:
saved.write(payload)
except IOError:
self.logger.error(
"Failed to store interaction for %s: %s" %
(hostname, traceback.format_exc().splitlines()[-1]))
os.unlink(tmp_file)
saved.close()
os.rename(tmp_file, save_file)
def store(self, hostname, metadata, stats):
"""Store the file to disk"""
try:
payload = cPickle.dumps(dict(hostname=hostname,
metadata=metadata,
stats=stats))
except: # pylint: disable=W0702
msg = "%s: Failed to build interaction object: %s" % \
(self.__class__.__name__,
traceback.format_exc().splitlines()[-1])
self.logger.error(msg)
raise TransportError(msg)
fname = "%s-%s" % (hostname, time.time())
save_file = os.path.join(self.work_path, fname)
tmp_file = os.path.join(self.work_path, "." + fname)
if os.path.exists(save_file):
self.logger.error("%s: Oops.. duplicate statistic in directory." %
self.__class__.__name__)
raise TransportError
# using a tmpfile to hopefully avoid the file monitor from grabbing too
# soon
saved = open(tmp_file, 'wb')
try:
saved.write(payload)
except IOError:
self.logger.error("Failed to store interaction for %s: %s" %
(hostname, traceback.format_exc().splitlines()[-1]))
os.unlink(tmp_file)
saved.close()
os.rename(tmp_file, save_file)
def cachekey(self):
""" A unique key for this source that will be used to generate
:attr:`cachefile` and other cache paths """
return md5(
cPickle.dumps([
self.version, self.components, self.url, self.rawurl,
self.arches
])).hexdigest()
def hash_entry(entry_dict):
"""
Build a key for this based on its data
entry_dict = a dict of all the data identifying this
"""
dataset = []
for key in sorted(entry_dict.keys()):
if key in ('id', 'hash_key') or key.startswith('_'):
continue
dataset.append((key, entry_dict[key]))
return hash(cPickle.dumps(dataset))
def hash_entry(entry_dict):
"""
Build a key for this based on its data
entry_dict = a dict of all the data identifying this
"""
dataset = []
for key in sorted(entry_dict.keys()):
if key in ('id', 'hash_key') or key.startswith('_'):
continue
dataset.append((key, entry_dict[key]))
return hash(cPickle.dumps(dataset))
def monitor_thread(self, rclient, collector):
"""Watch the COMMAND_KEY queue for rpc commands"""
self.logger.info("Command thread started")
while not collector.terminate.isSet():
try:
payload = rclient.blpop(RedisTransport.COMMAND_KEY, timeout=5)
if not payload:
continue
message = cPickle.loads(payload[1])
if not isinstance(message, RedisMessage):
self.logger.error("Message \"%s\" is not a RedisMessage" %
message)
if not message.method in collector.storage.__class__.__rmi__ or\
not hasattr(collector.storage, message.method):
self.logger.error(
"Unknown method %s called on storage engine %s" %
(message.method, collector.storage.__class__.__name__))
raise TransportError
try:
cls_method = getattr(collector.storage, message.method)
response = cls_method(*message.args, **message.kwargs)
response = cPickle.dumps(response)
except:
self.logger.error(
"RPC method %s failed: %s" %
(message.method,
traceback.format_exc().splitlines()[-1]))
raise TransportError
rclient.publish(message.channel, response)
except redis.RedisError:
self.logger.error("Failed to fetch an interaction: %s" %
(traceback.format_exc().splitlines()[-1]))
except cPickle.UnpicklingError:
self.logger.error("Failed to unpickle payload: %s" %
traceback.format_exc().splitlines()[-1])
except TransportError:
pass
except: # pylint: disable=W0702
self.logger.error("Unhandled exception in command thread: %s" %
traceback.format_exc().splitlines()[-1])
self.logger.info("Command thread shutdown")
def store(self, hostname, metadata, stats):
"""Store the file to disk"""
try:
payload = cPickle.dumps(dict(hostname=hostname,
metadata=metadata,
stats=stats))
except: # pylint: disable=W0702
msg = "%s: Failed to build interaction object: %s" % \
(self.__class__.__name__,
traceback.format_exc().splitlines()[-1])
self.logger.error(msg)
raise TransportError(msg)
try:
self._redis.rpush(RedisTransport.STATS_KEY, payload)
except redis.RedisError:
self.logger.error("Failed to store interaction for %s: %s" %
(hostname, traceback.format_exc().splitlines()[-1]))
def monitor_thread(self, rclient, collector):
"""Watch the COMMAND_KEY queue for rpc commands"""
self.logger.info("Command thread started")
while not collector.terminate.isSet():
try:
payload = rclient.blpop(RedisTransport.COMMAND_KEY, timeout=5)
if not payload:
continue
message = cPickle.loads(payload[1])
if not isinstance(message, RedisMessage):
self.logger.error("Message \"%s\" is not a RedisMessage" %
message)
if not message.method in collector.storage.__class__.__rmi__ or\
not hasattr(collector.storage, message.method):
self.logger.error(
"Unknown method %s called on storage engine %s" %
(message.method, collector.storage.__class__.__name__))
raise TransportError
try:
cls_method = getattr(collector.storage, message.method)
response = cls_method(*message.args, **message.kwargs)
response = cPickle.dumps(response)
except:
self.logger.error("RPC method %s failed: %s" %
(message.method, traceback.format_exc().splitlines()[-1]))
raise TransportError
rclient.publish(message.channel, response)
except redis.RedisError:
self.logger.error("Failed to fetch an interaction: %s" %
(traceback.format_exc().splitlines()[-1]))
except cPickle.UnpicklingError:
self.logger.error("Failed to unpickle payload: %s" %
traceback.format_exc().splitlines()[-1])
except TransportError:
pass
except: # pylint: disable=W0702
self.logger.error("Unhandled exception in command thread: %s" %
traceback.format_exc().splitlines()[-1])
self.logger.info("Command thread shutdown")
def store(self, hostname, metadata, stats):
"""Store the file to disk"""
try:
payload = cPickle.dumps(
dict(hostname=hostname, metadata=metadata, stats=stats))
except: # pylint: disable=W0702
msg = "%s: Failed to build interaction object: %s" % \
(self.__class__.__name__,
traceback.format_exc().splitlines()[-1])
self.logger.error(msg)
raise TransportError(msg)
try:
self._redis.rpush(RedisTransport.STATS_KEY, payload)
except redis.RedisError:
self.logger.error(
"Failed to store interaction for %s: %s" %
(hostname, traceback.format_exc().splitlines()[-1]))
def cachekey(self):
""" A unique key for this source that will be used to generate
:attr:`cachefile` and other cache paths """
return md5(cPickle.dumps([self.version, self.components, self.url,
self.rawurl, self.arches])).hexdigest()
def cachekey(self):
return md5(cPickle.dumps([self.version, self.components, self.url, self.rawurl, self.arches])).hexdigest()
