def rpc(self, method, *args, **kwargs): """ Send a command to the queue. Timeout after 10 seconds """ pubsub = self._redis.pubsub() channel = "%s%s" % (platform.node(), int(time.time())) pubsub.subscribe(channel) self._redis.rpush( RedisTransport.COMMAND_KEY, cPickle.dumps(RedisMessage(channel, method, args, kwargs))) resp = pubsub.listen() signal.signal(signal.SIGALRM, self.shutdown) signal.alarm(10) resp.next() # clear subscribe message response = resp.next() pubsub.unsubscribe() try: return cPickle.loads(response['data']) except: # pylint: disable=W0702 msg = "%s: Failed to receive response: %s" % \ (self.__class__.__name__, traceback.format_exc().splitlines()[-1]) self.logger.error(msg) return None
def rpc(self, method, *args, **kwargs): """ Send a command to the queue. Timeout after 10 seconds """ pubsub = self._redis.pubsub() channel = "%s%s" % (platform.node(), int(time.time())) pubsub.subscribe(channel) self._redis.rpush(RedisTransport.COMMAND_KEY, cPickle.dumps(RedisMessage(channel, method, args, kwargs))) resp = pubsub.listen() signal.signal(signal.SIGALRM, self.shutdown) signal.alarm(10) resp.next() # clear subscribe message response = resp.next() pubsub.unsubscribe() try: return cPickle.loads(response['data']) except: # pylint: disable=W0702 msg = "%s: Failed to receive response: %s" % \ (self.__class__.__name__, traceback.format_exc().splitlines()[-1]) self.logger.error(msg) return None
def store(self, hostname, metadata, stats): """Store the file to disk""" try: payload = cPickle.dumps( dict(hostname=hostname, metadata=metadata, stats=stats)) except: # pylint: disable=W0702 msg = "%s: Failed to build interaction object: %s" % \ (self.__class__.__name__, traceback.format_exc().splitlines()[-1]) self.logger.error(msg) raise TransportError(msg) fname = "%s-%s" % (hostname, time.time()) save_file = os.path.join(self.work_path, fname) tmp_file = os.path.join(self.work_path, "." + fname) if os.path.exists(save_file): self.logger.error("%s: Oops.. duplicate statistic in directory." % self.__class__.__name__) raise TransportError # using a tmpfile to hopefully avoid the file monitor from grabbing too # soon saved = open(tmp_file, 'wb') try: saved.write(payload) except IOError: self.logger.error( "Failed to store interaction for %s: %s" % (hostname, traceback.format_exc().splitlines()[-1])) os.unlink(tmp_file) saved.close() os.rename(tmp_file, save_file)
def store(self, hostname, metadata, stats): """Store the file to disk""" try: payload = cPickle.dumps(dict(hostname=hostname, metadata=metadata, stats=stats)) except: # pylint: disable=W0702 msg = "%s: Failed to build interaction object: %s" % \ (self.__class__.__name__, traceback.format_exc().splitlines()[-1]) self.logger.error(msg) raise TransportError(msg) fname = "%s-%s" % (hostname, time.time()) save_file = os.path.join(self.work_path, fname) tmp_file = os.path.join(self.work_path, "." + fname) if os.path.exists(save_file): self.logger.error("%s: Oops.. duplicate statistic in directory." % self.__class__.__name__) raise TransportError # using a tmpfile to hopefully avoid the file monitor from grabbing too # soon saved = open(tmp_file, 'wb') try: saved.write(payload) except IOError: self.logger.error("Failed to store interaction for %s: %s" % (hostname, traceback.format_exc().splitlines()[-1])) os.unlink(tmp_file) saved.close() os.rename(tmp_file, save_file)
def cachekey(self): """ A unique key for this source that will be used to generate :attr:`cachefile` and other cache paths """ return md5( cPickle.dumps([ self.version, self.components, self.url, self.rawurl, self.arches ])).hexdigest()
def hash_entry(entry_dict): """ Build a key for this based on its data entry_dict = a dict of all the data identifying this """ dataset = [] for key in sorted(entry_dict.keys()): if key in ('id', 'hash_key') or key.startswith('_'): continue dataset.append((key, entry_dict[key])) return hash(cPickle.dumps(dataset))
def monitor_thread(self, rclient, collector): """Watch the COMMAND_KEY queue for rpc commands""" self.logger.info("Command thread started") while not collector.terminate.isSet(): try: payload = rclient.blpop(RedisTransport.COMMAND_KEY, timeout=5) if not payload: continue message = cPickle.loads(payload[1]) if not isinstance(message, RedisMessage): self.logger.error("Message \"%s\" is not a RedisMessage" % message) if not message.method in collector.storage.__class__.__rmi__ or\ not hasattr(collector.storage, message.method): self.logger.error( "Unknown method %s called on storage engine %s" % (message.method, collector.storage.__class__.__name__)) raise TransportError try: cls_method = getattr(collector.storage, message.method) response = cls_method(*message.args, **message.kwargs) response = cPickle.dumps(response) except: self.logger.error( "RPC method %s failed: %s" % (message.method, traceback.format_exc().splitlines()[-1])) raise TransportError rclient.publish(message.channel, response) except redis.RedisError: self.logger.error("Failed to fetch an interaction: %s" % (traceback.format_exc().splitlines()[-1])) except cPickle.UnpicklingError: self.logger.error("Failed to unpickle payload: %s" % traceback.format_exc().splitlines()[-1]) except TransportError: pass except: # pylint: disable=W0702 self.logger.error("Unhandled exception in command thread: %s" % traceback.format_exc().splitlines()[-1]) self.logger.info("Command thread shutdown")
def store(self, hostname, metadata, stats): """Store the file to disk""" try: payload = cPickle.dumps(dict(hostname=hostname, metadata=metadata, stats=stats)) except: # pylint: disable=W0702 msg = "%s: Failed to build interaction object: %s" % \ (self.__class__.__name__, traceback.format_exc().splitlines()[-1]) self.logger.error(msg) raise TransportError(msg) try: self._redis.rpush(RedisTransport.STATS_KEY, payload) except redis.RedisError: self.logger.error("Failed to store interaction for %s: %s" % (hostname, traceback.format_exc().splitlines()[-1]))
def monitor_thread(self, rclient, collector): """Watch the COMMAND_KEY queue for rpc commands""" self.logger.info("Command thread started") while not collector.terminate.isSet(): try: payload = rclient.blpop(RedisTransport.COMMAND_KEY, timeout=5) if not payload: continue message = cPickle.loads(payload[1]) if not isinstance(message, RedisMessage): self.logger.error("Message \"%s\" is not a RedisMessage" % message) if not message.method in collector.storage.__class__.__rmi__ or\ not hasattr(collector.storage, message.method): self.logger.error( "Unknown method %s called on storage engine %s" % (message.method, collector.storage.__class__.__name__)) raise TransportError try: cls_method = getattr(collector.storage, message.method) response = cls_method(*message.args, **message.kwargs) response = cPickle.dumps(response) except: self.logger.error("RPC method %s failed: %s" % (message.method, traceback.format_exc().splitlines()[-1])) raise TransportError rclient.publish(message.channel, response) except redis.RedisError: self.logger.error("Failed to fetch an interaction: %s" % (traceback.format_exc().splitlines()[-1])) except cPickle.UnpicklingError: self.logger.error("Failed to unpickle payload: %s" % traceback.format_exc().splitlines()[-1]) except TransportError: pass except: # pylint: disable=W0702 self.logger.error("Unhandled exception in command thread: %s" % traceback.format_exc().splitlines()[-1]) self.logger.info("Command thread shutdown")
def store(self, hostname, metadata, stats): """Store the file to disk""" try: payload = cPickle.dumps( dict(hostname=hostname, metadata=metadata, stats=stats)) except: # pylint: disable=W0702 msg = "%s: Failed to build interaction object: %s" % \ (self.__class__.__name__, traceback.format_exc().splitlines()[-1]) self.logger.error(msg) raise TransportError(msg) try: self._redis.rpush(RedisTransport.STATS_KEY, payload) except redis.RedisError: self.logger.error( "Failed to store interaction for %s: %s" % (hostname, traceback.format_exc().splitlines()[-1]))
def cachekey(self): """ A unique key for this source that will be used to generate :attr:`cachefile` and other cache paths """ return md5(cPickle.dumps([self.version, self.components, self.url, self.rawurl, self.arches])).hexdigest()
def cachekey(self): return md5(cPickle.dumps([self.version, self.components, self.url, self.rawurl, self.arches])).hexdigest()