class RatchetHub: def __init__(self, hub_host, hub_req_port): self.hub_host = hub_host self.hub_req_port = hub_req_port self.requester = Requester(self.hub_host, self.hub_req_port) return def get_ratchet_hub_info(self): records = self.requester.make_request("get_ratchet_hub_info").split("\n") result = self.dict_from_records(records) return result def get_qplan_web_apps(self): app_records = self.requester.make_request("get_qplan_web_apps").split("\n") result = [] for rec in app_records: fields = rec.split("\t") # |fields| looks like [name, host, web_port] item = { 'team': fields[0], 'host': fields[1], 'web_port': fields[2] } result.append(item) return result def get_qplan_instance_config(self, team): instance_record = self.requester.make_request("get_qplan_instance_config %s" % team) instance_fields = instance_record.split("\t") # |instance_fields| will look like [host, web_port, # snapshot_req_port, snapshot_sub_port] # TODO: Use an assert here if len(instance_fields) != 4: sys.stderr.write("Unexpected qplan instance config: %s\n" % instance_record) return host = instance_fields[0] [web_port, req_port, sub_port] = [int(p) for p in instance_fields[1:]] result = { 'team': team, 'host': host, 'web_port': web_port, 'req_port': req_port, 'sub_port': sub_port } return result def get_vantage_config(self): records = self.requester.make_request("get_vantage_config").split("\n") result = self.dict_from_records(records) return result def get_qplan_config(self): records = self.requester.make_request("get_qplan_config").split("\n") result = self.dict_from_records(records) return result def get_vacationator_config(self): records = self.requester.make_request("get_vacationator_config").split("\n") result = self.dict_from_records(records) return result def get_src_root(self): result = self.requester.make_request("get_src_root") return result def get_header_file_map(self, section_name): records = self.requester.make_request("get_header_file_map %s" % section_name).split("\n") result = self.dict_from_records(records) return result def get_team_instances_info(self): records = self.requester.make_request("get_team_instances_info").split("\n") records_dict = self.dict_from_records(records) # CLAIM: values of records_dict look like: ["<host>", "<web port>", "<req port>", "<sub port>"] result = {} for team in records_dict: values = [v.strip() for v in records_dict[team].split(",")] items = {} items['host'] = values[0] items['web_port'] = int(values[1]) items['req_port'] = int(values[2]) items['sub_port'] = int(values[3]) result[team] = items return result def get_update_periods(self): records = self.requester.make_request("get_update_periods").split("\n") result = self.dict_from_records(records) return result def log_debug(self, reporter, message): self.log("DEBG", reporter, message) return def log_info(self, reporter, message): self.log("INFO", reporter, message) return def log_warn(self, reporter, message): self.log("WARN", reporter, message) return def log_error(self, reporter, message): self.log("ERRR", reporter, message) return def log(self, level, reporter, message): self.requester.make_request(utils.format_log_command(level, reporter, message)) return # ========================================================================= # Private functions def dict_from_records(self, records): result = {} for rec in records: fields = rec.split("\t") result[fields[0]] = fields[1] return result
class SnapshotFilter(Subscriber): def __init__(self, listen, snapshot_host, snapshot_req_port, snapshot_sub_port, retry_period, inputs, output): Subscriber.__init__(self, snapshot_host, snapshot_sub_port, inputs) self.inputs = inputs self.output = output self.listen = listen self.error_retry_period = retry_period self.processing_delay = 0.05 # sec self.waiting_to_process = False self.snapshot_requester = Requester(snapshot_host, snapshot_req_port) return # @Override def ping(self): # Start a timer that goes off after "processing_delay" seconds. In the # meantime, we'll ignore pings. When it fires, it should reset its state. if self.waiting_to_process: return self.waiting_to_process = True timer = Timer(self.processing_delay, self.do_ping) timer.start() return def do_ping(self): try: self.process_data_once() except Exception as e: sys.stderr.write("ERROR (%s) processing data: %s\n" % (self.__class__.__name__, str(e))) time.sleep(self.error_retry_period) self.waiting_to_process = False self.ping() return self.waiting_to_process = False return # TODO: Rename this to run def process_data(self): if self.listen == False: self.process_data_once() else: self.run() return # NOTE: This should be overridden def get_processed_data(self): raise "Implement get_processed_data in %s" % self.__class__.__name__ return def process_data_once(self): contents = self.get_processed_data() put_message = snapshot_service.construct_put_message( self.output.split("=====")[1], contents.split("\n"), {"add_newline": True} ) msg = self.snapshot_requester.make_request(put_message) print(msg) return def get_headers(self, additional=[]): inputs = self.inputs + additional result = ["=====GET %s" % h.split("=====")[1] for h in inputs] return result def get_inputs(self, headers=None): if not headers: headers = self.get_headers() result = [] for h in headers: input_data = StringIO.StringIO(self.snapshot_requester.make_request(h)) input_sections = sectionize(input_data) result.append(input_sections["data"]) return result
test_filter.run() return if __name__ == "__main__": reply_port = 10000 publish_port = 10001 # Start the snapshot service snapshot_service_thread = Thread(target=start_snapshot_service, args=(reply_port, publish_port)) snapshot_service_thread.daemon = True snapshot_service_thread.start() # Start the filter we want to exercise test_filter_thread = Thread(target=start_test_filter, args=(reply_port, publish_port)) test_filter_thread.daemon = True test_filter_thread.start() # Update file # NOTE: We should only see one "Simulate processing of data" message if # this works properly. requester = Requester("127.0.0.1", reply_port) for i in range(10): requester.make_request("=====PUT test_file\n\t%d" % i) # Wait until test is done exit_timer = Timer(5, sys.exit, args=(0,)) exit_timer.start()