def __init__(self, config, jira_config):
section = "Snapshot Service"
host = config.get(section, "host")
req_port = int(config.get(section, "request_port"))
# Read config items
self.jira_boards = config.items("JIRA Boards")
# Log in
self.user = jira_config.get("User info", "user")
self.password = jira_config.get("User info", "password")
Requester.__init__(self, host, req_port)
return
def __init__(self, listen, snapshot_host, snapshot_req_port, snapshot_sub_port, retry_period, inputs, output):
Subscriber.__init__(self, snapshot_host, snapshot_sub_port, inputs)
self.inputs = inputs
self.output = output
self.listen = listen
self.error_retry_period = retry_period
self.processing_delay = 0.05 # sec
self.waiting_to_process = False
self.snapshot_requester = Requester(snapshot_host, snapshot_req_port)
return
class SnapshotFilter(Subscriber):
def __init__(self, listen, snapshot_host, snapshot_req_port, snapshot_sub_port, retry_period, inputs, output):
Subscriber.__init__(self, snapshot_host, snapshot_sub_port, inputs)
self.inputs = inputs
self.output = output
self.listen = listen
self.error_retry_period = retry_period
self.processing_delay = 0.05 # sec
self.waiting_to_process = False
self.snapshot_requester = Requester(snapshot_host, snapshot_req_port)
return
# @Override
def ping(self):
# Start a timer that goes off after "processing_delay" seconds. In the
# meantime, we'll ignore pings. When it fires, it should reset its state.
if self.waiting_to_process:
return
self.waiting_to_process = True
timer = Timer(self.processing_delay, self.do_ping)
timer.start()
return
def do_ping(self):
try:
self.process_data_once()
except Exception as e:
sys.stderr.write("ERROR (%s) processing data: %s\n" % (self.__class__.__name__, str(e)))
time.sleep(self.error_retry_period)
self.waiting_to_process = False
self.ping()
return
self.waiting_to_process = False
return
# TODO: Rename this to run
def process_data(self):
if self.listen == False:
self.process_data_once()
else:
self.run()
return
# NOTE: This should be overridden
def get_processed_data(self):
raise "Implement get_processed_data in %s" % self.__class__.__name__
return
def process_data_once(self):
contents = self.get_processed_data()
put_message = snapshot_service.construct_put_message(
self.output.split("=====")[1], contents.split("\n"), {"add_newline": True}
)
msg = self.snapshot_requester.make_request(put_message)
print(msg)
return
def get_headers(self, additional=[]):
inputs = self.inputs + additional
result = ["=====GET %s" % h.split("=====")[1] for h in inputs]
return result
def get_inputs(self, headers=None):
if not headers:
headers = self.get_headers()
result = []
for h in headers:
input_data = StringIO.StringIO(self.snapshot_requester.make_request(h))
input_sections = sectionize(input_data)
result.append(input_sections["data"])
return result
def __init__(self, hub_host, hub_req_port):
self.hub_host = hub_host
self.hub_req_port = hub_req_port
self.requester = Requester(self.hub_host, self.hub_req_port)
return
class RatchetHub:
def __init__(self, hub_host, hub_req_port):
self.hub_host = hub_host
self.hub_req_port = hub_req_port
self.requester = Requester(self.hub_host, self.hub_req_port)
return
def get_ratchet_hub_info(self):
records = self.requester.make_request("get_ratchet_hub_info").split("\n")
result = self.dict_from_records(records)
return result
def get_qplan_web_apps(self):
app_records = self.requester.make_request("get_qplan_web_apps").split("\n")
result = []
for rec in app_records:
fields = rec.split("\t")
# |fields| looks like [name, host, web_port]
item = {
'team': fields[0],
'host': fields[1],
'web_port': fields[2]
}
result.append(item)
return result
def get_qplan_instance_config(self, team):
instance_record = self.requester.make_request("get_qplan_instance_config %s" % team)
instance_fields = instance_record.split("\t")
# |instance_fields| will look like [host, web_port,
# snapshot_req_port, snapshot_sub_port]
# TODO: Use an assert here
if len(instance_fields) != 4:
sys.stderr.write("Unexpected qplan instance config: %s\n" % instance_record)
return
host = instance_fields[0]
[web_port, req_port, sub_port] = [int(p) for p in instance_fields[1:]]
result = {
'team': team,
'host': host,
'web_port': web_port,
'req_port': req_port,
'sub_port': sub_port
}
return result
def get_vantage_config(self):
records = self.requester.make_request("get_vantage_config").split("\n")
result = self.dict_from_records(records)
return result
def get_qplan_config(self):
records = self.requester.make_request("get_qplan_config").split("\n")
result = self.dict_from_records(records)
return result
def get_vacationator_config(self):
records = self.requester.make_request("get_vacationator_config").split("\n")
result = self.dict_from_records(records)
return result
def get_src_root(self):
result = self.requester.make_request("get_src_root")
return result
def get_header_file_map(self, section_name):
records = self.requester.make_request("get_header_file_map %s" % section_name).split("\n")
result = self.dict_from_records(records)
return result
def get_team_instances_info(self):
records = self.requester.make_request("get_team_instances_info").split("\n")
records_dict = self.dict_from_records(records)
# CLAIM: values of records_dict look like: ["<host>", "<web port>", "<req port>", "<sub port>"]
result = {}
for team in records_dict:
values = [v.strip() for v in records_dict[team].split(",")]
items = {}
items['host'] = values[0]
items['web_port'] = int(values[1])
items['req_port'] = int(values[2])
items['sub_port'] = int(values[3])
result[team] = items
return result
def get_update_periods(self):
records = self.requester.make_request("get_update_periods").split("\n")
result = self.dict_from_records(records)
return result
def log_debug(self, reporter, message):
self.log("DEBG", reporter, message)
return
def log_info(self, reporter, message):
self.log("INFO", reporter, message)
return
def log_warn(self, reporter, message):
self.log("WARN", reporter, message)
return
def log_error(self, reporter, message):
self.log("ERRR", reporter, message)
return
def log(self, level, reporter, message):
self.requester.make_request(utils.format_log_command(level, reporter, message))
return
# =========================================================================
# Private functions
def dict_from_records(self, records):
result = {}
for rec in records:
fields = rec.split("\t")
result[fields[0]] = fields[1]
return result
test_filter.run()
return
if __name__ == "__main__":
reply_port = 10000
publish_port = 10001
# Start the snapshot service
snapshot_service_thread = Thread(target=start_snapshot_service, args=(reply_port, publish_port))
snapshot_service_thread.daemon = True
snapshot_service_thread.start()
# Start the filter we want to exercise
test_filter_thread = Thread(target=start_test_filter, args=(reply_port, publish_port))
test_filter_thread.daemon = True
test_filter_thread.start()
# Update file
# NOTE: We should only see one "Simulate processing of data" message if
# this works properly.
requester = Requester("127.0.0.1", reply_port)
for i in range(10):
requester.make_request("=====PUT test_file\n\t%d" % i)
# Wait until test is done
exit_timer = Timer(5, sys.exit, args=(0,))
exit_timer.start()
