class VantageUpdaterService(Subscriber):
def __init__(self, config):
hub_host = config.get('Ratchet Hub', 'host')
hub_port = int(config.get('Ratchet Hub', 'port'))
self.ratchet_hub = RatchetHub(hub_host, hub_port)
vantage_config = self.ratchet_hub.get_vantage_config()
snapshot_host = vantage_config['host']
snapshot_port = int(vantage_config['snapshot_sub_port'])
self.web_host = vantage_config['host']
self.web_port = int(vantage_config['web_port'])
ratchet_hub_info = self.ratchet_hub.get_ratchet_hub_info()
self.name = "%s/VantageUpdaterService" % (ratchet_hub_info['org_name'])
Subscriber.__init__(self, snapshot_host, snapshot_port, [
"=====app vantage"
])
return
# @Override
def ping(self):
self.ratchet_hub.log_info(self.name, "Asking web app to update")
h = httplib.HTTPConnection('%s:%d' % (self.web_host, self.web_port))
h.request("POST","/update", "", {"Content-type": "application/text"})
return
class CondQPlan(SnapshotFilter):
"""
Conditions raw data from a work spreadsheet, a staff spreadsheet, and a
vacation spreadsheet.
"""
def __init__(self, config, listen):
"""
Constructs CondQPlan using config file info and a listen flag. If the
listen flag is False, this runs a conditioning pass once and then exits.
If listen is True, this runs in a loop.
"""
self.team = config.get('Team', 'name')
hub_host = config.get('Ratchet Hub', 'host')
hub_port = int(config.get('Ratchet Hub', 'port'))
self.ratchet_hub = RatchetHub(hub_host, hub_port)
instance_config = self.ratchet_hub.get_qplan_instance_config(self.team)
update_periods = self.ratchet_hub.get_update_periods()
error_retry_period = update_periods['error_retry']
ratchet_hub_info = self.ratchet_hub.get_ratchet_hub_info()
self.name = "%s/%s/CondQPlan" % (ratchet_hub_info['org_name'], self.team)
qplan_config = self.ratchet_hub.get_qplan_config()
self.skill_headings = self.parse_skill_headings(qplan_config['skills'])
SnapshotFilter.__init__(self,
listen,
instance_config['host'],
instance_config['req_port'],
instance_config['sub_port'],
error_retry_period,
# TODO: Rename staff -> assignments and artists -> people
["=====raw work", "=====raw staff", "=====raw vacation", "=====raw artists"],
"=====cond qplan"
)
return
# @Override
def get_processed_data(self):
"""
Overrides function in SnapshotFilter. This is where the actual work is
coordinated from.
"""
self.ratchet_hub.log_info(self.name, "Start conditioning")
try:
[work_lines, staff_lines, vacation_lines, artists_lines] = self.get_inputs()
result = self.condition_qplan_data(work_lines, staff_lines, vacation_lines, artists_lines)
self.ratchet_hub.log_info(self.name, "Done conditioning")
except Exception, e:
self.ratchet_hub.log_error(self.name, "Problem conditioning data: %s" % e)
raise
return result
class CondVacationator(SnapshotFilter):
#---------------------------------------------------------------------------
# Constructs using a config file and a listen flag.
#
def __init__(self, config, listen):
self.team = config.get('Team', 'name')
hub_host = config.get('Ratchet Hub', 'host')
hub_port = int(config.get('Ratchet Hub', 'port'))
self.ratchet_hub = RatchetHub(hub_host, hub_port)
instance_config = self.ratchet_hub.get_qplan_instance_config(self.team)
# NOTE: The qplan instance is the same as the vacationator instance
update_periods = self.ratchet_hub.get_update_periods()
error_retry_period = update_periods['error_retry']
ratchet_hub_info = self.ratchet_hub.get_ratchet_hub_info()
self.name = "%s/%s/CondVacationator" % (ratchet_hub_info['org_name'], self.team)
vacationator_config = self.ratchet_hub.get_vacationator_config()
self.horizon = int(vacationator_config['horizon'])
SnapshotFilter.__init__(self,
listen,
instance_config['host'],
instance_config['req_port'],
instance_config['sub_port'],
error_retry_period,
[
"=====raw staff",
"=====raw vacation",
"=====raw oncall"
],
"=====cond vacationator"
)
return
#===========================================================================
# Internal functions
# @Override
def get_processed_data(self):
self.ratchet_hub.log_info(self.name, "Start conditioning")
try:
[staff_lines, vacation_lines, oncall_lines] = self.get_inputs()
result = self.condition_vacation_data(staff_lines, vacation_lines, oncall_lines)
except Exception, e:
self.ratchet_hub.log_error(self.name, "Problem conditioning data: %s" % e)
return result
class AppQPlan(SnapshotFilter):
def __init__(self, config, listen):
self.team = config.get('Team', 'name')
hub_host = config.get('Ratchet Hub', 'host')
hub_port = int(config.get('Ratchet Hub', 'port'))
self.ratchet_hub = RatchetHub(hub_host, hub_port)
instance_config = self.ratchet_hub.get_qplan_instance_config(self.team)
ratchet_hub_info = self.ratchet_hub.get_ratchet_hub_info()
self.src_root = ratchet_hub_info['src_root']
self.name = "%s/%s/AppQPlan" % (ratchet_hub_info['org_name'], self.team)
update_periods = self.ratchet_hub.get_update_periods()
error_retry_period = update_periods['error_retry']
SnapshotFilter.__init__(self,
listen,
instance_config['host'],
instance_config['req_port'],
instance_config['sub_port'],
error_retry_period,
["=====cond qplan"],
"=====app qplan"
)
return
# @Override
def get_processed_data(self):
self.ratchet_hub.log_info(self.name, "Start appifying")
p = None
result = ""
try:
[cond_data] = self.get_inputs()
qplan_path = "%s/apps/filters/QPlan/dist/build/qplan/qplan" % self.src_root
p = subprocess.Popen(qplan_path, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
result = p.communicate(input=cond_data)[0]
except Exception, e:
self.ratchet_hub.log_error(self.name, "Exception happened running qplan: %s" % e)
raise
if p.returncode != 0:
err_msg = "Something went wrong with the qplan filter (%d)" % p.returncode
self.ratchet_hub.log_error(self.name, err_msg)
raise Exception(err_msg)
self.ratchet_hub.log_info(self.name, "Done appifying")
return result
class AppVacationator(SnapshotFilter):
def __init__(self, config, listen):
self.team = config.get("Team", "name")
hub_host = config.get("Ratchet Hub", "host")
hub_port = int(config.get("Ratchet Hub", "port"))
self.ratchet_hub = RatchetHub(hub_host, hub_port)
# The qplan instance is the same as the vacationator instance
instance_config = self.ratchet_hub.get_qplan_instance_config(self.team)
update_periods = self.ratchet_hub.get_update_periods()
error_retry_period = update_periods["error_retry"]
ratchet_hub_info = self.ratchet_hub.get_ratchet_hub_info()
self.name = "%s/%s/AppVacationator" % (ratchet_hub_info["org_name"], self.team)
SnapshotFilter.__init__(
self,
listen,
instance_config["host"],
instance_config["req_port"],
instance_config["sub_port"],
error_retry_period,
["=====cond vacationator"],
"=====app vacationator",
)
return
# ===========================================================================
# Internal functions
# @Override
def get_processed_data(self):
self.ratchet_hub.log_info(self.name, "Start appifying")
try:
[cond_data] = self.get_inputs()
result = self.process_app_data(cond_data)
except Exception, e:
self.ratchet_hub.log_error(self.name, "Problem conditioning data: %s" % e)
raise
self.ratchet_hub.log_info(self.name, "Done appifying")
return result
class UpdaterService(Subscriber):
def __init__(self, config):
team = config.get("Team", "name")
hub_host = config.get("Ratchet Hub", "host")
hub_port = int(config.get("Ratchet Hub", "port"))
self.ratchet_hub = RatchetHub(hub_host, hub_port)
instance_config = self.ratchet_hub.get_qplan_instance_config(team)
snapshot_host = instance_config["host"]
snapshot_port = instance_config["sub_port"]
self.web_host = instance_config["host"]
self.web_port = instance_config["web_port"]
ratchet_hub_info = self.ratchet_hub.get_ratchet_hub_info()
self.name = "%s/%s/UpdaterService" % (ratchet_hub_info["org_name"], team)
Subscriber.__init__(self, snapshot_host, snapshot_port, ["=====app qplan", "=====app vacationator"])
return
# @Override
def ping(self):
self.ratchet_hub.log_info(self.name, "Asking web app to update")
h = httplib.HTTPConnection("%s:%d" % (self.web_host, self.web_port))
h.request("POST", "/update", "", {"Content-type": "application/text"})
return
import pdb
if __name__ == "__main__":
# Read config info
config = ConfigParser.ConfigParser()
config.readfp(open('config.ini'))
full_config_path = "%s/config.ini" % os.getcwd()
# Ask ratchet hub for port info
hub_host = config.get('Ratchet Hub', 'host')
hub_port = int(config.get('Ratchet Hub', 'port'))
ratchet_hub = RatchetHub(hub_host, hub_port)
try:
ratchet_hub_info = ratchet_hub.get_ratchet_hub_info()
name = "%s/vantage" % (ratchet_hub_info['org_name'])
vantage_config = ratchet_hub.get_vantage_config()
web_port = int(vantage_config['web_port'])
ratchet_src_root = ratchet_hub.get_src_root()
working_dir = "%s/apps/web/vantage" % ratchet_src_root
catserve_path = "%s/apps/misc/catserve/catserve" % ratchet_src_root
print("%s -r=%s -p=%d -c=%s" % (catserve_path, working_dir, web_port, full_config_path))
p = subprocess.Popen("%s -r=%s -p=%d -c=%s" % (catserve_path, working_dir, web_port, full_config_path), shell=True)
p.communicate()
if p.returncode != 0:
err_msg = "Something went wrong catserve (%s)" % p.returncode
raise Exception(err_msg)
class AppVantage(SnapshotFilter):
def __init__(self, config, listen):
hub_host = config.get('Ratchet Hub', 'host')
hub_port = int(config.get('Ratchet Hub', 'port'))
self.ratchet_hub = RatchetHub(hub_host, hub_port)
vantage_config = self.ratchet_hub.get_vantage_config()
ratchet_hub_info = self.ratchet_hub.get_ratchet_hub_info()
self.src_root = ratchet_hub_info['src_root']
self.name = "%s/AppVantage" % (ratchet_hub_info['org_name'])
update_periods = self.ratchet_hub.get_update_periods()
error_retry_period = update_periods['error_retry']
snapshot_host = vantage_config['host']
snapshot_req_port = int(vantage_config['snapshot_req_port'])
snapshot_sub_port = int(vantage_config['snapshot_sub_port'])
SnapshotFilter.__init__(self,
listen,
snapshot_host,
snapshot_req_port,
snapshot_sub_port,
error_retry_period,
["=====raw qplans"],
"=====app vantage"
)
return
#===========================================================================
# Internal functions
# @Override
def get_processed_data(self):
self.ratchet_hub.log_info(self.name, "Start appifying")
[cond_data] = self.get_inputs()
result = self.process_app_data(cond_data)
self.ratchet_hub.log_info(self.name, "Done appifying")
return result
def process_app_data(self, cond_data):
output = StringIO.StringIO()
sections = sectionize(StringIO.StringIO(cond_data))
parsed_data = {}
for team in sections.keys():
parsed_data[team] = json.loads(sections[team])
self.print_dimension_stream("teams", parsed_data.keys(), output)
# Extract skills
skills = self.gather_skills(parsed_data)
self.print_dimension_stream("skills", skills, output)
# Print data
self.print_quarters(parsed_data, skills, output)
self.print_resource_data(parsed_data, skills, output)
self.print_unassigned(parsed_data, skills, output)
result = output.getvalue()
output.close()
return result
def gather_skills(self, parsed_data):
# ASSUMPTION: All quarters in one team require identical skills
skills = []
for team in parsed_data.keys():
quarter_data = parsed_data[team].values()
if (len(quarter_data) > 0):
skills += quarter_data[0]['skills']
result = list(set(skills))
result.sort()
return result
def print_dimension_stream(self, dimension, items, output):
print >>output, "=====%s" % dimension
for it in items:
print >>output, "\t%s" % it
return
def pack_staff_stats(self, stats, skills, team_skills):
# ASSUMPTION: stats and team_skills are the same length
values = []
for sk in skills:
if sk in team_skills:
values.append(stats[team_skills.index(sk)])
else:
values.append(0)
result = ":".join(["%.2f" % v for v in values])
return result
def manpower_to_num_staff(self, stats, num_weeks):
result = {}
for key in stats.keys():
result[key] = []
for val in stats[key]:
result[key].append(float(val)/num_weeks)
return result
def sum_chart_stats(self, stats):
result = {}
for st in stats:
for key in st.keys():
num_values = len(st[key])
if key not in result:
result[key] = [0] * num_values
for i in range(num_values):
result[key][i] += st[key][i]
return result
def normalize_staffing(self, staffing_stats, skills, all_skills):
# ASSUMPTION: skills is a subset of all_skills
result = {}
for k in staffing_stats.keys():
result[k] = [0] * len(all_skills)
for i in range(len(skills)):
# Store staffing stat at appropriate index
index = all_skills.index(skills[i])
result[k][index] = staffing_stats[k][i]
return result
def print_quarters(self, parsed_data, skills, output):
print >>output, "=====quarters"
quarters = []
for team in parsed_data.keys():
# ASSUMPTION: Unknown quarters are named "Unknown Quarter"
quarters += [Quarter(key) for key in parsed_data[team].keys()]
quarters = list(set(quarters))
quarters.sort()
for quarter in quarters:
print >>output, "\t%s" % str(quarter)
return
def print_unassigned(self, parsed_data, skills, output):
print >>output, "=====unassigned"
for team in parsed_data.keys():
# Sort quarters
# ASSUMPTION: Unknown quarters are named "Unknown Quarter"
quarters = [Quarter(key) for key in parsed_data[team].keys()]
quarters.sort()
for quarter in quarters:
quarter_data = parsed_data[team][str(quarter)]
team_skills = quarter_data['skills']
num_weeks = quarter_data['num_weeks']
unassigned = [val/num_weeks for val in quarter_data['unassigned']]
print >>output, "\t%s" % "\t".join([
team,
str(quarter),
self.pack_staff_stats(unassigned, skills, team_skills)
])
return
def print_resource_data(self, parsed_data, skills, output):
# ASSUMPTION: parsed_data is a dictionary of team keys to team data.
# Each team data is a dictionary of quarters to quarter data. Quarter
# data consists of staffing_stats, skills, tracks, triages, and
# num_weeks.
print >>output, "=====resource data"
for team in parsed_data.keys():
for quarter in parsed_data[team].keys():
quarter_data = parsed_data[team][quarter]
for tri_index in range(len(quarter_data['triages'])):
triage = quarter_data['triages'][tri_index]
divisions = quarter_data['tracks']
for div_index in range(len(divisions)):
division = divisions[div_index]
team_skills = quarter_data['skills']
chart_stats = quarter_data['staffing_stats'][tri_index][div_index]
chart_stats = self.manpower_to_num_staff(chart_stats, quarter_data['num_weeks'])
# Print regular records
print >>output, "\t%s" % "\t".join([
team,
division,
quarter,
triage,
self.pack_staff_stats(chart_stats['available'], skills, team_skills),
self.pack_staff_stats(chart_stats['required'], skills, team_skills),
self.pack_staff_stats(chart_stats['net_left'], skills, team_skills)
])
return