def __init__(self, config):
self.source = None
self.work_items = None
self.states = []
self.config = config
if config['source']['type'] == 'fogbugz':
self.source = FogbugzWrapper(self.config)
elif config['source']['type'] == 'jira':
m = re.match("^ENV\(([^\']+)\)", self.config['source']['authentication']['password'])
if m is not None:
self.config['source']['authentication']['password'] = os.environ.get(m.group(1), 'undefined')
self.source = JiraWrapper(self.config)
if 'throughput_dow' in config:
self.throughput_dow = config['throughput_dow']
else:
self.throughput_dow = 4
try:
self.types = config['types']
self.counts_towards_throughput = config['counts_towards_throughput']
except KeyError as e:
raise exceptions.MissingConfigItem(e.message, "Missing Config Item:{0}".format(e.message))
# Optional
try:
self.states = config['states']
self.states.append(None)
except KeyError:
pass
def __init__(self, config):
self.source = None
self.work_items = None
self.states = []
self.config = config
if config['source']['type'] == 'fogbugz':
self.source = FogbugzWrapper(self.config)
elif config['source']['type'] == 'jira':
m = re.match("^ENV\(([^\']+)\)",
self.config['source']['authentication']['password'])
if m is not None:
self.config['source']['authentication'][
'password'] = os.environ.get(m.group(1), 'undefined')
self.source = JiraWrapper(self.config)
if 'throughput_dow' in config:
self.throughput_dow = config['throughput_dow']
else:
self.throughput_dow = 4
try:
self.types = config['types']
self.counts_towards_throughput = config[
'counts_towards_throughput']
except KeyError as e:
raise exceptions.MissingConfigItem(
e.message, "Missing Config Item:{0}".format(e.message))
# Optional
try:
self.states = config['states']
self.states.append(None)
except KeyError:
pass
class Metrics(object):
def __init__(self, config):
self.source = None
self.work_items = None
self.states = []
self.config = config
if config['source']['type'] == 'fogbugz':
self.source = FogbugzWrapper(self.config)
elif config['source']['type'] == 'jira':
m = re.match("^ENV\(([^\']+)\)", self.config['source']['authentication']['password'])
if m is not None:
self.config['source']['authentication']['password'] = os.environ.get(m.group(1), 'undefined')
self.source = JiraWrapper(self.config)
if 'throughput_dow' in config:
self.throughput_dow = config['throughput_dow']
else:
self.throughput_dow = 4
try:
self.types = config['types']
self.counts_towards_throughput = config['counts_towards_throughput']
except KeyError as e:
raise exceptions.MissingConfigItem(e.message, "Missing Config Item:{0}".format(e.message))
# Optional
try:
self.states = config['states']
self.states.append(None)
except KeyError:
pass
def work_item(self, id):
"""
Get an individual work item.
This is a case in FogBugz or an Issue in Jira.
"""
if self.work_items is None:
self.work_items = self.source.work_items()
matches = [work_item for work_item in self.work_items if work_item.id == id]
return matches[0]
def details(self, fields=None):
if self.work_items is None:
self.work_items = self.source.work_items()
details = []
for work_item in self.work_items:
details.append(work_item.detail())
df = pd.DataFrame(details)
if fields is None:
return df
else:
return df.filter(fields)
def history(self, from_date=None, until_date=None, types=None):
if self.work_items is None:
self.work_items = self.source.work_items()
history = {}
for work_item in self.work_items:
if types is None:
# HACK HACK HACK
# Also need some consistency around thing_date and date_thing
if isinstance(self.source, JiraWrapper):
history[work_item.id] = work_item.history
else:
history[work_item.id] = history_from_state_transitions(work_item.date_created.date(), work_item.history, until_date)
else:
for type_grouping in types:
if work_item.type in self.types[type_grouping]:
if isinstance(self.source, JiraWrapper):
history[work_item.id] = work_item.history
else:
history[work_item.id] = history_from_state_transitions(work_item.date_created.date(), work_item.history, until_date)
if history is not None:
df = pd.DataFrame(history)
return df
return None
def throughput(self,
from_date,
to_date,
cumulative=True,
category=None,
types=None):
"""
Return throughput for all work_items in a state where they are considered
to count towards throughput.
Might want to add some additional conditions other than state but state
allows us the most options as to where to place the 'finishing line'
"""
work_item_history = self.history(from_date, to_date)
work_item_rows = []
for work_item_key in work_item_history:
work_item = self.work_item(work_item_key)
if category is not None:
if category != work_item.category:
continue
swimlane = work_item.category
# Are we grouping by work type?
if types is not None:
for type_grouping in types:
if work_item.type in self.types[type_grouping]:
swimlane = swimlane + '-' + type_grouping
else:
continue
# print "Not counting " + f.work_itemtype.name
if swimlane == work_item.category:
continue
for day, state in work_item_history[work_item_key].iteritems():
if day.weekday() == self.throughput_dow:
if state in self.counts_towards_throughput:
work_item_row = {'swimlane': swimlane,
'id': work_item_key,
'week': day,
'count': 1}
work_item_rows.append(work_item_row)
df = pd.DataFrame(work_item_rows)
if len(df.index) > 0:
table = pd.tools.pivot.pivot_table(df, rows=['week'], cols=['swimlane'], values='count', aggfunc=np.count_nonzero)
if cumulative:
return table
else:
def de_cumulative(x):
for i in range(x.size-1, 0, -1):
x[i] = x[i] - x[i-1]
return x
table = table.apply(de_cumulative)
return table
else:
return None
def cfd(self, from_date=None, until_date=None, types=None):
"""
Cumulative Flow Diagram
"""
cfd = self.history(from_date, until_date, types)
days = {}
for day in cfd.index:
tickets = []
for ticket in cfd.ix[day]:
tickets.append(ticket)
def state_order(state):
try:
return self.states.index(state)
except ValueError:
if type(state) == float:
if math.isnan(state):
return -1
if type(state) == np.float64:
if math.isnan(state):
return -1
raise exceptions.MissingState(state, "Missing state:{0}".format(state))
days[day] = sorted(tickets, key=state_order)
return pd.DataFrame(days)
def cycle_time_histogram(self,
cycle,
types=None,
buckets=None):
"""
Time taken for work to complete one or more 'cycles' - i.e. transitions from a start state to an end state
"""
if self.work_items is None:
self.work_items = self.source.work_items()
cycle_time_data = {}
for work_item in self.work_items:
if types is not None:
include = False
for type_grouping in types:
if work_item.type in self.types[type_grouping]:
include = True
key = "{0}-{1}".format(type_grouping, cycle)
break
if not include:
continue
else:
key = cycle
try:
if work_item.cycles[cycle] is not None:
if key not in cycle_time_data:
cycle_time_data[key] = [work_item.cycles[cycle]]
else:
cycle_time_data[key].append(work_item.cycles[cycle])
except KeyError:
continue
histogram = None
for cycle in cycle_time_data:
if buckets is not None:
try:
li = buckets.index('max')
buckets[li] = max(cycle_time_data[cycle])
except ValueError:
pass
labels = bucket_labels(buckets)
count, division = np.histogram(cycle_time_data[cycle], bins=buckets)
else:
count, division = np.histogram(cycle_time_data[cycle])
labels = bucket_labels(division)
cycle_histogram = pd.DataFrame(count, index=labels, columns=[cycle])
cycle_histogram.index.name = 'bucket'
if histogram is None:
histogram = cycle_histogram.copy(deep=True)
else:
old_histogram = histogram.copy(deep=True)
histogram = old_histogram.join(cycle_histogram, how='outer')
return histogram
def demand(self,
from_date,
to_date,
types=None):
"""
Return the number of issues created each week - i.e. the demand on the system
"""
if self.work_items is None:
self.work_items = self.source.work_items()
details = []
for work_item in self.work_items:
detail = work_item.detail()
detail['count'] = 1 # Need to figure out where to put this
# resolution_date_str = f.resolutiondate
# if resolution_date_str is not None:
# resolution_date = datetime.strptime(resolution_date_str[:10],
# '%Y-%m-%d')
# week = week_start_date(resolution_date.isocalendar()[0],
# resolution_date.isocalendar()[1]).strftime('%Y-%m-%d')
# else:
# week = None
detail['week_created'] = week_start_date(detail['date_created'].isocalendar()[0],
detail['date_created'].isocalendar()[1]).strftime('%Y-%m-%d')
include = True
swimlane = work_item.category
if types is not None and self.types is not None:
include = False
for type_grouping in types:
if work_item.type in self.types[type_grouping]:
swimlane = swimlane + '-' + type_grouping
include = True
detail['swimlane'] = swimlane
if include:
details.append(detail)
df = pd.DataFrame(details)
table = pd.tools.pivot.pivot_table(df, rows=['week_created'], cols=['swimlane'], values='count', aggfunc=np.count_nonzero)
reindexed = table.reindex(index=fill_date_index_blanks(table.index), fill_value=np.int64(0))
reindexed.index.name = "week"
return reindexed
def arrival_rate(self,
from_date,
to_date):
"""
So that we can get an idea of the flow of work that has not been completed and so does not have a resolution date
and so does not count towards throughput, what is the rate at which that work arrived at states further up the
value chain?
"""
if self.work_items is None:
self.work_items = self.source.work_items()
arrivals_count = {}
for work_item in self.work_items:
try:
arrivals_count = arrivals(work_item.history, arrivals_count)
except AttributeError as e:
print e
df = pd.DataFrame.from_dict(arrivals_count, orient='index')
df.index = pd.to_datetime(df.index)
wf = df.resample('W-MON', how='sum')
return wf
def save_work_items(self, filename=None):
if filename is None:
if 'name' in self.config['name']:
filename = self.config['name'] + '.json'
else:
filename = 'local.json'
if self.work_items is None:
self.work_items = self.source.work_items()
output = []
for item in self.work_items:
# This is so wrong. We are decoding then encoding then decoding again...
output.append(json.loads(item.to_JSON()))
with open(filename, 'w') as outfile:
json.dump(output, outfile, indent=4, sort_keys=True)
class Metrics(object):
def __init__(self, config):
self.source = None
self.work_items = None
self.states = []
self.config = config
if config['source']['type'] == 'fogbugz':
self.source = FogbugzWrapper(self.config)
elif config['source']['type'] == 'jira':
m = re.match("^ENV\(([^\']+)\)",
self.config['source']['authentication']['password'])
if m is not None:
self.config['source']['authentication'][
'password'] = os.environ.get(m.group(1), 'undefined')
self.source = JiraWrapper(self.config)
if 'throughput_dow' in config:
self.throughput_dow = config['throughput_dow']
else:
self.throughput_dow = 4
try:
self.types = config['types']
self.counts_towards_throughput = config[
'counts_towards_throughput']
except KeyError as e:
raise exceptions.MissingConfigItem(
e.message, "Missing Config Item:{0}".format(e.message))
# Optional
try:
self.states = config['states']
self.states.append(None)
except KeyError:
pass
def work_item(self, id):
"""
Get an individual work item.
This is a case in FogBugz or an Issue in Jira.
"""
if self.work_items is None:
self.work_items = self.source.work_items()
matches = [
work_item for work_item in self.work_items if work_item.id == id
]
return matches[0]
def details(self, fields=None):
if self.work_items is None:
self.work_items = self.source.work_items()
details = []
for work_item in self.work_items:
details.append(work_item.detail())
df = pd.DataFrame(details)
if fields is None:
return df
else:
return df.filter(fields)
def history(self, from_date=None, until_date=None, types=None):
if self.work_items is None:
self.work_items = self.source.work_items()
history = {}
for work_item in self.work_items:
if types is None:
# HACK HACK HACK
# Also need some consistency around thing_date and date_thing
if isinstance(self.source, JiraWrapper):
history[work_item.id] = work_item.history
else:
history[work_item.id] = history_from_state_transitions(
work_item.date_created.date(), work_item.history,
until_date)
else:
for type_grouping in types:
if work_item.type in self.types[type_grouping]:
if isinstance(self.source, JiraWrapper):
history[work_item.id] = work_item.history
else:
history[
work_item.id] = history_from_state_transitions(
work_item.date_created.date(),
work_item.history, until_date)
if history is not None:
df = pd.DataFrame(history)
return df
return None
def throughput(self,
from_date,
to_date,
cumulative=True,
category=None,
types=None):
"""
Return throughput for all work_items in a state where they are considered
to count towards throughput.
Might want to add some additional conditions other than state but state
allows us the most options as to where to place the 'finishing line'
"""
work_item_history = self.history(from_date, to_date)
work_item_rows = []
for work_item_key in work_item_history:
work_item = self.work_item(work_item_key)
if category is not None:
if category != work_item.category:
continue
swimlane = work_item.category
# Are we grouping by work type?
if types is not None:
for type_grouping in types:
if work_item.type in self.types[type_grouping]:
swimlane = swimlane + '-' + type_grouping
else:
continue
# print "Not counting " + f.work_itemtype.name
if swimlane == work_item.category:
continue
for day, state in work_item_history[work_item_key].iteritems():
if day.weekday() == self.throughput_dow:
if state in self.counts_towards_throughput:
work_item_row = {
'swimlane': swimlane,
'id': work_item_key,
'week': day,
'count': 1
}
work_item_rows.append(work_item_row)
df = pd.DataFrame(work_item_rows)
if len(df.index) > 0:
table = pd.tools.pivot.pivot_table(df,
rows=['week'],
cols=['swimlane'],
values='count',
aggfunc=np.count_nonzero)
if cumulative:
return table
else:
def de_cumulative(x):
for i in range(x.size - 1, 0, -1):
x[i] = x[i] - x[i - 1]
return x
table = table.apply(de_cumulative)
return table
else:
return None
def cfd(self, from_date=None, until_date=None, types=None):
"""
Cumulative Flow Diagram
"""
cfd = self.history(from_date, until_date, types)
days = {}
for day in cfd.index:
tickets = []
for ticket in cfd.ix[day]:
tickets.append(ticket)
def state_order(state):
try:
return self.states.index(state)
except ValueError:
if type(state) == float:
if math.isnan(state):
return -1
if type(state) == np.float64:
if math.isnan(state):
return -1
raise exceptions.MissingState(
state, "Missing state:{0}".format(state))
days[day] = sorted(tickets, key=state_order)
return pd.DataFrame(days)
def cycle_time_histogram(self, cycle, types=None, buckets=None):
"""
Time taken for work to complete one or more 'cycles' - i.e. transitions from a start state to an end state
"""
if self.work_items is None:
self.work_items = self.source.work_items()
cycle_time_data = {}
for work_item in self.work_items:
if types is not None:
include = False
for type_grouping in types:
if work_item.type in self.types[type_grouping]:
include = True
key = "{0}-{1}".format(type_grouping, cycle)
break
if not include:
continue
else:
key = cycle
try:
if work_item.cycles[cycle] is not None:
if key not in cycle_time_data:
cycle_time_data[key] = [work_item.cycles[cycle]]
else:
cycle_time_data[key].append(work_item.cycles[cycle])
except KeyError:
continue
histogram = None
for cycle in cycle_time_data:
if buckets is not None:
try:
li = buckets.index('max')
buckets[li] = max(cycle_time_data[cycle])
except ValueError:
pass
labels = bucket_labels(buckets)
count, division = np.histogram(cycle_time_data[cycle],
bins=buckets)
else:
count, division = np.histogram(cycle_time_data[cycle])
labels = bucket_labels(division)
cycle_histogram = pd.DataFrame(count,
index=labels,
columns=[cycle])
cycle_histogram.index.name = 'bucket'
if histogram is None:
histogram = cycle_histogram.copy(deep=True)
else:
old_histogram = histogram.copy(deep=True)
histogram = old_histogram.join(cycle_histogram, how='outer')
return histogram
def demand(self, from_date, to_date, types=None):
"""
Return the number of issues created each week - i.e. the demand on the system
"""
if self.work_items is None:
self.work_items = self.source.work_items()
details = []
for work_item in self.work_items:
detail = work_item.detail()
detail['count'] = 1 # Need to figure out where to put this
# resolution_date_str = f.resolutiondate
# if resolution_date_str is not None:
# resolution_date = datetime.strptime(resolution_date_str[:10],
# '%Y-%m-%d')
# week = week_start_date(resolution_date.isocalendar()[0],
# resolution_date.isocalendar()[1]).strftime('%Y-%m-%d')
# else:
# week = None
detail['week_created'] = week_start_date(
detail['date_created'].isocalendar()[0],
detail['date_created'].isocalendar()[1]).strftime('%Y-%m-%d')
include = True
swimlane = work_item.category
if types is not None and self.types is not None:
include = False
for type_grouping in types:
if work_item.type in self.types[type_grouping]:
swimlane = swimlane + '-' + type_grouping
include = True
detail['swimlane'] = swimlane
if include:
details.append(detail)
df = pd.DataFrame(details)
table = pd.tools.pivot.pivot_table(df,
rows=['week_created'],
cols=['swimlane'],
values='count',
aggfunc=np.count_nonzero)
reindexed = table.reindex(index=fill_date_index_blanks(table.index),
fill_value=np.int64(0))
reindexed.index.name = "week"
return reindexed
def arrival_rate(self, from_date, to_date):
"""
So that we can get an idea of the flow of work that has not been completed and so does not have a resolution date
and so does not count towards throughput, what is the rate at which that work arrived at states further up the
value chain?
"""
if self.work_items is None:
self.work_items = self.source.work_items()
arrivals_count = {}
for work_item in self.work_items:
try:
arrivals_count = arrivals(work_item.history, arrivals_count)
except AttributeError as e:
print e
df = pd.DataFrame.from_dict(arrivals_count, orient='index')
df.index = pd.to_datetime(df.index)
wf = df.resample('W-MON', how='sum')
return wf
def save_work_items(self, filename=None):
if filename is None:
if 'name' in self.config['name']:
filename = self.config['name'] + '.json'
else:
filename = 'local.json'
if self.work_items is None:
self.work_items = self.source.work_items()
output = []
for item in self.work_items:
# This is so wrong. We are decoding then encoding then decoding again...
output.append(json.loads(item.to_JSON()))
with open(filename, 'w') as outfile:
json.dump(output, outfile, indent=4, sort_keys=True)
