def redmine_issues(redmine_id, stat_id, odbor_name):
""" redmine: pocet otevrenych podani odboru nebo jine slozky.
vytvori dva druhy metriky: prvni pro vsechna podani, druha pouze pro podani od 1.1.2019 (maji 'NEW' ve jmene metriky)
redmine_id identifikace odboru v Redmine (soucast url)
stat_id identifikace odboru ve statistikach, soucast ID statistiky, napr pro 'AO'
to bude REDMINE_AO_OPENTICKETS_COUNT
odbor_name jmeno odboru v dlouhem popisu statitstiky, napriklad 'Kancelar'
"""
base_url = 'https://redmine.pirati.cz/projects/%s/issues.json?tracker_id=12' % redmine_id
resp = func.get_json(base_url)
if resp:
original_count = resp['total_count']
all_issues = []
if original_count:
total_count, offset, total_sum = 0, 0, 0
while offset < original_count:
resp = func.get_json(base_url +
'&limit=100&offset=%s' % offset)
offset += 100
all_issues.extend(resp['issues'])
# ze ziskanych dat staci jen datumy
all_issues = lmap(
lambda x: datetime.datetime.strptime(x[
'start_date'][:10], "%Y-%m-%d").date(), all_issues)
new_issues = list(
filter(lambda x: x >= datetime.date(2019, 1, 1), all_issues))
sum_all_issues_ages = sum(
lmap(lambda x: (datetime.date.today() - x).days, all_issues))
sum_new_issues_ages = sum(
lmap(lambda x: (datetime.date.today() - x).days, new_issues))
func.Stat(
dbx, "REDMINE_%s_OPENTICKETS_COUNT" % stat_id, len(all_issues), 0,
'Pocet otevrenych podani slozky %s, REST dotazem do Redmine' %
odbor_name)
func.Stat(
dbx, "REDMINE_%s_NEWOPENTICKETS_COUNT" % stat_id, len(new_issues),
0,
'Prumerne stari otevrenych podani (po 1.1.2019) slozky %s, REST dotazem do Redmine'
% odbor_name)
if len(all_issues):
func.Stat(
dbx, "REDMINE_%s_OPENTICKETS_AGE" % stat_id,
round(sum_all_issues_ages / len(all_issues), 2), 0,
'Prumerne stari otevrenych podani slozky %s, REST dotazem do Redmine'
% odbor_name)
if len(new_issues):
func.Stat(
dbx, "REDMINE_%s_NEWOPENTICKETS_AGE" % stat_id,
round(sum_new_issues_ages / len(new_issues), 2), 0,
'Prumerne stari otevrenych podani (po 1.1.2019) slozky %s, REST dotazem do Redmine'
% odbor_name)
def _counts(url):
resp = func.get_json(url)
if resp and len(resp):
resp = lmap(
lambda x: (datetime.date.today() - datetime.datetime.strptime(
x['updatedStamp'], "%d.%m.%Y, %H:%M").date()).days, resp)
return (sum(resp), len(resp))
def get_oldest_timeline(rowlist_in):
""" Vraci klic te casove rady, ktera ma nejstarsi datum """
oldest_date, oldest_id, lastkey = datetime.datetime.now().date(
), None, None
rowlist = rowlist_in
for id in rowlist:
lastkey = id
datelist = func.lmap(lambda x: x[0], rowlist[id])
if datelist:
oldest_in_row = min(datelist)
if oldest_in_row < oldest_date:
oldest_date = oldest_in_row
oldest_id = id
return oldest_id if oldest_id else lastkey
def fill_range(self, min, max, value=None):
""" dopln do souboru dat chybejici hodnoty z rozsahu min-max, vcetne.
vysledek setrid podle data
"""
just_dates = func.lmap(lambda x: x[0], self.values)
startdate = min
while startdate <= max:
startdate += datetime.timedelta(days=1)
if not startdate in just_dates:
self.values.append([startdate, value])
newvalues = [list(x) for x in self.values]
newvalues.sort()
self.values = newvalues
def main():
# testovaci nahodna hodnota
func.Stat(dbx, "RANDOM", random.randint(1, 1000), 0,
'Nahodna hodnota bez vyznamu, jako test funkcnosti statistik')
# Pocet lidi *se smlouvami* placenych piraty - jako pocet radku z payroll.csv, obsahujich 2 ciselne udaje oddelene carkou
lines = func.getLines(
'https://raw.githubusercontent.com/pirati-byro/transparence/master/payroll.csv',
arg('v'))
if lines:
func.Stat(
dbx, "PAYROLL_COUNT", len(func.grep(r'[0-9]+,[0-9]+', lines)), 0,
'Pocet lidi placenych piraty, zrejme zastarale: jako pocet radku v souboru https://raw.githubusercontent.com/pirati-byro/transparence/master/payroll.csv'
)
# piroplaceni: pocet a prumerne stari (od data posledni upravy) zadosti ve stavu "Schvalena hospodarem" (state=3)
resp = func.get_json(
'https://piroplaceni.pirati.cz/rest/realItem/?format=json&state=3')
if resp:
func.Stat(
dbx, "PP_APPROVED_COUNT", len(resp), 0,
'Pocet zadosti o proplaceni ve stavu Schvalena hospodarem, REST dotazem do piroplaceni'
)
if len(resp):
resp = lmap(
lambda x: (datetime.date.today() - datetime.datetime.strptime(
x['updatedStamp'], "%d.%m.%Y, %H:%M").date()).days, resp)
func.Stat(
dbx, "PP_APPROVED_AGE", round(sum(resp) / len(resp), 2), 0,
'Prumerne stari zadosti o proplaceni ve stavu Schvalena hospodarem, REST dotazem do piroplaceni'
)
# piroplaceni: pocet a prumerne stari (od data posledni upravy) zadosti ve stavu "Ke schvaleni hospodarem" (state=2)
resp = func.get_json(
'https://piroplaceni.pirati.cz/rest/realItem/?format=json&state=2')
if resp:
func.Stat(
dbx, "PP_TOAPPROVE_COUNT", len(resp), 0,
'Pocet zadosti o proplaceni ve stavu Ke schvaleni hospodarem, REST dotazem do piroplaceni'
)
if len(resp):
resp = lmap(
lambda x: (datetime.date.today() - datetime.datetime.strptime(
x['updatedStamp'], "%d.%m.%Y, %H:%M").date()).days, resp)
func.Stat(
dbx, "PP_TOAPPROVE_AGE", round(sum(resp) / len(resp), 2), 0,
'Prumerne stari zadosti o proplaceni ve stavu Ke schvaleni hospodarem, REST dotazem do piroplaceni'
)
# piroplaceni: prumerne stari (od data posledni upravy) zadosti ve stavu "Ke schvaleni hospodarem" nebo "Rozpracovana"
def _counts(url):
resp = func.get_json(url)
if resp and len(resp):
resp = lmap(
lambda x: (datetime.date.today() - datetime.datetime.strptime(
x['updatedStamp'], "%d.%m.%Y, %H:%M").date()).days, resp)
return (sum(resp), len(resp))
sums = list(
_counts(
'https://piroplaceni.pirati.cz/rest/realItem/?format=json&state=1'
)) # rozprac
x = _counts(
'https://piroplaceni.pirati.cz/rest/realItem/?format=json&state=2'
) # ke schvaleni hosp
sums[0] += x[0]
sums[1] += x[1]
func.Stat(
dbx, "PP_UNAPPROVED_AGE", round(sums[0] / sums[1], 2), 0,
'Prumerne stari zadosti o proplaceni ve stavu Ke schvaleni hospodarem nebo Rozpracovana, pocitano od data posledni upravy. REST dotazem do piroplaceni'
)
# pocet priznivcu, z fora
stat_from_regex('PI_REGP_COUNT',
'https://forum.pirati.cz/memberlist.php?mode=group&g=74',
r'<div class=\"pagination\">\s*(.*?)\s*už',
"Pocet registrovanych priznivcu")
# redmine: pocty a prumerna stari otevrenych podani pro jednotlive organizacni slozky
redminers = json.loads(
func.getUrlContent(
'https://raw.githubusercontent.com/Jarmil1/pistat-conf/yt-rm-to-conf/redminers.json'
))
for acc in redminers:
redmine_issues(acc['redmine_id'], acc['stat_id'],
acc['department_name'])
# Zustatky na vsech transparentnich FIO uctech uvedenych na wiki FO
content = func.getUrlContent("https://wiki.pirati.cz/fo/seznam_uctu")
if content:
fioAccounts = list(
set(re.findall(r'[0-9]{6,15}[ \t]*/[ \t]*2010', content)))
total = 0
for account in fioAccounts:
account = account.split("/")[0].strip()
total += statFioBalance(account)
func.Stat(
dbx, "BALANCE_FIO_TOTAL", total, 0,
'Soucet zustatku na vsech FIO transparentnich uctech, sledovanych k danemu dni'
)
# Pocty clenu v jednotlivych KS a celkem ve strane (prosty soucet dilcich)
total = 0
for id in PIRATI_KS:
total += statNrOfMembers(id, PIRATI_KS[id])
func.Stat(dbx, "PI_MEMBERS_TOTAL", total, 0,
'Pocet clenu CPS celkem, jako soucet poctu clenu v KS')
# piratske forum
stat_forum()
youtubers = json.loads(
func.getUrlContent(
'https://raw.githubusercontent.com/Jarmil1/pistat-conf/yt-rm-to-conf/youtubers.json'
))
# pocty odberatelu vybranych Youtube kanalu
for id in youtubers:
# odberatelu
content = func.getUrlContent(youtubers[id][0])
m = re.findall(r'([\xa00-9]+)[ ]+odb.{1,1}ratel', content)
value = int(re.sub(r'\xa0', '', m[0])) if m else 0
func.Stat(
dbx, id + '_SUBSCRIBERS', value, 0,
"Odberatelu youtube kanalu, scrappingem verejne Youtube stranky")
# shlednuti
content = func.getUrlContent(youtubers[id][1])
m = re.findall(r'<b>([\xa00-9]+)</b> zhl.{1,1}dnut', content)
value = int(re.sub(r'\xa0', '', m[0])) if m else 0
func.Stat(
dbx, id + '_VIEWS', value, 0,
"Pocet shlednuti youtube kanalu, scrappingem verejne Youtube stranky"
)
# pocty followeru a tweetu ve vybranych twitter kanalech, konfiguraci nacti z druheho gitu
twitter_accounts = func.filter_config(
func.getLines(
'https://raw.githubusercontent.com/Jarmil1/pistat-conf/master/twitters'
))[:200]
for id in twitter_accounts:
content = func.getUrlContent("https://twitter.com/%s" % id)
if content:
m = re.findall(r'data-count=([0-9]*)', content)
if m:
func.Stat(
dbx, "TWITTER_%s_FOLLOWERS" % id.upper(), int(m[2]), 0,
"Followers uzivatele, scrappingem verejneho profilu na Twitteru (treti nalezene cislo)"
) # hack, predpoklada toto cislo jako treti nalezene
func.Stat(
dbx, "TWITTER_%s_TWEETS" % id.upper(), int(m[0]), 0,
"Tweets uzivatele, scrappingem verejneho profilu na Twitteru (prvni nalezene cislo)"
) # hack dtto
if len(m) > 3:
func.Stat(
dbx, "TWITTER_%s_LIKES" % id.upper(), int(m[3]), 0,
"Likes uzivatele, scrappingem verejneho profilu na Twitteru (ctvrte nalezene cislo)"
) # hack dtto
else:
print(id, "skipped: no likes found")
else:
print(id, "skipped: this account does not exist?")
def stat_max_date(stat):
''' obdobne vrat nejvetsi datum '''
return max(func.lmap(lambda x: x[0], stat)) if stat else None
def stat_min_date(stat):
''' vrat nejmensi datum v datove rade statistiky stat = [ (datum, hodnota), (datum, hodnota) ...] '''
return min(func.lmap(lambda x: x[0], stat)) if stat else None
def make_pages(dbx, dirname):
""" Nageneruj stranky a obrazky do adresare dirname """
def add_stat_to_group(groups, groupname, statid):
try:
groups[groupname].append(statid)
except KeyError:
groups[groupname] = [statid]
def stat_min_date(stat):
''' vrat nejmensi datum v datove rade statistiky stat = [ (datum, hodnota), (datum, hodnota) ...] '''
return min(func.lmap(lambda x: x[0], stat)) if stat else None
def stat_max_date(stat):
''' obdobne vrat nejvetsi datum '''
return max(func.lmap(lambda x: x[0], stat)) if stat else None
# priprava adresare
try:
shutil.rmtree(dirname)
except:
pass
try:
func.makedir(dirname)
except:
pass
try:
func.makedir(dirname + "/img")
except:
pass
s = func.clsMyStat(dbx, '')
stats = s.getAllStats()
i, statnames, statnames_index, groups = 0, {}, {}, {}
# vytvor seznam vsech generovanych grafu:
mixed_graphs = {}
# pridej automaticky vytvareny seznam nejvice tweetujicich uzivatelu
best_twitters = {}
for stat in stats:
if re.search(r'TWITTER_(.+?)_TWEETS', stat):
mystat = Stat(stat, get_stat_for_graph(dbx, stat))
best_twitters[stat] = mystat.max()
sorted_twitters = sorted(best_twitters.items(),
key=operator.itemgetter(1))[-7:]
stat_id = 'BEST_TWITTERS'
mixed_graphs[stat_id] = [x[0] for x in sorted_twitters]
add_stat_to_group(groups, 'Porovnání', stat_id)
# 1) nacti ty z konfigurace, preved na hashtabulku
for line in func.getconfig('config/graphs'):
lineparts = func.lmap(str.strip, line.split(' '))
mixed_graphs[lineparts[0]] = lineparts[1:]
statnames[lineparts[0]] = lineparts[0]
add_stat_to_group(groups, 'Porovnání', lineparts[0])
# 2) pridej automaticky vytvarene twitter kombinovane grafy
# TWEETS, FOLLOWERS a LIKES
for stat in stats:
found = re.search(r'TWITTER_(.+?)_TWEETS', stat)
if found:
statid = "TWITTER_%s" % found.group(1)
mixed_graphs[statid] = [
stat,
"TWITTER_%s_FOLLOWERS" % found.group(1),
"TWITTER_%s_LIKES" % found.group(1)
]
statnames[statid] = "Twitter %s" % found.group(1) # default jmeno
statnames_index[statid] = "%s" % found.group(
1) # default jmeno na titulni stranku
add_stat_to_group(groups, 'Twitteři', statid)
# 3) pridej vsechny ostatni statistiky, vynechej TWITTERY
# vytvor ponekud nesystemove defaultni nazvy
for stat in stats:
if not re.search(r'TWITTER_(.+)', stat):
mixed_graphs[stat] = [stat]
found = re.search(r'BALANCE_(.+)', stat)
if found:
statnames[stat] = "Zůstatek %s" % found.group(1)
add_stat_to_group(groups, 'Finance', stat)
continue
found = re.search(r'PI_MEMBERS_(.+)', stat)
if found:
statnames[stat] = "Počet členů %s" % found.group(1)
add_stat_to_group(groups, 'Členové', stat)
continue
found = re.search(r'YOUTUBE_(.+)', stat)
if found:
statnames[stat] = "Youtube %s" % found.group(1)
add_stat_to_group(groups, 'Youtube', stat)
continue
found = re.search(r'PP_(.+)', stat)
if found:
add_stat_to_group(groups, 'Finanční tým', stat)
continue
found = re.search(r'REDMINE_(.+)', stat)
if found:
add_stat_to_group(groups, 'Odbory a složky strany na Redmine',
stat)
continue
add_stat_to_group(groups, 'Ostatní', stat)
# donacti jmena statistik z konfigurace
for line in func.getconfig('config/statnames'):
try:
(a, b) = line.split('\t', 2)
statnames[a] = b
except ValueError:
pass
# titulni stranka & assets
mybody = ""
for groupname in groups:
paragraph = []
for statid in groups[groupname]:
if statid in statnames_index.keys():
statname = statnames_index[statid]
elif statid in statnames.keys():
statname = statnames[statid]
else:
statname = statid
paragraph.append(html.a("%s.delta.htm" % statid, statname))
paragraph.sort()
mybody += html.h2(groupname) + html.p(",\n".join(paragraph))
page = func.replace_all(
func.readfile('templates/index.htm'), {
'%body%': mybody,
'%stat_date%': '{0:%d.%m.%Y %H:%M:%S}'.format(
datetime.datetime.now())
})
func.writefile(page, "%s/index.htm" % dirname)
shutil.copytree('templates/assets', "%s/assets" % dirname)
# Vytvor vsechny kombinovane grafy, vynech statistiky s nejvyse jednou hodnotou
for statid in mixed_graphs:
if arg('s') and statid != arg('s'):
continue
i += 1
# graf
involved_stats, involved_deltas = {}, {}
statInstances = []
for invstat in mixed_graphs[statid]:
tmpstat = get_stat_for_graph(dbx, invstat)
involved_stats[invstat] = tmpstat
statInstances.append(Stat(invstat, involved_stats[invstat]))
# spocitej delta statistiku
deltastat, lastvalue = [], None
for entry in tmpstat:
deltastat.append([
entry[0], 0 if lastvalue is None else entry[1] - lastvalue
])
lastvalue = entry[1]
involved_deltas[invstat] = deltastat
singlestat = (len(involved_stats.values()) == 1)
if max(func.lmap(len, involved_stats.values(
))) > 0: # involved_stats musi obsahovat aspon 1 radu o >=1 hodnotach
print("[%s/%s]: Creating %s \r" %
(i, len(mixed_graphs), statid),
end='\r')
# zakladni a delta graf
make_graph(involved_stats,
"%s/img/%s.png" % (dirname, statid),
delta=False)
make_graph(involved_deltas,
"%s/img/%s.delta.png" % (dirname, statid),
delta=True)
# metody ziskani dat
method_list = ""
for stat in involved_stats:
try:
desc = involved_stats[stat][-1:][0][2]
except IndexError:
desc = "Neznámá metoda"
method_list += "%s: %s<br>" % (stat, desc)
# html stranka
statname = statnames[statid] if statid in statnames.keys(
) else statid
min_date = min(
func.lmap(stat_min_date,
filter(lambda x: x,
involved_stats.values()))) # rozsah dat
max_date = max(
func.lmap(stat_max_date,
filter(lambda x: x, involved_stats.values())))
bottom_links = html.h2("Metody získání dat") + \
html.p("Vypsána je vždy poslední použitá metoda, úplný seznam je v CSV souboru." + html.br()*2 + method_list) + \
((html.a("%s.csv" % statid, "Zdrojová data ve formátu CSV") + html.br()) if singlestat else "") + \
html.a("index.htm", "Všechny metriky")
try:
min_value = str(min(map(lambda x: x.min(), statInstances)))
except TypeError:
min_value = '-'
try:
max_value = str(max(map(lambda x: x.max(), statInstances)))
except TypeError:
max_value = '-'
common_replaces = {
'%stat_name%':
statname,
'%stat_desc%':
'',
'%stat_id%':
statid,
'%stat_date%':
'{0:%d.%m.%Y %H:%M:%S}'.format(datetime.datetime.now()),
'%bottomlinks%':
bottom_links,
'%daterange%':
'%s - %s' % (min_date, max_date),
'%max%':
max_value,
'%min%':
min_value
}
page = func.replace_all(
func.readfile('templates/stat.htm'),
merge_dicts(
common_replaces, {
'%stat_image%': "img/%s.png" % statid,
'%stat_type%': "Absolutní hodnoty"
}))
func.writefile(page, "%s/%s.htm" % (dirname, statid))
page = func.replace_all(
func.readfile('templates/stat.htm'),
merge_dicts(
common_replaces, {
'%stat_image%': "img/%s.delta.png" % statid,
'%stat_type%': "Denní přírůstky (delta)"
}))
func.writefile(page, "%s/%s.delta.htm" % (dirname, statid))
# vytvor CSV soubor se zdrojovymi daty
if singlestat:
csv_rows = [
"%s;%s;%s;%s;" %
(statid, "{:%d.%m.%Y}".format(x[0]), x[1], x[2])
for x in list(involved_stats.values())[0]
]
func.writefile(
"stat_id;date;value;method;\n" + "\n".join(csv_rows),
"%s/%s.csv" % (dirname, statid))
def make_graph(rowlist, filename, delta):
""" Vytvori carovy graf. Ulozi jej do souboru,
neni-li jmeno souboru definovano, zobrazi jej
rowlist list s datovymi radami
delta zda je vytvaren graf typu delta hodnoty
"""
rowlist_count = len(rowlist)
minimal_value = min(
func.lmap(lambda x: x[1], sum(list(rowlist.values()), [])))
# datove rady mohou obsahovat chybejici hodnoty, diky nimz
# graf vypada zmatene. Je treba data normalizovat:
# preved data na objekty Stat, zjisti rozsah dat, normalizuj
# zabudovanou funkci a preved zpet na rowlist
stats = []
for row in rowlist:
stats.append(Stat(row, rowlist[row]))
oldest_date = min(filter(lambda x: x, map(lambda x: x.oldest(), stats)))
newest_date = max(filter(lambda x: x, map(lambda x: x.newest(), stats)))
rowlist = {}
for s in stats:
s.fill_range(oldest_date, newest_date)
rowlist[s.name] = s.values
# create graph
figure(num=None, figsize=(16, 10), dpi=80, facecolor='w', edgecolor='w')
ax = plt.axes()
ax.xaxis.set_major_locator(plt.MaxNLocator(6)) # pocet ticku na X ose
i = 0
while len(rowlist.keys()):
# zakladni rada
X, Y, oldest = [], [], get_oldest_timeline(rowlist)
actual_line = rowlist[oldest]
for row in actual_line:
X.append('{0:%d.%m.%Y}'.format(row[0]))
Y.append(row[1])
plt.plot(X, Y, '%s-' % LINE_COLORS[i], linewidth=4.0, label=oldest)
# moving average jen pro trendy (u delty nema moc vyznam)
avg_length = 9 # delka klouzaveho prumeru, tedy kolik dni zpet se vytvari
if not delta:
X, Y = [], []
for j in range(avg_length, len(actual_line)):
row = actual_line[j]
rows_for_avg = func.lmap(lambda x: x[1],
actual_line[j - avg_length:j])
moving_avg = None if None in rows_for_avg else sum(
rows_for_avg) / float(len(rows_for_avg))
X.append('{0:%d.%m.%Y}'.format(row[0]))
Y.append(moving_avg)
plt.plot(X, Y, '%s:' % LINE_COLORS[i], linewidth=2.0, label=oldest)
rowlist = {i: rowlist[i] for i in rowlist if i != oldest}
i += 1
ax.spines['top'].set_visible(False) # odstran horni a pravy ramecek grafu
ax.spines['right'].set_visible(False)
if minimal_value > 0: # osa Y zacina od nuly u kladnych grafu
plt.ylim(bottom=0)
plt.ticklabel_format(style='plain', axis='y')
plt.tick_params(axis='both', which='major',
labelsize=16) # velikost fontu na osach
if rowlist_count > 1:
ax.legend()
if filename:
plt.savefig(filename, bbox_inches='tight')
else:
plt.show()
plt.close() # pri vetsim poctu grafu nutne (memory leak)
def __matmul__(self, f):
return Table(lmap(f, self.vs), ixs=self.ixs)