class ReportingOrgMaker(object):
NO_CANDIDATE = 'no candidate'
MULTIPLE_CANDIDATES = 'multiple candidates'
SYNC_NOT_SUPPORT = 'sync not support'
def __init__(self, options):
self.stdout = OutputWrapper(options.get('stdout', sys.stdout))
self.keyword_based_reporting_orgs = {
'WASH Alliance': 8,
'Connect4Change': 34,
'SRHR Alliance': 1043,
'WvW': 43,
'wvw2014': 43,
'wvw2015': 43,
'wvw2016': 43,
'WfW': 43,
'wfw2014': 43,
'wfw2015': 43,
'wfw2016': 43,
}
self.keywords_set = set(self.keyword_based_reporting_orgs.keys())
self.migrate = options['migrate']
self.ok_list = []
self.fix_list = []
def add_to_ok(self, project, org):
self.ok_list += [OKProject(project, org)]
def add_to_fix(self, project, reason, partners, sync_owner=None):
self.fix_list += [FixProject(project, reason, partners, sync_owner)]
def find_reporting_org_for_projects(self):
# loop over all projects, trying ot figure reporting-org for each.
self.stdout.write('\nData gathering progress:')
i = 1
for project in Project.objects.published().prefetch_related(
'partnerships', 'partnerships__organisation', 'keywords'):
if not i % 100:
self.stdout.write(str(i))
else:
self.stdout.write(".", ending='')
i += 1
self.stdout.flush()
# first check if we have a keyword from the keyword_based_reporting_orgs.keys() list
reporting_keyword = self.keywords_set.intersection(
set(project.keywords.values_list('label', flat=True)))
if reporting_keyword:
# if we do, set the reporting-org to the org connected to the keyword
self.add_to_ok(
project,
Organisation.objects.get(
pk=self.keyword_based_reporting_orgs[list(
reporting_keyword)[0]]))
else:
# otherwise try to find the reporting org among sync_owner and accountable partners
support_partners = project.partnerships.filter(
iati_organisation_role=Partnership.IATI_ACCOUNTABLE_PARTNER
).select_related('organisation')
# If there's no support partner, we set the sync_owner as reporting-org,
# if there is one. Otherwise we report the problem.
if support_partners.count() == 0:
if project.sync_owner:
self.add_to_ok(project, project.sync_owner)
else:
self.add_to_fix(project, self.NO_CANDIDATE, [])
# If we have exactly one support partner, then things are in order if either:
# 1) the sync_owner matches the support partner
# 2) there is no sync_owner
# In both cases we should be fine to set the sync_owner/support partner as the
# reporting-org.
elif support_partners.count() == 1:
if project.sync_owner:
# 1)
if project.sync_owner == support_partners[
0].organisation:
self.add_to_ok(project, project.sync_owner)
else:
self.add_to_fix(project, self.SYNC_NOT_SUPPORT,
support_partners,
project.sync_owner)
# 2)
else:
self.add_to_ok(project,
support_partners[0].organisation)
# If there are multiple support partners we check if one of the partners is sync_owner
# we set that organisation to reporting. Otherwise we report the problem.
else:
if project.sync_owner:
if project.sync_owner.id in [
p.organisation.id for p in support_partners
]:
self.add_to_ok(project, project.sync_owner)
else:
self.add_to_fix(project, self.MULTIPLE_CANDIDATES,
support_partners)
def create_reporting_orgs(self):
try:
reporting_org_choice = Partnership.IATI_REPORTING_ORGANISATION
self.stdout.write(
u"\n*** Assigning reporting-org partners to the following projects ***"
)
self.stdout.write(
u"project ID, project title, organisation id, organisation name"
)
for data in self.ok_list:
partner = Partnership(
organisation=data.organisation,
iati_organisation_role=reporting_org_choice)
data.project.partnerships.add(partner)
self.print_ok_data(data)
except:
self.stdout.write(
u"\n*** Reporting organisation choice not available for Partnerships ***"
)
def print_ok_data(self, data):
self.stdout.write(u'{},"{}",{},"{}"'.format(data.project.id,
data.project.title,
data.organisation.id,
data.organisation.name))
def print_fix_data(self, data, partner):
self.stdout.write(u'{},"{}",{},"{}","{}",{},"{}"'.format(
data.project.id, data.project.title, partner.organisation.id,
partner.organisation.name, data.reason,
data.sync_owner.id if data.sync_owner else '',
data.sync_owner.name if data.sync_owner else ''))
def output_ok_list(self):
self.stdout.write(
u"\n*** List of projects and the <reporting-org> partner they will get when migrating ***"
)
self.stdout.write(
u"project ID, project title, organisation id, organisation name")
for data in self.ok_list:
self.print_ok_data(data)
def output_fix_list(self):
self.stdout.write(
u"\n*** List of projects where no clear-cut reporting-org candidate was found ***"
)
self.stdout.write(
u"project ID, project title, support partner id, support partner name, type of problem, sync_owner id, sync_owner name"
)
for data in self.fix_list:
for partner in data.partners:
self.print_fix_data(data, partner)
def parse_logs(qs, stdout=None):
"""
Parse logs for kudos.
"""
names = collections.deque(maxlen=200)
unattributed = 0
count = 0
kudos = {}
kudos_count = 0
kudos_first = {}
kudos_recent = {}
if stdout and not isinstance(stdout, OutputWrapper):
stdout = OutputWrapper(stdout)
def set_thanked(nick):
timestamp = log[3]
kudos[nick] = kudos.get(nick, 0) + 1
kudos_first.setdefault(nick, timestamp)
kudos_recent[nick] = timestamp
qs = qs.order_by('pk').filter(command='PRIVMSG')
qs = qs.values_list('pk', 'nick', 'text', 'timestamp')
for log in _iterate_log(qs):
log_nick = log[1].lower()
log_text = log[2]
count += 1
directed = directed_message(log_text)
if directed:
directed = directed.lower()
if directed == log_nick:
# Can't thank yourself :P
directed = None
if RE_KUDOS.search(log_text):
kudos_count += 1
attributed = False
if directed:
for nick, _ in names:
if nick == directed:
set_thanked(nick)
attributed = True
break
if not attributed:
lower_text = log_text.lower()
for recent in (
bits[0] for bits in names if bits[0] != log_nick):
re_text = '(?:^| )@?{}(?:$|\W)'.format(re.escape(recent))
if re.search(re_text, lower_text):
set_thanked(recent)
attributed = True
if not attributed:
for nick, directed in names:
if directed == log_nick:
set_thanked(nick)
attributed = True
break
if not attributed:
unattributed += 1
names.append((log_nick, directed))
if stdout and not count % 10000:
stdout.write('.', ending='')
stdout.flush()
if stdout:
stdout.write('')
kudos_list = []
for c, nick in sorted((c, nick) for nick, c in kudos.items()):
kudos_list.append({
'nick': nick,
'count': c,
'first': kudos_first[nick],
'recent': kudos_recent[nick]
})
return {
'kudos': kudos_list,
'message_count': count,
'kudos_given': kudos_count,
'unattributed': unattributed,
}
class ReportingOrgMaker(object):
NO_CANDIDATE = 'no candidate'
MULTIPLE_CANDIDATES = 'multiple candidates'
SYNC_NOT_SUPPORT = 'sync not support'
def __init__(self, options):
self.stdout = OutputWrapper(options.get('stdout', sys.stdout))
self.keyword_based_reporting_orgs = {
'WASH Alliance': 8,
'Connect4Change': 34,
'SRHR Alliance': 1043,
'WvW': 43,
'wvw2014': 43,
'wvw2015': 43,
'wvw2016': 43,
'WfW': 43,
'wfw2014': 43,
'wfw2015': 43,
'wfw2016': 43,
}
self.keywords_set = set(self.keyword_based_reporting_orgs.keys())
self.migrate = options['migrate']
self.ok_list = []
self.fix_list = []
def add_to_ok(self, project, org):
self.ok_list += [OKProject(project, org)]
def add_to_fix(self, project, reason, partners, sync_owner=None):
self.fix_list += [FixProject(project, reason, partners, sync_owner)]
def find_reporting_org_for_projects(self):
# loop over all projects, trying ot figure reporting-org for each.
self.stdout.write('\nData gathering progress:')
i = 1
for project in Project.objects.published().prefetch_related(
'partnerships', 'partnerships__organisation', 'keywords'):
if not i % 100:
self.stdout.write(str(i))
else:
self.stdout.write(".", ending='')
i += 1
self.stdout.flush()
# first check if we have a keyword from the keyword_based_reporting_orgs.keys() list
reporting_keyword = self.keywords_set.intersection(
set(project.keywords.values_list('label', flat=True)))
if reporting_keyword:
# if we do, set the reporting-org to the org connected to the keyword
self.add_to_ok(project, Organisation.objects.get(
pk=self.keyword_based_reporting_orgs[list(reporting_keyword)[0]]))
else:
# otherwise try to find the reporting org among sync_owner and accountable partners
support_partners = project.partnerships.filter(
iati_organisation_role=Partnership.IATI_ACCOUNTABLE_PARTNER
).select_related('organisation')
# If there's no support partner, we set the sync_owner as reporting-org,
# if there is one. Otherwise we report the problem.
if support_partners.count() == 0:
if project.sync_owner:
self.add_to_ok(project, project.sync_owner)
else:
self.add_to_fix(project, self.NO_CANDIDATE, [])
# If we have exactly one support partner, then things are in order if either:
# 1) the sync_owner matches the support partner
# 2) there is no sync_owner
# In both cases we should be fine to set the sync_owner/support partner as the
# reporting-org.
elif support_partners.count() == 1:
if project.sync_owner:
# 1)
if project.sync_owner == support_partners[0].organisation:
self.add_to_ok(project, project.sync_owner)
else:
self.add_to_fix(project, self.SYNC_NOT_SUPPORT, support_partners,
project.sync_owner)
# 2)
else:
self.add_to_ok(project, support_partners[0].organisation)
# If there are multiple support partners we check if one of the partners is sync_owner
# we set that organisation to reporting. Otherwise we report the problem.
else:
if project.sync_owner:
if project.sync_owner.id in [p.organisation.id for p in support_partners]:
self.add_to_ok(project, project.sync_owner)
else:
self.add_to_fix(project, self.MULTIPLE_CANDIDATES, support_partners)
def create_reporting_orgs(self):
try:
reporting_org_choice = Partnership.IATI_REPORTING_ORGANISATION
self.stdout.write(
u"\n*** Assigning reporting-org partners to the following projects ***"
)
self.stdout.write(
u"project ID, project title, organisation id, organisation name"
)
for data in self.ok_list:
partner = Partnership(
organisation=data.organisation,
iati_organisation_role=reporting_org_choice)
data.project.partnerships.add(partner)
self.print_ok_data(data)
except:
self.stdout.write(
u"\n*** Reporting organisation choice not available for Partnerships ***"
)
def print_ok_data(self, data):
self.stdout.write(
u'{},"{}",{},"{}"'.format(data.project.id, data.project.title, data.organisation.id,
data.organisation.name))
def print_fix_data(self, data, partner):
self.stdout.write(
u'{},"{}",{},"{}","{}",{},"{}"'.format(
data.project.id,
data.project.title,
partner.organisation.id,
partner.organisation.name,
data.reason,
data.sync_owner.id if data.sync_owner else '',
data.sync_owner.name if data.sync_owner else ''))
def output_ok_list(self):
self.stdout.write(
u"\n*** List of projects and the <reporting-org> partner they will get when migrating ***"
)
self.stdout.write(
u"project ID, project title, organisation id, organisation name"
)
for data in self.ok_list:
self.print_ok_data(data)
def output_fix_list(self):
self.stdout.write(
u"\n*** List of projects where no clear-cut reporting-org candidate was found ***"
)
self.stdout.write(
u"project ID, project title, support partner id, support partner name, type of problem, sync_owner id, sync_owner name"
)
for data in self.fix_list:
for partner in data.partners:
self.print_fix_data(data, partner)
def parse_logs(qs, stdout=None):
"""
Parse logs for kudos.
"""
names = collections.deque(maxlen=200)
unattributed = 0
count = 0
kudos = {}
kudos_count = 0
kudos_first = {}
kudos_recent = {}
if stdout and not isinstance(stdout, OutputWrapper):
stdout = OutputWrapper(stdout)
def set_thanked(nick):
timestamp = log[3]
kudos[nick] = kudos.get(nick, 0) + 1
kudos_first.setdefault(nick, timestamp)
kudos_recent[nick] = timestamp
qs = qs.order_by('pk').filter(command='PRIVMSG')
qs = qs.values_list('pk', 'nick', 'text', 'timestamp')
for log in _iterate_log(qs):
log_nick = log[1].lower()
log_text = log[2]
count += 1
directed = directed_message(log_text)
if directed:
directed = directed.lower()
if directed == log_nick:
# Can't thank yourself :P
directed = None
if RE_KUDOS.search(log_text):
kudos_count += 1
attributed = False
if directed:
for nick, _ in names:
if nick == directed:
set_thanked(nick)
attributed = True
break
if not attributed:
lower_text = log_text.lower()
for recent in (bits[0] for bits in names
if bits[0] != log_nick):
re_text = '(?:^| )@?{}(?:$|\W)'.format(re.escape(recent))
if re.search(re_text, lower_text):
set_thanked(recent)
attributed = True
if not attributed:
for nick, directed in names:
if directed == log_nick:
set_thanked(nick)
attributed = True
break
if not attributed:
unattributed += 1
names.append((log_nick, directed))
if stdout and not count % 10000:
stdout.write('.', ending='')
stdout.flush()
if stdout:
stdout.write('')
kudos_list = []
for c, nick in sorted((c, nick) for nick, c in kudos.items()):
kudos_list.append({
'nick': nick,
'count': c,
'first': kudos_first[nick],
'recent': kudos_recent[nick]
})
return {
'kudos': kudos_list,
'message_count': count,
'kudos_given': kudos_count,
'unattributed': unattributed,
}
