class ReportingOrgMaker(object): NO_CANDIDATE = 'no candidate' MULTIPLE_CANDIDATES = 'multiple candidates' SYNC_NOT_SUPPORT = 'sync not support' def __init__(self, options): self.stdout = OutputWrapper(options.get('stdout', sys.stdout)) self.keyword_based_reporting_orgs = { 'WASH Alliance': 8, 'Connect4Change': 34, 'SRHR Alliance': 1043, 'WvW': 43, 'wvw2014': 43, 'wvw2015': 43, 'wvw2016': 43, 'WfW': 43, 'wfw2014': 43, 'wfw2015': 43, 'wfw2016': 43, } self.keywords_set = set(self.keyword_based_reporting_orgs.keys()) self.migrate = options['migrate'] self.ok_list = [] self.fix_list = [] def add_to_ok(self, project, org): self.ok_list += [OKProject(project, org)] def add_to_fix(self, project, reason, partners, sync_owner=None): self.fix_list += [FixProject(project, reason, partners, sync_owner)] def find_reporting_org_for_projects(self): # loop over all projects, trying ot figure reporting-org for each. self.stdout.write('\nData gathering progress:') i = 1 for project in Project.objects.published().prefetch_related( 'partnerships', 'partnerships__organisation', 'keywords'): if not i % 100: self.stdout.write(str(i)) else: self.stdout.write(".", ending='') i += 1 self.stdout.flush() # first check if we have a keyword from the keyword_based_reporting_orgs.keys() list reporting_keyword = self.keywords_set.intersection( set(project.keywords.values_list('label', flat=True))) if reporting_keyword: # if we do, set the reporting-org to the org connected to the keyword self.add_to_ok( project, Organisation.objects.get( pk=self.keyword_based_reporting_orgs[list( reporting_keyword)[0]])) else: # otherwise try to find the reporting org among sync_owner and accountable partners support_partners = project.partnerships.filter( iati_organisation_role=Partnership.IATI_ACCOUNTABLE_PARTNER ).select_related('organisation') # If there's no support partner, we set the sync_owner as reporting-org, # if there is one. Otherwise we report the problem. if support_partners.count() == 0: if project.sync_owner: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.NO_CANDIDATE, []) # If we have exactly one support partner, then things are in order if either: # 1) the sync_owner matches the support partner # 2) there is no sync_owner # In both cases we should be fine to set the sync_owner/support partner as the # reporting-org. elif support_partners.count() == 1: if project.sync_owner: # 1) if project.sync_owner == support_partners[ 0].organisation: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.SYNC_NOT_SUPPORT, support_partners, project.sync_owner) # 2) else: self.add_to_ok(project, support_partners[0].organisation) # If there are multiple support partners we check if one of the partners is sync_owner # we set that organisation to reporting. Otherwise we report the problem. else: if project.sync_owner: if project.sync_owner.id in [ p.organisation.id for p in support_partners ]: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.MULTIPLE_CANDIDATES, support_partners) def create_reporting_orgs(self): try: reporting_org_choice = Partnership.IATI_REPORTING_ORGANISATION self.stdout.write( u"\n*** Assigning reporting-org partners to the following projects ***" ) self.stdout.write( u"project ID, project title, organisation id, organisation name" ) for data in self.ok_list: partner = Partnership( organisation=data.organisation, iati_organisation_role=reporting_org_choice) data.project.partnerships.add(partner) self.print_ok_data(data) except: self.stdout.write( u"\n*** Reporting organisation choice not available for Partnerships ***" ) def print_ok_data(self, data): self.stdout.write(u'{},"{}",{},"{}"'.format(data.project.id, data.project.title, data.organisation.id, data.organisation.name)) def print_fix_data(self, data, partner): self.stdout.write(u'{},"{}",{},"{}","{}",{},"{}"'.format( data.project.id, data.project.title, partner.organisation.id, partner.organisation.name, data.reason, data.sync_owner.id if data.sync_owner else '', data.sync_owner.name if data.sync_owner else '')) def output_ok_list(self): self.stdout.write( u"\n*** List of projects and the <reporting-org> partner they will get when migrating ***" ) self.stdout.write( u"project ID, project title, organisation id, organisation name") for data in self.ok_list: self.print_ok_data(data) def output_fix_list(self): self.stdout.write( u"\n*** List of projects where no clear-cut reporting-org candidate was found ***" ) self.stdout.write( u"project ID, project title, support partner id, support partner name, type of problem, sync_owner id, sync_owner name" ) for data in self.fix_list: for partner in data.partners: self.print_fix_data(data, partner)
def parse_logs(qs, stdout=None): """ Parse logs for kudos. """ names = collections.deque(maxlen=200) unattributed = 0 count = 0 kudos = {} kudos_count = 0 kudos_first = {} kudos_recent = {} if stdout and not isinstance(stdout, OutputWrapper): stdout = OutputWrapper(stdout) def set_thanked(nick): timestamp = log[3] kudos[nick] = kudos.get(nick, 0) + 1 kudos_first.setdefault(nick, timestamp) kudos_recent[nick] = timestamp qs = qs.order_by('pk').filter(command='PRIVMSG') qs = qs.values_list('pk', 'nick', 'text', 'timestamp') for log in _iterate_log(qs): log_nick = log[1].lower() log_text = log[2] count += 1 directed = directed_message(log_text) if directed: directed = directed.lower() if directed == log_nick: # Can't thank yourself :P directed = None if RE_KUDOS.search(log_text): kudos_count += 1 attributed = False if directed: for nick, _ in names: if nick == directed: set_thanked(nick) attributed = True break if not attributed: lower_text = log_text.lower() for recent in ( bits[0] for bits in names if bits[0] != log_nick): re_text = '(?:^| )@?{}(?:$|\W)'.format(re.escape(recent)) if re.search(re_text, lower_text): set_thanked(recent) attributed = True if not attributed: for nick, directed in names: if directed == log_nick: set_thanked(nick) attributed = True break if not attributed: unattributed += 1 names.append((log_nick, directed)) if stdout and not count % 10000: stdout.write('.', ending='') stdout.flush() if stdout: stdout.write('') kudos_list = [] for c, nick in sorted((c, nick) for nick, c in kudos.items()): kudos_list.append({ 'nick': nick, 'count': c, 'first': kudos_first[nick], 'recent': kudos_recent[nick] }) return { 'kudos': kudos_list, 'message_count': count, 'kudos_given': kudos_count, 'unattributed': unattributed, }
class ReportingOrgMaker(object): NO_CANDIDATE = 'no candidate' MULTIPLE_CANDIDATES = 'multiple candidates' SYNC_NOT_SUPPORT = 'sync not support' def __init__(self, options): self.stdout = OutputWrapper(options.get('stdout', sys.stdout)) self.keyword_based_reporting_orgs = { 'WASH Alliance': 8, 'Connect4Change': 34, 'SRHR Alliance': 1043, 'WvW': 43, 'wvw2014': 43, 'wvw2015': 43, 'wvw2016': 43, 'WfW': 43, 'wfw2014': 43, 'wfw2015': 43, 'wfw2016': 43, } self.keywords_set = set(self.keyword_based_reporting_orgs.keys()) self.migrate = options['migrate'] self.ok_list = [] self.fix_list = [] def add_to_ok(self, project, org): self.ok_list += [OKProject(project, org)] def add_to_fix(self, project, reason, partners, sync_owner=None): self.fix_list += [FixProject(project, reason, partners, sync_owner)] def find_reporting_org_for_projects(self): # loop over all projects, trying ot figure reporting-org for each. self.stdout.write('\nData gathering progress:') i = 1 for project in Project.objects.published().prefetch_related( 'partnerships', 'partnerships__organisation', 'keywords'): if not i % 100: self.stdout.write(str(i)) else: self.stdout.write(".", ending='') i += 1 self.stdout.flush() # first check if we have a keyword from the keyword_based_reporting_orgs.keys() list reporting_keyword = self.keywords_set.intersection( set(project.keywords.values_list('label', flat=True))) if reporting_keyword: # if we do, set the reporting-org to the org connected to the keyword self.add_to_ok(project, Organisation.objects.get( pk=self.keyword_based_reporting_orgs[list(reporting_keyword)[0]])) else: # otherwise try to find the reporting org among sync_owner and accountable partners support_partners = project.partnerships.filter( iati_organisation_role=Partnership.IATI_ACCOUNTABLE_PARTNER ).select_related('organisation') # If there's no support partner, we set the sync_owner as reporting-org, # if there is one. Otherwise we report the problem. if support_partners.count() == 0: if project.sync_owner: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.NO_CANDIDATE, []) # If we have exactly one support partner, then things are in order if either: # 1) the sync_owner matches the support partner # 2) there is no sync_owner # In both cases we should be fine to set the sync_owner/support partner as the # reporting-org. elif support_partners.count() == 1: if project.sync_owner: # 1) if project.sync_owner == support_partners[0].organisation: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.SYNC_NOT_SUPPORT, support_partners, project.sync_owner) # 2) else: self.add_to_ok(project, support_partners[0].organisation) # If there are multiple support partners we check if one of the partners is sync_owner # we set that organisation to reporting. Otherwise we report the problem. else: if project.sync_owner: if project.sync_owner.id in [p.organisation.id for p in support_partners]: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.MULTIPLE_CANDIDATES, support_partners) def create_reporting_orgs(self): try: reporting_org_choice = Partnership.IATI_REPORTING_ORGANISATION self.stdout.write( u"\n*** Assigning reporting-org partners to the following projects ***" ) self.stdout.write( u"project ID, project title, organisation id, organisation name" ) for data in self.ok_list: partner = Partnership( organisation=data.organisation, iati_organisation_role=reporting_org_choice) data.project.partnerships.add(partner) self.print_ok_data(data) except: self.stdout.write( u"\n*** Reporting organisation choice not available for Partnerships ***" ) def print_ok_data(self, data): self.stdout.write( u'{},"{}",{},"{}"'.format(data.project.id, data.project.title, data.organisation.id, data.organisation.name)) def print_fix_data(self, data, partner): self.stdout.write( u'{},"{}",{},"{}","{}",{},"{}"'.format( data.project.id, data.project.title, partner.organisation.id, partner.organisation.name, data.reason, data.sync_owner.id if data.sync_owner else '', data.sync_owner.name if data.sync_owner else '')) def output_ok_list(self): self.stdout.write( u"\n*** List of projects and the <reporting-org> partner they will get when migrating ***" ) self.stdout.write( u"project ID, project title, organisation id, organisation name" ) for data in self.ok_list: self.print_ok_data(data) def output_fix_list(self): self.stdout.write( u"\n*** List of projects where no clear-cut reporting-org candidate was found ***" ) self.stdout.write( u"project ID, project title, support partner id, support partner name, type of problem, sync_owner id, sync_owner name" ) for data in self.fix_list: for partner in data.partners: self.print_fix_data(data, partner)
def parse_logs(qs, stdout=None): """ Parse logs for kudos. """ names = collections.deque(maxlen=200) unattributed = 0 count = 0 kudos = {} kudos_count = 0 kudos_first = {} kudos_recent = {} if stdout and not isinstance(stdout, OutputWrapper): stdout = OutputWrapper(stdout) def set_thanked(nick): timestamp = log[3] kudos[nick] = kudos.get(nick, 0) + 1 kudos_first.setdefault(nick, timestamp) kudos_recent[nick] = timestamp qs = qs.order_by('pk').filter(command='PRIVMSG') qs = qs.values_list('pk', 'nick', 'text', 'timestamp') for log in _iterate_log(qs): log_nick = log[1].lower() log_text = log[2] count += 1 directed = directed_message(log_text) if directed: directed = directed.lower() if directed == log_nick: # Can't thank yourself :P directed = None if RE_KUDOS.search(log_text): kudos_count += 1 attributed = False if directed: for nick, _ in names: if nick == directed: set_thanked(nick) attributed = True break if not attributed: lower_text = log_text.lower() for recent in (bits[0] for bits in names if bits[0] != log_nick): re_text = '(?:^| )@?{}(?:$|\W)'.format(re.escape(recent)) if re.search(re_text, lower_text): set_thanked(recent) attributed = True if not attributed: for nick, directed in names: if directed == log_nick: set_thanked(nick) attributed = True break if not attributed: unattributed += 1 names.append((log_nick, directed)) if stdout and not count % 10000: stdout.write('.', ending='') stdout.flush() if stdout: stdout.write('') kudos_list = [] for c, nick in sorted((c, nick) for nick, c in kudos.items()): kudos_list.append({ 'nick': nick, 'count': c, 'first': kudos_first[nick], 'recent': kudos_recent[nick] }) return { 'kudos': kudos_list, 'message_count': count, 'kudos_given': kudos_count, 'unattributed': unattributed, }