def is_fr_test(test):
    if test.label and test.banners and test.campaign:
        is_chapter = re.search(config.fr_chapter_test, test.banners[0])
        if is_chapter:
            log.debug("Determined test {title} belongs to a chapter".format(title=test.label))
        else:
            log.debug("Determined test {title} belongs to Fundraising".format(title=test.label))
        return not is_chapter

    log.warn("missing data for test {title}".format(title=test.label))
def is_fr_test(test):
    if test.label and test.banners and test.campaign:
        is_chapter = re.search(config.fr_chapter_test, test.banners[0])
        if is_chapter:
            log.debug("Determined test {title} belongs to a chapter".format(
                title=test.label))
        else:
            log.debug("Determined test {title} belongs to Fundraising".format(
                title=test.label))
        return not is_chapter

    log.warn("missing data for test {title}".format(title=test.label))
Exemple #3
0
    def __init__(self,
                 label=None,
                 type="",
                 campaign=None,
                 banners=None,
                 start=None,
                 end=None,
                 disabled=False,
                 **ignore):
        for key in config.ignored_columns:
            if key in ignore:
                ignore.pop(key)
        if ignore:
            log.warn("ignoring columns: {columns}".format(
                columns=", ".join(ignore.keys())))

        self.campaign = mediawiki.centralnotice.api.get_campaign(campaign)
        if not self.campaign:
            log.warn("no such campaign '{campaign}'".format(campaign=campaign))

        self.type = type.lower()

        self.banners = []
        if self.type.count('banner') > 0:
            self.is_banner_test = True
            if banners:
                if hasattr(banners, 'strip'):
                    banners = [s.strip() for s in banners.split(",")]
                self.banners = banners
            else:
                if self.campaign['banners']:
                    self.banners = self.campaign['banners'].keys()

            #self.variations = [ FrTestVariation(banner=name) for name in self.banners ]

        self.is_country_test = (self.type.count('country') > 0)
        self.is_lp_test = (self.type.count('lp') > 0)

        self.start_time = start
        self.end_time = end

        self.label = label
        if not self.label:
            # FIXME
            self.label = campaign

        self.enabled = not disabled

        self.results = []
def update_gdoc_results(doc=None, results=[]):
    log.info("Updating results in {url}".format(url=doc))
    doc = Spreadsheet(doc=doc)
    existing = list(doc.get_all_rows())

    def find_matching_cases(criteria):
        matching = []

        def fuzzy_compare_row(row, criteria):
            if not row:
                return False
            if criteria['banner'] == row['banner'] and criteria[
                    'campaign'] == row['campaign'] and criteria[
                        'start'] == row['start']:
                return True

        for n, row in enumerate(existing, 1):
            if fuzzy_compare_row(row, criteria):
                matching.append(n)

        return matching

    for result in results:
        if not result:
            continue

        matching = find_matching_cases(result['criteria'])

        props = {}
        props.update(result['results'])
        props.update(result['criteria'])

        if len(matching) == 0:
            doc.append_row(props)
        else:
            if len(matching) > 1:
                log.warn(
                    "more than one result row {match} matches criteria: {criteria}"
                    .format(match=matching, criteria=result['criteria']))
            index = matching[-1]
            log.debug("updating row {rownum} with {banner}".format(
                rownum=index, banner=result['criteria']['banner']))
            doc.update_row(props, index=index)
Exemple #5
0
    def load_results(self):
        if self.is_banner_test and self.banners:
            cases = []
            for name in self.banners:
                test_case = self.get_case(
                    campaign=self.campaign['name'],
                    banner=name,
                    languages=self.campaign['languages'],
                    countries=self.campaign['countries'],
                )
                cases.append(test_case)

            self.results.extend(get_banner_results(cases))

        if self.is_country_test:
            # results = [ calculate_result(country=code) for code in campaign['countries'] ]
            # self.results.extend(results)
            log.warn("country test type not implemented")

        if self.is_lp_test:
            log.warn("LP test type not implemented")
Exemple #6
0
    def load_results(self):
        if self.is_banner_test and self.banners:
            cases = []
            for name in self.banners:
                test_case = self.get_case(
                    campaign=self.campaign['name'],
                    banner=name,
                    languages=self.campaign['languages'],
                    countries=self.campaign['countries'],
                )
                cases.append(test_case)

            self.results.extend(get_banner_results(cases))

        if self.is_country_test:
            #results = [ calculate_result(country=code) for code in campaign['countries'] ]
            #self.results.extend(results)
            log.warn("country test type not implemented")

        if self.is_lp_test:
            log.warn("LP test type not implemented")
Exemple #7
0
    def __init__(self, label=None, type="", campaign=None, banners=None, start=None, end=None, disabled=False, **ignore):
        for key in config.ignored_columns:
            if key in ignore:
                ignore.pop(key)
        if ignore:
            log.warn("ignoring columns: {columns}".format(columns=", ".join(ignore.keys())))

        self.campaign = mediawiki.centralnotice.api.get_campaign(campaign)
        if not self.campaign:
            log.warn("no such campaign '{campaign}'".format(campaign=campaign))

        self.type = type.lower()

        self.banners = []
        if self.type.count('banner') > 0:
            self.is_banner_test = True
            if banners:
                if hasattr(banners, 'strip'):
                    banners = [s.strip() for s in banners.split(",")]
                self.banners = banners
            else:
                if self.campaign['banners']:
                    self.banners = self.campaign['banners'].keys()

            # self.variations = [ FrTestVariation(banner=name) for name in self.banners ]

        self.is_country_test = (self.type.count('country') > 0)
        self.is_lp_test = (self.type.count('lp') > 0)

        self.start_time = start
        self.end_time = end

        self.label = label
        if not self.label:
            # FIXME
            self.label = campaign

        self.enabled = not disabled

        self.results = []
Exemple #8
0
    def reviewBatch(self):
        '''For each new contact, find the oldest contact with the same email address.'''

        matchDescription = EmailMatch("Exact match").json()

        self.contactCache.fetch()
        for contact in self.contactCache.contacts:
            if contact['email']:
                query = db.Query()
                query.columns = [
                    'MIN(contact_id) AS contact_id',
                ]
                query.tables = [
                    'civicrm_email',
                ]
                query.where.extend([
                    'email = %(email)s',
                    'contact_id < %(new_id)s',
                ])
                query.group_by.extend([
                    'email',
                ])
                query.params = {
                    'new_id': contact['id'],
                    'email': contact['email'],
                }
                result = db.get_db().execute(query)

                if result:
                    for row in result:
                        ReviewQueue.addMatch(self.job_id, row['contact_id'], contact['id'], Autoreview.REC_DUP, matchDescription)

            ReviewQueue.tag(contact['id'], QuickAutoreview.QUICK_REVIEWED)

        if not self.contactCache.contacts:
            log.warn("Searched an empty batch of contacts!")
        else:
            last_seen = self.contactCache.contacts[-1]['id']
            log.info("End of batch.  Last contact scanned was ID {id}".format(id=last_seen))
Exemple #9
0
    def pull():
        '''Pull down new remote files'''

        config = process.globals.get_config()

        # Check against both unprocessed and processed files to find new remote files
        local_paths = [
            config.incoming_path,
            config.archive_path,
        ]
        if hasattr(config, 'extra_paths'):
            local_paths.extend(config.extra_paths)
        local_files = walk_files(local_paths)

        remote = Client()
        remote_files = remote.ls(config.sftp.remote_root)
        empty_failures = []

        for filename in remote_files:
            if filename in local_files:
                log.info("Skipping already downloaded file {filename}".format(filename=filename))
                continue

            log.info("Downloading file {filename}".format(filename=filename))
            dest_path = os.path.join(config.incoming_path, filename)
            remote.get(os.path.join(config.sftp.remote_root, filename), dest_path)

            # Assert that the file is not empty
            if os.path.getsize(dest_path) == 0:
                os.unlink(dest_path)
                empty_failures.append(filename)
                log.warn("Stupid file was empty, removing locally: {path}".format(path=dest_path))

        if empty_failures:
            log.error("The following files were empty, please contact your provider: {failures}".format(failures=", ".join(empty_failures)))

            if hasattr(config, 'panic_on_empty') and config.panic_on_empty:
                raise RuntimeError("Stupid files did not download correctly.")
def update_gdoc_results(doc=None, results=[]):
    log.info("Updating results in {url}".format(url=doc))
    doc = Spreadsheet(doc=doc)
    existing = list(doc.get_all_rows())

    def find_matching_cases(criteria):
        matching = []

        def fuzzy_compare_row(row, criteria):
            if not row:
                return False
            if criteria['banner'] == row['banner'] and criteria['campaign'] == row['campaign'] and criteria['start'] == row['start']:
                return True

        for n, row in enumerate(existing, 1):
            if fuzzy_compare_row(row, criteria):
                matching.append(n)

        return matching

    for result in results:
        if not result:
            continue

        matching = find_matching_cases(result['criteria'])

        props = {}
        props.update(result['results'])
        props.update(result['criteria'])

        if len(matching) == 0:
            doc.append_row(props)
        else:
            if len(matching) > 1:
                log.warn("more than one result row {match} matches criteria: {criteria}".format(match=matching, criteria=result['criteria']))
            index = matching[-1]
            log.debug("updating row {rownum} with {banner}".format(rownum=index, banner=result['criteria']['banner']))
            doc.update_row(props, index=index)
 def kill_connection(self):
     log.warn('Query taking too long - killing connection {}'.format(self.connection_id))
     killerConnection = Dbi.connect(**self.connectionArgs)
     cursor = killerConnection.cursor()
     cursor.execute('KILL CONNECTION {}'.format(self.connection_id))
     killerConnection.close()