def is_fr_test(test): if test.label and test.banners and test.campaign: is_chapter = re.search(config.fr_chapter_test, test.banners[0]) if is_chapter: log.debug("Determined test {title} belongs to a chapter".format(title=test.label)) else: log.debug("Determined test {title} belongs to Fundraising".format(title=test.label)) return not is_chapter log.warn("missing data for test {title}".format(title=test.label))
def is_fr_test(test): if test.label and test.banners and test.campaign: is_chapter = re.search(config.fr_chapter_test, test.banners[0]) if is_chapter: log.debug("Determined test {title} belongs to a chapter".format( title=test.label)) else: log.debug("Determined test {title} belongs to Fundraising".format( title=test.label)) return not is_chapter log.warn("missing data for test {title}".format(title=test.label))
def __init__(self, label=None, type="", campaign=None, banners=None, start=None, end=None, disabled=False, **ignore): for key in config.ignored_columns: if key in ignore: ignore.pop(key) if ignore: log.warn("ignoring columns: {columns}".format( columns=", ".join(ignore.keys()))) self.campaign = mediawiki.centralnotice.api.get_campaign(campaign) if not self.campaign: log.warn("no such campaign '{campaign}'".format(campaign=campaign)) self.type = type.lower() self.banners = [] if self.type.count('banner') > 0: self.is_banner_test = True if banners: if hasattr(banners, 'strip'): banners = [s.strip() for s in banners.split(",")] self.banners = banners else: if self.campaign['banners']: self.banners = self.campaign['banners'].keys() #self.variations = [ FrTestVariation(banner=name) for name in self.banners ] self.is_country_test = (self.type.count('country') > 0) self.is_lp_test = (self.type.count('lp') > 0) self.start_time = start self.end_time = end self.label = label if not self.label: # FIXME self.label = campaign self.enabled = not disabled self.results = []
def update_gdoc_results(doc=None, results=[]): log.info("Updating results in {url}".format(url=doc)) doc = Spreadsheet(doc=doc) existing = list(doc.get_all_rows()) def find_matching_cases(criteria): matching = [] def fuzzy_compare_row(row, criteria): if not row: return False if criteria['banner'] == row['banner'] and criteria[ 'campaign'] == row['campaign'] and criteria[ 'start'] == row['start']: return True for n, row in enumerate(existing, 1): if fuzzy_compare_row(row, criteria): matching.append(n) return matching for result in results: if not result: continue matching = find_matching_cases(result['criteria']) props = {} props.update(result['results']) props.update(result['criteria']) if len(matching) == 0: doc.append_row(props) else: if len(matching) > 1: log.warn( "more than one result row {match} matches criteria: {criteria}" .format(match=matching, criteria=result['criteria'])) index = matching[-1] log.debug("updating row {rownum} with {banner}".format( rownum=index, banner=result['criteria']['banner'])) doc.update_row(props, index=index)
def load_results(self): if self.is_banner_test and self.banners: cases = [] for name in self.banners: test_case = self.get_case( campaign=self.campaign['name'], banner=name, languages=self.campaign['languages'], countries=self.campaign['countries'], ) cases.append(test_case) self.results.extend(get_banner_results(cases)) if self.is_country_test: # results = [ calculate_result(country=code) for code in campaign['countries'] ] # self.results.extend(results) log.warn("country test type not implemented") if self.is_lp_test: log.warn("LP test type not implemented")
def load_results(self): if self.is_banner_test and self.banners: cases = [] for name in self.banners: test_case = self.get_case( campaign=self.campaign['name'], banner=name, languages=self.campaign['languages'], countries=self.campaign['countries'], ) cases.append(test_case) self.results.extend(get_banner_results(cases)) if self.is_country_test: #results = [ calculate_result(country=code) for code in campaign['countries'] ] #self.results.extend(results) log.warn("country test type not implemented") if self.is_lp_test: log.warn("LP test type not implemented")
def __init__(self, label=None, type="", campaign=None, banners=None, start=None, end=None, disabled=False, **ignore): for key in config.ignored_columns: if key in ignore: ignore.pop(key) if ignore: log.warn("ignoring columns: {columns}".format(columns=", ".join(ignore.keys()))) self.campaign = mediawiki.centralnotice.api.get_campaign(campaign) if not self.campaign: log.warn("no such campaign '{campaign}'".format(campaign=campaign)) self.type = type.lower() self.banners = [] if self.type.count('banner') > 0: self.is_banner_test = True if banners: if hasattr(banners, 'strip'): banners = [s.strip() for s in banners.split(",")] self.banners = banners else: if self.campaign['banners']: self.banners = self.campaign['banners'].keys() # self.variations = [ FrTestVariation(banner=name) for name in self.banners ] self.is_country_test = (self.type.count('country') > 0) self.is_lp_test = (self.type.count('lp') > 0) self.start_time = start self.end_time = end self.label = label if not self.label: # FIXME self.label = campaign self.enabled = not disabled self.results = []
def reviewBatch(self): '''For each new contact, find the oldest contact with the same email address.''' matchDescription = EmailMatch("Exact match").json() self.contactCache.fetch() for contact in self.contactCache.contacts: if contact['email']: query = db.Query() query.columns = [ 'MIN(contact_id) AS contact_id', ] query.tables = [ 'civicrm_email', ] query.where.extend([ 'email = %(email)s', 'contact_id < %(new_id)s', ]) query.group_by.extend([ 'email', ]) query.params = { 'new_id': contact['id'], 'email': contact['email'], } result = db.get_db().execute(query) if result: for row in result: ReviewQueue.addMatch(self.job_id, row['contact_id'], contact['id'], Autoreview.REC_DUP, matchDescription) ReviewQueue.tag(contact['id'], QuickAutoreview.QUICK_REVIEWED) if not self.contactCache.contacts: log.warn("Searched an empty batch of contacts!") else: last_seen = self.contactCache.contacts[-1]['id'] log.info("End of batch. Last contact scanned was ID {id}".format(id=last_seen))
def pull(): '''Pull down new remote files''' config = process.globals.get_config() # Check against both unprocessed and processed files to find new remote files local_paths = [ config.incoming_path, config.archive_path, ] if hasattr(config, 'extra_paths'): local_paths.extend(config.extra_paths) local_files = walk_files(local_paths) remote = Client() remote_files = remote.ls(config.sftp.remote_root) empty_failures = [] for filename in remote_files: if filename in local_files: log.info("Skipping already downloaded file {filename}".format(filename=filename)) continue log.info("Downloading file {filename}".format(filename=filename)) dest_path = os.path.join(config.incoming_path, filename) remote.get(os.path.join(config.sftp.remote_root, filename), dest_path) # Assert that the file is not empty if os.path.getsize(dest_path) == 0: os.unlink(dest_path) empty_failures.append(filename) log.warn("Stupid file was empty, removing locally: {path}".format(path=dest_path)) if empty_failures: log.error("The following files were empty, please contact your provider: {failures}".format(failures=", ".join(empty_failures))) if hasattr(config, 'panic_on_empty') and config.panic_on_empty: raise RuntimeError("Stupid files did not download correctly.")
def update_gdoc_results(doc=None, results=[]): log.info("Updating results in {url}".format(url=doc)) doc = Spreadsheet(doc=doc) existing = list(doc.get_all_rows()) def find_matching_cases(criteria): matching = [] def fuzzy_compare_row(row, criteria): if not row: return False if criteria['banner'] == row['banner'] and criteria['campaign'] == row['campaign'] and criteria['start'] == row['start']: return True for n, row in enumerate(existing, 1): if fuzzy_compare_row(row, criteria): matching.append(n) return matching for result in results: if not result: continue matching = find_matching_cases(result['criteria']) props = {} props.update(result['results']) props.update(result['criteria']) if len(matching) == 0: doc.append_row(props) else: if len(matching) > 1: log.warn("more than one result row {match} matches criteria: {criteria}".format(match=matching, criteria=result['criteria'])) index = matching[-1] log.debug("updating row {rownum} with {banner}".format(rownum=index, banner=result['criteria']['banner'])) doc.update_row(props, index=index)
def kill_connection(self): log.warn('Query taking too long - killing connection {}'.format(self.connection_id)) killerConnection = Dbi.connect(**self.connectionArgs) cursor = killerConnection.cursor() cursor.execute('KILL CONNECTION {}'.format(self.connection_id)) killerConnection.close()