def is_fr_test(test):
if test.label and test.banners and test.campaign:
is_chapter = re.search(config.fr_chapter_test, test.banners[0])
if is_chapter:
log.debug("Determined test {title} belongs to a chapter".format(title=test.label))
else:
log.debug("Determined test {title} belongs to Fundraising".format(title=test.label))
return not is_chapter
log.warn("missing data for test {title}".format(title=test.label))
def is_fr_test(test):
if test.label and test.banners and test.campaign:
is_chapter = re.search(config.fr_chapter_test, test.banners[0])
if is_chapter:
log.debug("Determined test {title} belongs to a chapter".format(
title=test.label))
else:
log.debug("Determined test {title} belongs to Fundraising".format(
title=test.label))
return not is_chapter
log.warn("missing data for test {title}".format(title=test.label))
def __init__(self,
label=None,
type="",
campaign=None,
banners=None,
start=None,
end=None,
disabled=False,
**ignore):
for key in config.ignored_columns:
if key in ignore:
ignore.pop(key)
if ignore:
log.warn("ignoring columns: {columns}".format(
columns=", ".join(ignore.keys())))
self.campaign = mediawiki.centralnotice.api.get_campaign(campaign)
if not self.campaign:
log.warn("no such campaign '{campaign}'".format(campaign=campaign))
self.type = type.lower()
self.banners = []
if self.type.count('banner') > 0:
self.is_banner_test = True
if banners:
if hasattr(banners, 'strip'):
banners = [s.strip() for s in banners.split(",")]
self.banners = banners
else:
if self.campaign['banners']:
self.banners = self.campaign['banners'].keys()
#self.variations = [ FrTestVariation(banner=name) for name in self.banners ]
self.is_country_test = (self.type.count('country') > 0)
self.is_lp_test = (self.type.count('lp') > 0)
self.start_time = start
self.end_time = end
self.label = label
if not self.label:
# FIXME
self.label = campaign
self.enabled = not disabled
self.results = []
def update_gdoc_results(doc=None, results=[]):
log.info("Updating results in {url}".format(url=doc))
doc = Spreadsheet(doc=doc)
existing = list(doc.get_all_rows())
def find_matching_cases(criteria):
matching = []
def fuzzy_compare_row(row, criteria):
if not row:
return False
if criteria['banner'] == row['banner'] and criteria[
'campaign'] == row['campaign'] and criteria[
'start'] == row['start']:
return True
for n, row in enumerate(existing, 1):
if fuzzy_compare_row(row, criteria):
matching.append(n)
return matching
for result in results:
if not result:
continue
matching = find_matching_cases(result['criteria'])
props = {}
props.update(result['results'])
props.update(result['criteria'])
if len(matching) == 0:
doc.append_row(props)
else:
if len(matching) > 1:
log.warn(
"more than one result row {match} matches criteria: {criteria}"
.format(match=matching, criteria=result['criteria']))
index = matching[-1]
log.debug("updating row {rownum} with {banner}".format(
rownum=index, banner=result['criteria']['banner']))
doc.update_row(props, index=index)
def load_results(self):
if self.is_banner_test and self.banners:
cases = []
for name in self.banners:
test_case = self.get_case(
campaign=self.campaign['name'],
banner=name,
languages=self.campaign['languages'],
countries=self.campaign['countries'],
)
cases.append(test_case)
self.results.extend(get_banner_results(cases))
if self.is_country_test:
# results = [ calculate_result(country=code) for code in campaign['countries'] ]
# self.results.extend(results)
log.warn("country test type not implemented")
if self.is_lp_test:
log.warn("LP test type not implemented")
def load_results(self):
if self.is_banner_test and self.banners:
cases = []
for name in self.banners:
test_case = self.get_case(
campaign=self.campaign['name'],
banner=name,
languages=self.campaign['languages'],
countries=self.campaign['countries'],
)
cases.append(test_case)
self.results.extend(get_banner_results(cases))
if self.is_country_test:
#results = [ calculate_result(country=code) for code in campaign['countries'] ]
#self.results.extend(results)
log.warn("country test type not implemented")
if self.is_lp_test:
log.warn("LP test type not implemented")
def __init__(self, label=None, type="", campaign=None, banners=None, start=None, end=None, disabled=False, **ignore):
for key in config.ignored_columns:
if key in ignore:
ignore.pop(key)
if ignore:
log.warn("ignoring columns: {columns}".format(columns=", ".join(ignore.keys())))
self.campaign = mediawiki.centralnotice.api.get_campaign(campaign)
if not self.campaign:
log.warn("no such campaign '{campaign}'".format(campaign=campaign))
self.type = type.lower()
self.banners = []
if self.type.count('banner') > 0:
self.is_banner_test = True
if banners:
if hasattr(banners, 'strip'):
banners = [s.strip() for s in banners.split(",")]
self.banners = banners
else:
if self.campaign['banners']:
self.banners = self.campaign['banners'].keys()
# self.variations = [ FrTestVariation(banner=name) for name in self.banners ]
self.is_country_test = (self.type.count('country') > 0)
self.is_lp_test = (self.type.count('lp') > 0)
self.start_time = start
self.end_time = end
self.label = label
if not self.label:
# FIXME
self.label = campaign
self.enabled = not disabled
self.results = []
def reviewBatch(self):
'''For each new contact, find the oldest contact with the same email address.'''
matchDescription = EmailMatch("Exact match").json()
self.contactCache.fetch()
for contact in self.contactCache.contacts:
if contact['email']:
query = db.Query()
query.columns = [
'MIN(contact_id) AS contact_id',
]
query.tables = [
'civicrm_email',
]
query.where.extend([
'email = %(email)s',
'contact_id < %(new_id)s',
])
query.group_by.extend([
'email',
])
query.params = {
'new_id': contact['id'],
'email': contact['email'],
}
result = db.get_db().execute(query)
if result:
for row in result:
ReviewQueue.addMatch(self.job_id, row['contact_id'], contact['id'], Autoreview.REC_DUP, matchDescription)
ReviewQueue.tag(contact['id'], QuickAutoreview.QUICK_REVIEWED)
if not self.contactCache.contacts:
log.warn("Searched an empty batch of contacts!")
else:
last_seen = self.contactCache.contacts[-1]['id']
log.info("End of batch. Last contact scanned was ID {id}".format(id=last_seen))
def pull():
'''Pull down new remote files'''
config = process.globals.get_config()
# Check against both unprocessed and processed files to find new remote files
local_paths = [
config.incoming_path,
config.archive_path,
]
if hasattr(config, 'extra_paths'):
local_paths.extend(config.extra_paths)
local_files = walk_files(local_paths)
remote = Client()
remote_files = remote.ls(config.sftp.remote_root)
empty_failures = []
for filename in remote_files:
if filename in local_files:
log.info("Skipping already downloaded file {filename}".format(filename=filename))
continue
log.info("Downloading file {filename}".format(filename=filename))
dest_path = os.path.join(config.incoming_path, filename)
remote.get(os.path.join(config.sftp.remote_root, filename), dest_path)
# Assert that the file is not empty
if os.path.getsize(dest_path) == 0:
os.unlink(dest_path)
empty_failures.append(filename)
log.warn("Stupid file was empty, removing locally: {path}".format(path=dest_path))
if empty_failures:
log.error("The following files were empty, please contact your provider: {failures}".format(failures=", ".join(empty_failures)))
if hasattr(config, 'panic_on_empty') and config.panic_on_empty:
raise RuntimeError("Stupid files did not download correctly.")
def update_gdoc_results(doc=None, results=[]):
log.info("Updating results in {url}".format(url=doc))
doc = Spreadsheet(doc=doc)
existing = list(doc.get_all_rows())
def find_matching_cases(criteria):
matching = []
def fuzzy_compare_row(row, criteria):
if not row:
return False
if criteria['banner'] == row['banner'] and criteria['campaign'] == row['campaign'] and criteria['start'] == row['start']:
return True
for n, row in enumerate(existing, 1):
if fuzzy_compare_row(row, criteria):
matching.append(n)
return matching
for result in results:
if not result:
continue
matching = find_matching_cases(result['criteria'])
props = {}
props.update(result['results'])
props.update(result['criteria'])
if len(matching) == 0:
doc.append_row(props)
else:
if len(matching) > 1:
log.warn("more than one result row {match} matches criteria: {criteria}".format(match=matching, criteria=result['criteria']))
index = matching[-1]
log.debug("updating row {rownum} with {banner}".format(rownum=index, banner=result['criteria']['banner']))
doc.update_row(props, index=index)
def kill_connection(self):
log.warn('Query taking too long - killing connection {}'.format(self.connection_id))
killerConnection = Dbi.connect(**self.connectionArgs)
cursor = killerConnection.cursor()
cursor.execute('KILL CONNECTION {}'.format(self.connection_id))
killerConnection.close()
