def _setup_config_manager(self):
return get_config_manager_for_crontabber(
jobs=(
'socorro.cron.jobs.fetch_adi_from_hive'
'.FAKEFetchADIFromHiveCronApp|1d'
),
)
def _setup_config_manager(
self,
delay_between_emails=7,
exacttarget_user='',
exacttarget_password='',
restrict_products=['WaterWolf'],
email_template='socorro_dev_test'
):
overrides = {
'crontabber.class-AutomaticEmailsCronApp.delay_between_emails':
delay_between_emails,
'crontabber.class-AutomaticEmailsCronApp.exacttarget_user':
exacttarget_user,
'crontabber.class-AutomaticEmailsCronApp.exacttarget_password':
exacttarget_password,
'crontabber.class-AutomaticEmailsCronApp.restrict_products':
restrict_products,
'crontabber.class-AutomaticEmailsCronApp.email_template':
email_template,
'crontabber.class-AutomaticEmailsCronApp.elasticsearch.'
'elasticsearch_index':
'socorro_integration_test',
'crontabber.class-AutomaticEmailsCronApp.elasticsearch.'
'elasticsearch_emails_index':
'socorro_integration_test_emails',
'crontabber.class-AutomaticEmailsCronApp.elasticsearch.'
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.automatic_emails'
'.AutomaticEmailsCronApp|1h',
overrides=overrides
)
def _setup_config_manager(
self,
product='WaterWolf',
output_path=None,
public_output_path=None,
**kwargs
#version=None,
#private_user='******',
#private_server='secure.mozilla.org',
#private_location='/var/logs/',
#public_user='******',
#public_server='ftp.mozilla.org',
#public_location='/tmp/%Y%m%d/',
):
if output_path is None:
output_path = self.tempdir
if public_output_path is None:
public_output_path = self.tempdir
extra_value_source = {
'crontabber.class-DailyURLCronApp.output_path':
output_path,
'crontabber.class-DailyURLCronApp.public_output_path':
public_output_path,
'crontabber.class-DailyURLCronApp.product':
product,
}
for key, value in kwargs.items():
extra_value_source['crontabber.class-DailyURLCronApp.%s' %
key] = value
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.daily_url.DailyURLCronApp|1d',
overrides=extra_value_source)
def _setup_config_manager(self,
delay_between_emails=7,
exacttarget_user='',
exacttarget_password='',
restrict_products=['WaterWolf'],
email_template='socorro_dev_test'):
overrides = {
'crontabber.class-AutomaticEmailsCronApp.delay_between_emails':
delay_between_emails,
'crontabber.class-AutomaticEmailsCronApp.exacttarget_user':
exacttarget_user,
'crontabber.class-AutomaticEmailsCronApp.exacttarget_password':
exacttarget_password,
'crontabber.class-AutomaticEmailsCronApp.restrict_products':
restrict_products,
'crontabber.class-AutomaticEmailsCronApp.email_template':
email_template,
'crontabber.class-AutomaticEmailsCronApp.elasticsearch.'
'elasticsearch_index':
'socorro_integration_test',
'crontabber.class-AutomaticEmailsCronApp.elasticsearch.'
'elasticsearch_emails_index':
'socorro_integration_test_emails',
'crontabber.class-AutomaticEmailsCronApp.elasticsearch.'
'elasticsearch_timeout':
5,
'backoff_delays': [1],
}
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.automatic_emails'
'.AutomaticEmailsCronApp|1h',
overrides=overrides)
def _setup_config_manager(self, days_into_past):
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.bugzilla.BugzillaCronApp|1d',
overrides={
'crontabber.class-BugzillaCronApp.days_into_past': days_into_past,
}
)
def _setup_config_manager(
self,
product='WaterWolf',
output_path=None,
public_output_path=None,
**kwargs
#version=None,
#private_user='******',
#private_server='secure.mozilla.org',
#private_location='/var/logs/',
#public_user='******',
#public_server='ftp.mozilla.org',
#public_location='/tmp/%Y%m%d/',
):
if output_path is None:
output_path = self.tempdir
if public_output_path is None:
public_output_path = self.tempdir
extra_value_source = {
'crontabber.class-DailyURLCronApp.output_path': output_path,
'crontabber.class-DailyURLCronApp.public_output_path':
public_output_path,
'crontabber.class-DailyURLCronApp.product': product,
}
for key, value in kwargs.items():
extra_value_source['crontabber.class-DailyURLCronApp.%s' %
key] = value
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.daily_url.DailyURLCronApp|1d',
overrides=extra_value_source)
def _setup_config_manager(self):
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.ftpscraper.FTPScraperCronApp|1d',
overrides={
'crontabber.class-FTPScraperCronApp.products': 'mobile',
}
)
def _setup_simple_config(
self,
common_email_domains=None,
restrict_products=['WaterWolf'],
email_template='socorro_dev_test'
):
required_config = automatic_emails.AutomaticEmailsCronApp \
.get_required_config()
overrides = {
'delay_between_emails': 7,
'exacttarget_user': '',
'exacttarget_password': '',
'restrict_products': restrict_products,
'email_template': email_template,
'elasticsearch.elasticsearch_index': 'socorro_integration_test',
'elasticsearch.elasticsearch_emails_index':
'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
if common_email_domains:
overrides['common_email_domains'] = common_email_domains
return get_config_manager_for_crontabber(
more_definitions=required_config,
overrides=overrides
)
def _setup_config_manager(self, overrides=None):
return get_config_manager_for_crontabber(
jobs=(
'socorro.cron.jobs.fetch_adi_from_hive'
'.FetchADIFromHiveCronApp|1d'
),
overrides=overrides,
)
def _setup_config_manager(self, overrides=None):
return get_config_manager_for_crontabber(
jobs=(
'socorro.cron.jobs.fetch_adi_from_hive'
'.FetchADIFromHiveCronApp|1d'
),
overrides=overrides,
)
def _setup_config_manager(self):
super(TestWeeklyReportsPartitions, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs=(
'socorro.cron.jobs.weekly_reports_partitions.'
'WeeklyReportsPartitionsCronApp|1d'
),
)
def _setup_config_manager(self):
super(TestWeeklyReportsPartitions, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs=(
'socorro.cron.jobs.weekly_reports_partitions.'
'WeeklyReportsPartitionsCronApp|1d'
),
)
def _setup_config_manager(self):
self.rabbit_queue_mocked = Mock()
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.reprocessingjobs.ReprocessingJobsApp|5m',
overrides={
'crontabber.class-ReprocessingJobsApp.queuing.queuing_class':
self.rabbit_queue_mocked
})
def _setup_config_manager(self):
return get_config_manager_for_crontabber(
jobs=('socorro.cron.jobs.featured_versions_automatic'
'.FeaturedVersionsAutomaticCronApp|1d'),
overrides={
'crontabber.class-FeaturedVersionsAutomaticCronApp'
'.api_endpoint_url':
('https://example.com/{product}_versions.json'),
})
def _setup_config_manager(self, api_endpoint_url='https://whatever.urg'):
return get_config_manager_for_crontabber(
jobs=('socorro.cron.jobs.featured_versions_sync'
'.FeaturedVersionsSyncCronApp|1d'),
overrides={
'crontabber.class-FeaturedVersionsSyncCronApp'
'.api_endpoint_url':
api_endpoint_url,
})
def _setup_config_manager(self, days_to_keep=None):
super(TestCleanRawADICronApp, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs=('socorro.cron.jobs.clean_raw_adi.'
'CleanRawADICronApp|1d'),
overrides={
'crontabber.class-CleanRawADICronApp'
'.days_to_keep': days_to_keep
},
)
def _setup_config_manager(self, days_to_keep=None):
super(TestMissingSymbolsCronApp, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs=('socorro.cron.jobs.missingsymbols.MissingSymbolsCronApp|1d'),
overrides={
'crontabber.class-MissingSymbolsCronApp'
'.boto_class':
self.mock_boto_class
},
)
def _setup_config_manager(self):
_super = super(TestSymbolsUnpack, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.symbolsunpack.SymbolsUnpackCronApp|1h',
overrides={
'crontabber.class-SymbolsUnpackCronApp.source_directory':
self.temp_source_directory,
'crontabber.class-SymbolsUnpackCronApp.destination_directory':
self.temp_destination_directory
})
def _setup_config_manager_firefox(self):
# Set a completely bogus looking base_url so it can never
# accidentally work if the network request mocking leaks
base_url = 'https://archive.muzilla.hej/pub/'
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.ftpscraper.FTPScraperCronApp|1d',
overrides={
'crontabber.class-FTPScraperCronApp.products': 'firefox',
'crontabber.class-FTPScraperCronApp.base_url': base_url,
})
def _setup_config_manager(self):
self.rabbit_queue_mocked = Mock()
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.reprocessingjobs.ReprocessingJobsApp|5m',
overrides={
'crontabber.class-ReprocessingJobsApp.queuing.queuing_class':
self.rabbit_queue_mocked
}
)
def _setup_config_manager(self):
overrides = {
'crontabber.class-ModulelistCronApp.pig_classpath':
'/some/place',
'crontabber.class-ModulelistCronApp.output_file':
'/some/other/place/%(date)s-modulelist.txt'
}
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.modulelist.ModulelistCronApp|1d',
overrides=overrides)
def _setup_config_manager(self):
overrides = {
'crontabber.class-ModulelistCronApp.pig_classpath': '/some/place',
'crontabber.class-ModulelistCronApp.output_file':
'/some/other/place/%(date)s-modulelist.txt'
}
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.modulelist.ModulelistCronApp|1d',
overrides=overrides
)
def _setup_config_manager(self):
_super = super(TestSymbolsUnpack, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.symbolsunpack.SymbolsUnpackCronApp|1h',
overrides={
'crontabber.class-SymbolsUnpackCronApp.source_directory':
self.temp_source_directory,
'crontabber.class-SymbolsUnpackCronApp.destination_directory':
self.temp_destination_directory
}
)
def _setup_config_manager_firefox(self):
# Set a completely bogus looking base_url so it can never
# accidentally work if the network request mocking leaks
base_url = 'https://archive.muzilla.hej/pub/'
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.ftpscraper.FTPScraperCronApp|1d',
overrides={
'crontabber.class-FTPScraperCronApp.products': 'firefox',
'crontabber.class-FTPScraperCronApp.base_url': base_url,
}
)
def _setup_config_manager(self, api_endpoint_url='https://whatever.urg'):
return get_config_manager_for_crontabber(
jobs=(
'socorro.cron.jobs.featured_versions_sync'
'.FeaturedVersionsSyncCronApp|1d'
),
overrides={
'crontabber.class-FeaturedVersionsSyncCronApp'
'.api_endpoint_url': api_endpoint_url,
}
)
def _setup_config_manager(self, days_to_keep=None):
super(TestCleanMissingSymbolsCronApp, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs=('socorro.cron.jobs.clean_missing_symbols.'
'CleanMissingSymbolsCronApp|1d'),
overrides={
'crontabber.class-CleanMissingSymbolsCronApp'
'.days_to_keep':
days_to_keep
},
)
def _setup_config_manager(self, days_to_keep=None):
super(TestMissingSymbolsCronApp, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs=(
'socorro.cron.jobs.missingsymbols.MissingSymbolsCronApp|1d'
),
overrides={
'crontabber.class-MissingSymbolsCronApp'
'.boto_class': self.mock_boto_class
},
)
def get_standard_config(cls):
"""this method overrides the crontabber version of the same name.
It is not used by Socorro clients directly, but the base crontabber
class uses this method during setup. By overriding the implementation
here, we get a default Socorro configuration file with many of the
standard Socorro defaults already in place: logging, executors, etc.
This allows the bootstraping of the integration tests to participate
fully with the environment variables, commandline arguments, and
configurations files that the Socorro installation/test system of
Makefiles and shell scripts offers"""
config = get_config_manager_for_crontabber().get_config()
return config
def _setup_storage_config(self):
required_config = ElasticSearchCrashStorage.get_required_config()
overrides = {
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return get_config_manager_for_crontabber(
more_definitions=required_config, overrides=overrides)
def get_standard_config(self):
"""this method overrides the crontabber version of the same name.
It is not used by Socorro clients directly, but the base crontabber
class uses this method during setup. By overriding the implementation
here, we get a default Socorro configuration file with many of the
standard Socorro defaults already in place: logging, executors, etc.
This allows the bootstraping of the integration tests to participate
fully with the environment variables, commandline arguments, and
configurations files that the Socorro installation/test system of
Makefiles and shell scripts offers"""
config = get_config_manager_for_crontabber().get_config()
return config
def _setup_config_manager(self, days_to_keep=None):
super(TestCleanMissingSymbolsCronApp, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs=(
'socorro.cron.jobs.clean_missing_symbols.'
'CleanMissingSymbolsCronApp|1d'
),
overrides={
'crontabber.class-CleanMissingSymbolsCronApp'
'.days_to_keep': days_to_keep
},
)
def _setup_config_manager(self):
queue_mock = Mock()
queue_mock.return_value.return_value = MagicMock()
queue_mock.return_value.return_value.queue_status_standard \
.method.message_count = 1
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.serverstatus.ServerStatusCronApp|5m',
overrides={
'crontabber.class-ServerStatusCronApp.queuing.queuing_class':
queue_mock
})
def _setup_config_manager(self, days_to_keep=None):
super(TestCleanRawADICronApp, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs=(
'socorro.cron.jobs.clean_raw_adi.'
'CleanRawADICronApp|1d'
),
overrides={
'crontabber.class-CleanRawADICronApp'
'.days_to_keep': days_to_keep
},
)
def _setup_config_manager(self):
return get_config_manager_for_crontabber(
jobs=(
'socorro.cron.jobs.featured_versions_automatic'
'.FeaturedVersionsAutomaticCronApp|1d'
),
overrides={
'crontabber.class-FeaturedVersionsAutomaticCronApp'
'.api_endpoint_url': (
'https://example.com/{product}_versions.json'
),
}
)
def _setup_config_manager(self):
queue_mock = Mock()
queue_mock.return_value.return_value = MagicMock()
queue_mock.return_value.return_value.queue_status_standard \
.method.message_count = 1
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.serverstatus.ServerStatusCronApp|5m',
overrides={
'crontabber.class-ServerStatusCronApp.queuing.queuing_class':
queue_mock
}
)
def _setup_storage_config(self):
required_config = ElasticSearchCrashStorage.get_required_config()
overrides = {
'elasticsearch_index': 'socorro_integration_test',
'elasticsearch_emails_index': 'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return get_config_manager_for_crontabber(
more_definitions=required_config,
overrides=overrides
)
def _setup_config_manager(self, **kwargs):
kwargs.setdefault('training_data_length', 10)
kwargs.setdefault('data_bin_length', 10)
kwargs.setdefault('model', 'SlopeBased')
# This is the one we modified
kwargs.setdefault('min_count', 5)
evs = {}
for k, v in kwargs.iteritems():
evs['crontabber.class-SuspiciousCrashesApp.' + k] = v
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.suspicious_crashes'
'.SuspiciousCrashesApp|1d',
overrides=evs)
def _setup_config_manager(self, **kwargs):
kwargs.setdefault('training_data_length', 10)
kwargs.setdefault('data_bin_length', 10)
kwargs.setdefault('model', 'SlopeBased')
# This is the one we modified
kwargs.setdefault('min_count', 5)
evs = {}
for k, v in kwargs.iteritems():
evs['crontabber.class-SuspiciousCrashesApp.' + k] = v
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.suspicious_crashes'
'.SuspiciousCrashesApp|1d',
overrides=evs
)
def _setup_config_manager(self, days_into_past):
PST = tz.gettz('PST8PDT')
datestring = ((datetime.datetime.now(PST) - datetime.timedelta(
days=days_into_past)).astimezone(PST).strftime('%Y-%m-%d'))
filename = os.path.join(self.tempdir, 'sample-%s.csv' % datestring)
with open(filename, 'w') as f:
f.write('\n'.join(SAMPLE_CSV))
query = 'file://' + filename.replace(datestring, '%s')
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.bugzilla.BugzillaCronApp|1d',
overrides={
'crontabber.class-BugzillaCronApp.query': query,
'crontabber.class-BugzillaCronApp.days_into_past':
days_into_past,
})
def _setup_config_manager(self, days_into_past):
PST = tz.gettz('PST8PDT')
datestring = ((datetime.datetime.now(PST) -
datetime.timedelta(days=days_into_past))
.astimezone(PST)
.strftime('%Y-%m-%d'))
filename = os.path.join(self.tempdir, 'sample-%s.csv' % datestring)
with open(filename, 'w') as f:
f.write('\n'.join(SAMPLE_CSV))
query = 'file://' + filename.replace(datestring, '%s')
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.bugzilla.BugzillaCronApp|1d',
overrides={
'crontabber.class-BugzillaCronApp.query': query,
'crontabber.class-BugzillaCronApp.days_into_past': days_into_past,
}
)
def _setup_test_mode_config(self):
required_config = automatic_emails.AutomaticEmailsCronApp \
.get_required_config()
overrides = {
'delay_between_emails': 7,
'exacttarget_user': '',
'exacttarget_password': '',
'restrict_products': ['WaterWolf'],
'test_mode': True,
'email_template': 'socorro_dev_test',
'elasticsearch.elasticsearch_index': 'socorro_integration_test',
'elasticsearch.elasticsearch_emails_index':
'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return get_config_manager_for_crontabber(
more_definitions=required_config, overrides=overrides)
def _setup_test_mode_config(self):
required_config = automatic_emails.AutomaticEmailsCronApp \
.get_required_config()
overrides = {
'delay_between_emails': 7,
'exacttarget_user': '',
'exacttarget_password': '',
'restrict_products': ['WaterWolf'],
'test_mode': True,
'email_template': 'socorro_dev_test',
'elasticsearch.elasticsearch_index':
'socorro_integration_test',
'elasticsearch.elasticsearch_emails_index':
'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
return get_config_manager_for_crontabber(
more_definitions=required_config,
overrides=overrides
)
def _setup_simple_config(self,
common_email_domains=None,
restrict_products=['WaterWolf'],
email_template='socorro_dev_test'):
required_config = automatic_emails.AutomaticEmailsCronApp \
.get_required_config()
overrides = {
'delay_between_emails': 7,
'exacttarget_user': '',
'exacttarget_password': '',
'restrict_products': restrict_products,
'email_template': email_template,
'elasticsearch.elasticsearch_index': 'socorro_integration_test',
'elasticsearch.elasticsearch_emails_index':
'socorro_integration_test_emails',
'elasticsearch_timeout': 5,
'backoff_delays': [1],
}
if common_email_domains:
overrides['common_email_domains'] = common_email_domains
return get_config_manager_for_crontabber(
more_definitions=required_config, overrides=overrides)
def get_standard_config(self):
return get_config_manager_for_crontabber().get_config()
def _setup_config_manager(self):
super(TestTruncatePartitions, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.truncate_partitions.'
'TruncatePartitionsCronApp|1m',
)
def _setup_config_manager(self):
super(TestDropOldPartitions, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.drop_old_partitions.'
'DropOldPartitionsCronApp|1m', )
def _setup_config_manager(self):
_super = super(TestDropOldPartitions, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.drop_old_partitions.'
'DropOldPartitionsCronApp|1m',
)
def _setup_config_manager(self, jobs):
return get_config_manager_for_crontabber(jobs=jobs, )
def _setup_config_manager(self):
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.upload_crash_report_json_schema.'
'UploadCrashReportJSONSchemaCronApp|30d',
)
def _setup_config_manager(self, jobs):
return get_config_manager_for_crontabber(
jobs=jobs,
)
def _setup_config_manager(self):
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.elasticsearch_cleanup.'
'ElasticsearchCleanupCronApp|30d', )
def _setup_config_manager(self):
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.elasticsearch_cleanup.'
'ElasticsearchCleanupCronApp|30d',
)
def _setup_config_manager(self, api_endpoint_url="https://whatever.urg"):
return get_config_manager_for_crontabber(
jobs=("socorro.cron.jobs.featured_versions_sync" ".FeaturedVersionsSyncCronApp|1d"),
overrides={"crontabber.class-FeaturedVersionsSyncCronApp" ".api_endpoint_url": api_endpoint_url},
)
def _setup_config_manager(self):
super(TestTruncatePartitions, self)._setup_config_manager
return get_config_manager_for_crontabber(
jobs='socorro.cron.jobs.truncate_partitions.'
'TruncatePartitionsCronApp|1m', )
