def __init__(self, options):
self._upload_age_limit = options.age_to_upload
self._delete_age_limit = options.age_to_delete
if options.delete_only:
self._gs_offloader = FakeGSOffloader()
else:
self.gs_uri = utils.get_offload_gsuri()
logging.debug('Offloading to: %s', self.gs_uri)
multiprocessing = False
if options.multiprocessing:
multiprocessing = True
elif options.multiprocessing is None:
multiprocessing = GS_OFFLOADER_MULTIPROCESSING
logging.info('Offloader multiprocessing is set to:%r',
multiprocessing)
console_client = None
if (cloud_console_client
and cloud_console_client.is_cloud_notification_enabled()):
console_client = cloud_console_client.PubSubBasedClient()
self._gs_offloader = GSOffloader(self.gs_uri, multiprocessing,
self._delete_age_limit,
console_client)
classlist = [
job_directories.SwarmingJobDirectory,
]
if options.process_hosts_only or options.process_all:
classlist.append(job_directories.SpecialJobDirectory)
if not options.process_hosts_only:
classlist.append(job_directories.RegularJobDirectory)
self._jobdir_classes = classlist
assert self._jobdir_classes
self._processes = options.parallelism
self._open_jobs = {}
self._pusub_topic = None
self._offload_count_limit = 3
def __init__(self, options):
self._pubsub_topic = None
self._upload_age_limit = options.age_to_upload
self._delete_age_limit = options.age_to_delete
if options.delete_only:
self._offload_func = delete_files
else:
self.gs_uri = utils.get_offload_gsuri()
logging.debug('Offloading to: %s', self.gs_uri)
multiprocessing = False
if options.multiprocessing:
multiprocessing = True
elif options.multiprocessing is None:
multiprocessing = GS_OFFLOADER_MULTIPROCESSING
logging.info(
'Offloader multiprocessing is set to:%r', multiprocessing)
if options.pubsub_topic_for_job_upload:
self._pubsub_topic = options.pubsub_topic_for_job_upload
elif _PUBSUB_ENABLED:
self._pubsub_topic = _PUBSUB_TOPIC
self._offload_func = get_offload_dir_func(
self.gs_uri, multiprocessing, self._delete_age_limit,
self._pubsub_topic)
classlist = []
if options.process_hosts_only or options.process_all:
classlist.append(job_directories.SpecialJobDirectory)
if not options.process_hosts_only:
classlist.append(job_directories.RegularJobDirectory)
self._jobdir_classes = classlist
assert self._jobdir_classes
self._processes = options.parallelism
self._open_jobs = {}
self._pusub_topic = None
def test_get_default_lab_offload_gsuri(self):
"""Test default lab offload gsuri ."""
self.god.mock_up(utils.CONFIG, 'CONFIG')
self.god.stub_function_to_return(lsbrelease_utils, 'is_moblab', False)
self.assertEqual(utils.DEFAULT_OFFLOAD_GSURI,
utils.get_offload_gsuri())
self.god.check_playback()
def test_get_moblab_offload_gsuri(self):
"""Test default lab offload gsuri ."""
self.god.mock_up(utils.CONFIG, 'CONFIG')
self.god.stub_function_to_return(utils, 'is_moblab', True)
self.god.stub_function_to_return(utils, 'get_interface_mac_address',
'test_mac')
self.god.stub_function_to_return(utils, 'get_moblab_id', 'test_id')
gsuri = '%s%s/%s/' % (utils.DEFAULT_OFFLOAD_GSURI, 'test_mac',
'test_id')
self.assertEqual(gsuri, utils.get_offload_gsuri())
self.god.check_playback()
def get_gs_url(cls, logdir):
"""Return a GS URL to job results.
The URL is constructed from a base URL determined by the
global config, plus the relative path of the job's log
directory.
@param logdir Relative path of the results log directory.
@return A URL to the requested results log.
"""
return os.path.join(utils.get_offload_gsuri(), logdir)
def test_get_default_moblab_offload_gsuri(self):
self.god.mock_up(utils.CONFIG, 'CONFIG')
self.god.stub_function_to_return(utils, 'is_moblab', True)
utils.CONFIG.get_config_value.expect_call(
'CROS',
'image_storage_server').and_return(self._IMAGE_STORAGE_SERVER)
self.god.stub_function_to_return(utils, 'get_interface_mac_address',
'test_mac')
self.god.stub_function_to_return(utils, 'get_moblab_id', 'test_id')
expected_gsuri = '%sresults/%s/%s/' % (self._IMAGE_STORAGE_SERVER,
'test_mac', 'test_id')
cached_gsuri = utils.DEFAULT_OFFLOAD_GSURI
utils.DEFAULT_OFFLOAD_GSURI = None
gsuri = utils.get_offload_gsuri()
utils.DEFAULT_OFFLOAD_GSURI = cached_gsuri
self.assertEqual(expected_gsuri, gsuri)
self.god.check_playback()
def __init__(self, options):
if options.delete_only:
self._offload_func = delete_files
else:
self.gs_uri = utils.get_offload_gsuri()
logging.debug('Offloading to: %s', self.gs_uri)
self._offload_func = get_offload_dir_func(self.gs_uri,
options.multiprocessing)
classlist = []
if options.process_hosts_only or options.process_all:
classlist.append(job_directories.SpecialJobDirectory)
if not options.process_hosts_only:
classlist.append(job_directories.RegularJobDirectory)
self._jobdir_classes = classlist
assert self._jobdir_classes
self._processes = options.parallelism
self._age_limit = options.days_old
self._open_jobs = {}
self._next_report_time = time.time()
