def main():
parser = get_parser()
(options, _) = parser.parse_args()
level = logging.DEBUG if options.verbose else logging.INFO
logging.basicConfig(
format=u'%(asctime)s %(levelname)s %(name)s %(message)s',
level=level)
# Lower the warning levels of a number of loggers
for logger_name in ['paramiko.transport', 'paramiko.transport.sftp',
'requests.packages.urllib3.connectionpool',
'swiftclient']:
logging.getLogger(logger_name).setLevel(logging.WARNING)
database = db.DB(Configuration().DATABASE_URL)
queue = JobQueue(
database=database,
nodepool=NodePool(Configuration().NODEPOOL_IMAGE),
filesystem=filesystem_services.RealFilesystem(),
uploader=swift_upload.SwiftUploader(),
executor=utils.execute_command)
if options.flush:
queue.flush()
return
if options.change_ref:
change_num, patchset = options.change_ref.split('/')[-2:]
patch_details = utils.get_patchset_details(change_num, patchset)
# Verify we got the right patch back
queue.addJob(patch_details['ref'], patch_details['project'], patch_details['revision'])
return
if options.run_job:
queue.triggerJob(options.run_job)
return
if options.recheck:
for jobnum in options.recheck:
queue.recheckJob(jobnum)
return
queue.startCleanupThreads()
try:
while True:
try:
queue.postResults()
queue.processResults()
queue.triggerJobs()
Configuration().check_reload()
except Exception, e:
logging.exception(e)
# Ignore exception and try again; keeps the app polling
time.sleep(Configuration().get_int('POLL'))
except KeyboardInterrupt:
logging.info("Terminated by user")
def test_upload_failed_auth(self, mock_os, mock_pyrax):
class AuthenticationFailed(Exception):
pass
swift_upload.pyrax.exceptions.AuthenticationFailed = AuthenticationFailed
swift_upload.pyrax.set_credentials.side_effect = swift_upload.pyrax.exceptions.AuthenticationFailed
self.assertRaises(AuthenticationFailed,
swift_upload.SwiftUploader().upload, 'a', 'b')
def test_upload_no_files(self, mock_one_file, mock_os, mock_pyrax):
mock_os.listdir.return_value = []
mock_container = mock.Mock()
mock_pyrax.cloudfiles.create_container.return_value = mock_container
mock_container.cdn_uri = 'uri'
result = swift_upload.SwiftUploader().upload(['localdir'], 'prefix')
self.assertEqual(result, 'uri/prefix/index.html')
# Nothing should have been uploaded
self.assertEqual(0, mock_one_file.call_count)
def test_upload_subdir(self, mock_one_file, mock_os_stat, mock_os_listdir,
mock_pyrax):
mock_os_listdir.side_effect = [['subdir'], ['file']]
mock_os_stat.side_effect = lambda x: self.mock_stat_dir if 'dir' in os.path.split(
x)[-1] else self.mock_stat_file
mock_container = mock.Mock()
mock_pyrax.cloudfiles.create_container.return_value = mock_container
mock_container.cdn_uri = 'uri'
result = swift_upload.SwiftUploader().upload(['localdir'], 'prefix')
self.assertEqual(result, 'uri/prefix/index.html')
expected_calls = [
mock.call(mock_container, 'localdir/subdir/file',
'prefix/localdir/subdir/file')
]
mock_one_file.assert_has_calls(expected_calls)
def test_upload_one_happy_path(self, mock_pyrax):
mock_pyrax.utils.get_checksum.return_value = 'calc_checksum'
mock_container = mock.Mock()
source = 'source.txt'
target = 'target.txt'
mock_obj = mock.Mock()
mock_obj.etag = 'calc_checksum'
mock_container.upload_file.return_value = mock_obj
swift_upload.SwiftUploader().upload_one_file(mock_container, source,
target)
mock_container.upload_file.assert_called_with(
'source.txt',
'target.txt',
etag='calc_checksum',
content_encoding=None,
content_type='text/plain')
def test_upload_run_tests_first(self, mock_one_file, mock_os_stat,
mock_os_listdir, mock_pyrax):
mock_os_stat.side_effect = lambda x: self.mock_stat_dir if 'dir' in os.path.split(
x)[-1] else self.mock_stat_file
mock_container = mock.Mock()
mock_pyrax.cloudfiles.create_container.return_value = mock_container
mock_container.cdn_uri = 'uri'
result = swift_upload.SwiftUploader().upload(
['b', 'c', 'run_tests.log', 'a'], 'prefix')
self.assertEqual(result, 'uri/prefix/index.html')
expected_calls = [
mock.call(mock_container, 'run_tests.log', 'prefix/run_tests.log')
]
expected_calls.append(mock.call(mock_container, 'a', 'prefix/a'))
expected_calls.append(mock.call(mock_container, 'b', 'prefix/b'))
expected_calls.append(mock.call(mock_container, 'c', 'prefix/c'))
mock_one_file.assert_has_calls(expected_calls)
def test_upload_fails(self, mock_get_int, mock_pyrax):
mock_pyrax.utils.get_checksum.return_value = 'calc_checksum'
mock_container = mock.Mock()
mock_get_int.return_value = 0
source = 'source.txt'
target = 'target.txt'
mock_obj_fail = mock.Mock()
mock_obj_fail.etag = 'bad_checksum'
mock_container.upload_file.return_value = mock_obj_fail
self.assertRaises(swift_upload.UploadException,
swift_upload.SwiftUploader().upload_one_file,
mock_container, source, target)
expected = mock.call('source.txt',
'target.txt',
etag='calc_checksum',
content_encoding=None,
content_type='text/plain')
mock_container.upload_file.assert_has_calls([expected])
def test_upload_html(self, mock_one_file, mock_os_stat, mock_os_listdir,
mock_pyrax):
mock_os_listdir.return_value = ['b', 'c', 'run_tests.log', 'a.txt']
mock_os_stat.side_effect = lambda x: self.mock_stat_dir if 'dir' in os.path.split(
x)[-1] else self.mock_stat_file
mock_container = mock.Mock()
mock_pyrax.cloudfiles.create_container.return_value = mock_container
mock_container.cdn_uri = 'uri'
result = swift_upload.SwiftUploader().upload(['localdir'], 'prefix')
# Rather than parse the whole HTML - just verify that we've got the filename and size
store_calls = mock_container.store_object.call_args_list
store_args, _ = store_calls[0]
self.assertEqual('prefix/localdir/index.html', store_args[0])
self.assertIn('Index of prefix', store_args[1])
self.assertIn('<a href="run_tests.log">run_tests.log</a>',
store_args[1])
self.assertIn('1.0 KiB', store_args[1])
def test_upload_subdir2(self, mock_one_file, mock_os_stat, mock_os_listdir,
mock_pyrax):
fake_dirs = {'dir1': ['filea', 'dir2'], 'dir1/dir2': ['fileb']}
mock_os_listdir.side_effect = lambda x: fake_dirs[x.strip('/')]
mock_os_stat.side_effect = lambda x: self.mock_stat_dir if 'dir' in os.path.split(
x)[-1] else self.mock_stat_file
mock_container = mock.Mock()
mock_pyrax.cloudfiles.create_container.return_value = mock_container
mock_container.cdn_uri = 'uri'
result = swift_upload.SwiftUploader().upload(['dir1', 'filec'],
'prefix')
self.assertEqual(result, 'uri/prefix/index.html')
expected_calls = [
mock.call(mock_container, 'dir1/dir2/fileb',
'prefix/dir1/dir2/fileb'),
mock.call(mock_container, 'dir1/filea', 'prefix/dir1/filea'),
mock.call(mock_container, 'filec', 'prefix/filec')
]
mock_one_file.assert_has_calls(expected_calls)
def test_upload_one_failed(self, mock_get_int, mock_pyrax):
mock_pyrax.utils.get_checksum.return_value = 'calc_checksum'
mock_container = mock.Mock()
mock_get_int.return_value = 1
source = 'source.txt'
target = 'target.txt'
mock_obj_ok = mock.Mock()
mock_obj_ok.etag = 'calc_checksum'
mock_obj_fail = mock.Mock()
mock_obj_fail.etag = 'bad_checksum'
mock_container.upload_file.side_effect = [mock_obj_fail, mock_obj_ok]
swift_upload.SwiftUploader().upload_one_file(mock_container, source,
target)
expected = mock.call('source.txt',
'target.txt',
etag='calc_checksum',
content_encoding=None,
content_type='text/plain')
mock_container.upload_file.assert_has_calls([expected, expected])