def testCTSCIPDYamlOperations(self):
with tempfile.NamedTemporaryFile('w+t') as yamlFile:
yamlFile.writelines(CIPD_DATA['yaml'])
yamlFile.flush()
yaml_data = cts_utils.CTSCIPDYaml(yamlFile.name)
self.assertEqual(CIPD_DATA['package'], yaml_data.get_package())
self.assertEqual([
CIPD_DATA['file1'], CIPD_DATA['file2'], CIPD_DATA['file3'],
CIPD_DATA['file4']
], yaml_data.get_files())
yaml_data.append_file('arch2/platform3/file5.zip')
self.assertEqual([
CIPD_DATA['file1'], CIPD_DATA['file2'], CIPD_DATA['file3'],
CIPD_DATA['file4']
] + ['arch2/platform3/file5.zip'], yaml_data.get_files())
yaml_data.remove_file(CIPD_DATA['file1'])
self.assertEqual([
CIPD_DATA['file2'], CIPD_DATA['file3'], CIPD_DATA['file4'],
'arch2/platform3/file5.zip'
], yaml_data.get_files())
with tempfile.NamedTemporaryFile() as yamlFile:
yaml_data.write(yamlFile.name)
new_yaml_contents = readfile(yamlFile.name)
self.assertEqual(
CIPD_DATA['template'] %
(CIPD_DATA['package'], CIPD_DATA['file2'], CIPD_DATA['file3'],
CIPD_DATA['file4'], 'arch2/platform3/file5.zip'), new_yaml_contents)
def testCTSCIPDYamlSanity(self):
yaml_data = cts_utils.CTSCIPDYaml(cts_utils.CIPD_PATH)
self.assertTrue(yaml_data.get_package())
self.assertTrue(yaml_data.get_files())
with tempfile.NamedTemporaryFile('w+t') as outputFile:
yaml_data.write(outputFile.name)
with open(cts_utils.CIPD_PATH) as cipdFile:
self.assertEqual(cipdFile.readlines(), outputFile.readlines())
def update_repository(self):
"""Update chromium checkout with changes for this update.
After this is called, git add -u && git commit && git cl upload
will still be needed to generate the CL.
Raises:
MissingFileError: If CIPD has not yet been staged or updated.
UncommittedChangeException: If repo files have uncommitted changes.
InconsistentFilesException: If errors are detected in staged config files.
"""
if not os.path.exists(self._version_file):
raise MissingFileError(self._version_file)
staged_yaml_path = os.path.join(self._stage_dir,
self._CIPDYaml.get_file_basename())
if not os.path.exists(staged_yaml_path):
raise MissingFileError(staged_yaml_path)
with open(self._version_file) as vf:
new_cipd_version = vf.read()
logging.info('Read in new CIPD version %s from %s',
new_cipd_version, vf.name)
repo_cipd_yaml = self._CIPDYaml.get_file_path()
for f in self._repo_helper.cipd_referrers + [repo_cipd_yaml]:
git_status = self._repo_helper.git_status(f)
if git_status:
raise UncommittedChangeException(f)
repo_cipd_package = self._repo_helper.cts_cipd_package
staged_yaml = cts_utils.CTSCIPDYaml(file_path=staged_yaml_path)
if repo_cipd_package != staged_yaml.get_package():
raise InconsistentFilesException(
'Inconsistent CTS package name, {} in {}, but {} in {}'.format(
repo_cipd_package, cts_utils.DEPS_FILE,
staged_yaml.get_package(), staged_yaml.get_file_path()))
logging.info('Updating files that reference %s under %s.',
cts_utils.CTS_DEP_PACKAGE, self._repo_root)
self._repo_helper.update_cts_cipd_rev(new_cipd_version)
logging.info('Regenerate buildbot json files under %s.',
self._repo_root)
self._repo_helper.update_testing_json()
logging.info('Copy staged %s to %s.', staged_yaml_path,
repo_cipd_yaml)
cmd_helper.RunCmd(['cp', staged_yaml_path, repo_cipd_yaml])
logging.info('Ensure CIPD CTS package at %s to the new version %s',
repo_cipd_yaml, new_cipd_version)
cts_utils.cipd_ensure(self._CIPDYaml.get_package(), new_cipd_version,
os.path.dirname(repo_cipd_yaml))
def testCTSCIPDDownload(self, run_mock):
fake_cipd = FakeCIPD()
fake_run_cmd = FakeRunCmd(cipd=fake_cipd)
run_mock.side_effect = fake_run_cmd.run_cmd
with tempfile.NamedTemporaryFile('w+t') as yamlFile,\
tempfile_ext.NamedTemporaryDirectory() as tempDir:
yamlFile.writelines(CIPD_DATA['yaml'])
yamlFile.flush()
fake_version = fake_cipd.create(yamlFile.name)
archive = cts_utils.CTSCIPDYaml(yamlFile.name)
cts_utils.cipd_download(archive, fake_version, tempDir)
self.assertEqual(CIPD_DATA['file1'],
readfile(os.path.join(tempDir, CIPD_DATA['file1'])))
self.assertEqual(CIPD_DATA['file2'],
readfile(os.path.join(tempDir, CIPD_DATA['file2'])))
def __init__(self, work_dir, repo_root):
"""Construct UpdateCTS instance.
Args:
work_dir: Directory used to download and stage cipd updates
repo_root: Repository root (e.g. /path/to/chromium/src) to base
all configuration files
"""
self._work_dir = os.path.abspath(work_dir)
self._download_dir = os.path.join(self._work_dir, 'downloaded')
self._filter_dir = os.path.join(self._work_dir, 'filtered')
self._cipd_dir = os.path.join(self._work_dir, 'cipd')
self._stage_dir = os.path.join(self._work_dir, 'staged')
self._version_file = os.path.join(self._work_dir, 'cipd_version.txt')
self._repo_root = os.path.abspath(repo_root)
helper = cts_utils.ChromiumRepoHelper(self._repo_root)
self._repo_helper = helper
self._CTSConfig = cts_utils.CTSConfig(
helper.rebase(cts_utils.TOOLS_DIR, cts_utils.CONFIG_FILE))
self._CIPDYaml = cts_utils.CTSCIPDYaml(
helper.rebase(cts_utils.TOOLS_DIR, cts_utils.CIPD_FILE))
def commit_staged_cipd(self):
"""Upload the staged CIPD files to CIPD.
Raises:
MissingDirError: If staged/ does not exist in work_dir.
InconsistentFilesException: If errors are detected in staged config files.
MissingFileExcepition: If files are missing from CTS zip files.
"""
if not os.path.isdir(self._stage_dir):
raise MissingDirError(self._stage_dir)
staged_yaml_path = os.path.join(self._stage_dir,
self._CIPDYaml.get_file_basename())
staged_yaml = cts_utils.CTSCIPDYaml(file_path=staged_yaml_path)
staged_yaml_files = staged_yaml.get_files()
if cts_utils.CTS_DEP_PACKAGE != staged_yaml.get_package():
raise InconsistentFilesException(
'Bad CTS package name in staged yaml '
'{}: {} '.format(staged_yaml_path, staged_yaml.get_package()))
for p, a in self._CTSConfig.iter_platform_archs():
cipd_zip = self._CTSConfig.get_cipd_zip(p, a)
cipd_zip_path = os.path.join(self._stage_dir, cipd_zip)
if not os.path.exists(cipd_zip_path):
raise MissingFileError(cipd_zip_path)
with zipfile.ZipFile(cipd_zip_path) as zf:
cipd_zip_contents = zf.namelist()
missing_apks = set(
self._CTSConfig.get_apks(p)) - set(cipd_zip_contents)
if missing_apks:
raise MissingFileError('%s in %s' %
(str(missing_apks), cipd_zip_path))
if cipd_zip not in staged_yaml_files:
raise InconsistentFilesException(
cipd_zip + ' missing from staged cipd.yaml file')
logging.info('Updating CIPD CTS version using %s', staged_yaml_path)
new_cipd_version = cts_utils.update_cipd_package(staged_yaml_path)
with open(self._version_file, 'w') as vf:
logging.info('Saving new CIPD version %s to %s', new_cipd_version,
vf.name)
vf.write(new_cipd_version)