def get_revisions(root_dir):
revisions = []
ensure_dir(get_candidates_path(root_dir))
for candidate in find_candidates(root_dir):
_, rev = get_candidate_info(candidate)
revisions.append(rev)
return revisions
def get_revisions(root_dir):
revisions = []
ensure_dir(get_candidates_path(root_dir))
for candidate in find_candidates(root_dir):
_, rev = get_candidate_info(candidate)
revisions.append(rev)
return revisions
def test_get_candidate_info(self):
with temp_dir() as dir_path:
candidate_dir = os.path.join(dir_path, 'candidate')
ensure_dir(candidate_dir)
for ver, rev in [['1.24.3', '2870'], ['1.24.5', '2999']]:
ver_dir = os.path.join(candidate_dir, ver)
ensure_dir(ver_dir)
make_build_var_file(ver_dir, version=ver, revision_build=rev)
rev = get_revisions(dir_path)
self.assertEqual(sorted(rev), ['2870', '2999'])
def factory(cls, workspace, log_count=1, dry_run=False):
log_dir_path = os.path.join(workspace, 'log')
ensure_dir(log_dir_path)
log_file = os.path.join(log_dir_path, 'results.log')
cmd_log_file = os.path.join(log_dir_path, 'chaos_run_list.log')
cmd_log_name = 'cmd_log'
setup_logging(log_path=log_file, log_count=log_count)
setup_logging(
log_path=cmd_log_file, log_count=log_count, name=cmd_log_name,
add_stream=False, disable_formatter=True)
chaos_monkey = ChaosMonkey.factory()
return cls(workspace, chaos_monkey, log_count, dry_run, cmd_log_name)
def factory(cls, workspace, log_count=1, dry_run=False):
log_dir_path = os.path.join(workspace, 'log')
ensure_dir(log_dir_path)
log_file = os.path.join(log_dir_path, 'results.log')
cmd_log_file = os.path.join(log_dir_path, 'chaos_run_list.log')
cmd_log_name = 'cmd_log'
setup_logging(log_path=log_file, log_count=log_count)
setup_logging(log_path=cmd_log_file,
log_count=log_count,
name=cmd_log_name,
add_stream=False,
disable_formatter=True)
chaos_monkey = ChaosMonkey.factory()
return cls(workspace, chaos_monkey, log_count, dry_run, cmd_log_name)
def hosted_environment(system_client, log_dir, suffix):
env_name = '{}-{}'.format(system_client.env.environment, suffix)
client = system_client.add_model(env_name)
try:
yield client
except:
logging.exception('Exception while environment "{}" active'.format(
client.env.environment))
sys.exit(1)
finally:
safe_print_status(client)
hosted_log_dir = os.path.join(log_dir, suffix)
ensure_dir(hosted_log_dir)
dump_env_logs(client, None, hosted_log_dir)
client.destroy_model()
def hosted_environment(system_client, log_dir, suffix):
env_name = '{}-{}'.format(system_client.env.environment, suffix)
client = system_client.add_model(env_name)
try:
yield client
except:
logging.exception(
'Exception while environment "{}" active'.format(
client.env.environment))
sys.exit(1)
finally:
safe_print_status(client)
hosted_log_dir = os.path.join(log_dir, suffix)
ensure_dir(hosted_log_dir)
dump_env_logs(client, None, hosted_log_dir)
client.destroy_model()
def write_aws_config_file(user, tmp_dir, access_key, secret_key):
"""Write aws credentials file to tmp_dir
:return: String path of created credentials file.
"""
config_dir = os.path.join(tmp_dir, '.aws')
config_file = os.path.join(config_dir, 'credentials')
ensure_dir(config_dir)
config_contents = dedent("""\
[{}]
aws_access_key_id={}
aws_secret_access_key={}
""".format(user, access_key, secret_key))
with open(config_file, 'w') as f:
f.write(config_contents)
return config_file
def write_aws_config_file(user, tmp_dir, access_key, secret_key):
"""Write aws credentials file to tmp_dir
:return: String path of created credentials file.
"""
config_dir = os.path.join(tmp_dir, '.aws')
config_file = os.path.join(config_dir, 'credentials')
ensure_dir(config_dir)
config_contents = dedent("""\
[{}]
aws_access_key_id={}
aws_secret_access_key={}
""".format(user, access_key, secret_key))
with open(config_file, 'w') as f:
f.write(config_contents)
return config_file
def dump_env_logs_known_hosts(client, artifacts_dir, runtime_config=None,
known_hosts=None):
if known_hosts is None:
known_hosts = {}
if client.env.local:
logging.info("Retrieving logs for local environment")
copy_local_logs(client.env, artifacts_dir)
else:
remote_machines = get_remote_machines(client, known_hosts)
for machine_id in sorted(remote_machines, key=int):
remote = remote_machines[machine_id]
if not _can_run_ssh() and not remote.is_windows():
logging.info("No ssh, skipping logs for machine-%s using %r",
machine_id, remote)
continue
logging.info("Retrieving logs for machine-%s using %r", machine_id,
remote)
machine_dir = os.path.join(artifacts_dir,
"machine-%s" % machine_id)
ensure_dir(machine_dir)
copy_remote_logs(remote, machine_dir)
archive_logs(artifacts_dir)
retain_config(runtime_config, artifacts_dir)
def test_ensure_dir_raises_when_existing(self):
with self.assertRaises(BaseException):
ensure_dir('/tmp/nofoo8765/new_dir')
def test_ensure_dir(self):
with temp_dir() as directory:
expected_dir = os.path.join(directory, 'new_dir')
ensure_dir(expected_dir)
self.assertTrue(os.path.isdir(expected_dir))
def test_ensure_dir_leaves_existing_files(self):
with temp_dir() as directory:
expected_file = os.path.join(directory, 'some_file')
open(expected_file, 'a').close()
ensure_dir(directory)
self.assertTrue(os.path.isfile(expected_file))
def execute(self,args):
base = git.baseDir()
if base == "":
return False
dotGit = git.gitDir()
utility.printMsg("Optimizing git performance on slow file systems...")
#runs file system intensive tasks such as git status and git commit
# in parallel (important for NFS systems such as LC)
git.config("core.preloadindex","true")
#have git automatically do some garbage collection / optimization
utility.printMsg("Setting up automatic git garbage collection...")
git.config("gc.auto","1")
#prevents false conflict detection due to differences in filesystem
# time stamps
utility.printMsg("Optimizing cross platform portability...")
git.config("core.trustctime","false")
# stores login info for 12 hrs (max allowed by RZBitbucket)
if not args["--nocredcache"]:
cache = args["--credcache"]
if not cache:
cache = utility.userInput("Would you like to enable git-managed credential caching?", 'y')
if cache:
utility.printMsg("Enabling 12 hr caching of https credentials...")
if os.name == "nt":
git.config("--global credential.helper", "wincred")
else :
git.config("--global credential.helper", "cache --timeout=43200")
# enables 'as' option for merge strategies -forces a conflict if two branches
# modify the same file
mergeVerifyPath = os.path.join(os.path.dirname(__file__),"..","merge-and-verify-driver")
if os.path.exists(mergeVerifyPath):
utility.printMsg("Enabling safe merges (triggers conflicts any time same file is modified),\n\t see 'as' option for grape m and grape md...")
git.config("merge.verify.name","merge and verify driver")
git.config("merge.verify.driver","%s/merge-and-verify-driver %A %O %B")
else:
utility.printMsg("WARNING: merge and verify script not detected, safe merges ('as' option to grape m / md) will not work!")
# enables lg as an alias to print a pretty-font summary of
# key junctions in the history for this branch.
utility.printMsg("Setting lg as an alias for a pretty log call...")
git.config("alias.lg","log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit --date=relative --simplify-by-decoration")
# perform an update of the active subprojects if asked.
ask = not args["--nouv"]
updateView = ask and (args["--uv"] or utility.userInput("Do you want to edit your active subprojects?"
" (you can do this later using grape uv) [y/n]", "n"))
if updateView:
grapeMenu.menu().applyMenuChoice("uv", args["--uvArg"])
# configure git to use p4merge for conflict resolution
# and diffing
useP4Merge = not args["--nop4merge"] and (args["--p4merge"] or utility.userInput("Would you like to use p4merge as your merge tool? [y/n]","y"))
# note that this relies on p4merge being in your path somewhere
if (useP4Merge):
git.config("merge.keepBackup","false")
git.config("merge.tool","p4merge")
git.config("mergetool.keepBackup","false")
git.config("mergetool.p4merge.cmd",'p4merge \"\$BASE\" \"\$LOCAL\" \"\$REMOTE\" \"\$MERGED\"')
git.config("mergetool.p4merge.keepTemporaries","false")
git.config("mergetool.p4merge.trustExitCode","false")
git.config("mergetool.p4merge.keepBackup","false")
utility.printMsg("Configured repo to use p4merge for conflict resolution")
else:
git.config("merge.tool","tkdiff")
useP4Diff = not args["--nop4diff"] and (args["--p4diff"] or utility.userInput("Would you like to use p4merge as your diff tool? [y/n]","y"))
# this relies on p4diff being defined as a custom bash script, with the following one-liner:
# [ $# -eq 7 ] && p4merge "$2" "$5"
if (useP4Diff):
p4diffScript = os.path.join(os.path.dirname(__file__),"..","p4diff")
if os.path.exists(p4diffScript):
git.config("diff.external",p4diffScript)
utility.printMsg("Configured repo to use p4merge for diff calls - p4merge must be in your path")
else:
utility.printMsg("Could not find p4diff script at %s" % p4diffScript)
useGitP4 = args["--git-p4"]
if (useGitP4 ):
git.config("git-p4.useclientspec","true")
# create p4 references to enable imports from p4
p4remotes = os.path.join(dotGit,"refs","remotes","p4","")
utility.ensure_dir(p4remotes)
commit = utility.userInput("Please enter a descriptor (e.g. SHA, branch if tip, tag name) of the current git commit that mirrors the p4 repo","master")
sha = git.SHA(commit)
with open(os.path.join(p4remotes,"HEAD"),'w') as f:
f.write(sha)
with open(os.path.join(p4remotes,"master"),'w') as f:
f.write(sha)
# to enable exports to p4, a maindev client needs to be set up
haveCopied = False
while (not haveCopied):
p4settings = utility.userInput("Enter a path to a .p4settings file describing the maindev client you'd like to use for p4 updates",".p4settings")
try:
shutil.copyfile(p4settings,os.path.join(base,".p4settings"))
haveCopied = True
except:
print("could not find p4settings file, please check your path and try again")
return False
# install hooks here and in all submodules
utility.printMsg("Installing hooks in all repos...")
cwd = git.baseDir()
grapeMenu.menu().applyMenuChoice("installHooks")
# ensure all public branches are available in all repos
submodules = git.getActiveSubmodules()
config = grapeConfig.grapeConfig()
publicBranches = config.getPublicBranchList()
submodulePublicBranches = set(config.getMapping('workspace', 'submoduleTopicPrefixMappings').values())
for sub in submodules:
self.ensurePublicBranchesExist(grapeConfig.grapeRepoConfig(sub),sub, submodulePublicBranches)
# reset config to the workspace grapeconfig, use that one for all nested projects' public branches.
wsDir = utility.workspaceDir()
config = grapeConfig.grapeRepoConfig(wsDir)
for proj in grapeConfig.GrapeConfigParser.getAllActiveNestedSubprojectPrefixes():
self.ensurePublicBranchesExist(config, os.path.join(wsDir,proj), publicBranches)
self.ensurePublicBranchesExist(config, wsDir, publicBranches)
return True
def execute(self, args):
base = git.baseDir()
if base == "":
return False
dotGit = git.gitDir()
utility.printMsg("Optimizing git performance on slow file systems...")
#runs file system intensive tasks such as git status and git commit
# in parallel (important for NFS systems such as LC)
git.config("core.preloadindex", "true")
#have git automatically do some garbage collection / optimization
utility.printMsg("Setting up automatic git garbage collection...")
git.config("gc.auto", "1")
#prevents false conflict detection due to differences in filesystem
# time stamps
utility.printMsg("Optimizing cross platform portability...")
git.config("core.trustctime", "false")
# stores login info for 12 hrs (max allowed by RZBitbucket)
if not args["--nocredcache"]:
cache = args["--credcache"]
if not cache:
cache = utility.userInput(
"Would you like to enable git-managed credential caching?",
'y')
if cache:
utility.printMsg(
"Enabling 12 hr caching of https credentials...")
if os.name == "nt":
git.config("--global credential.helper", "wincred")
else:
git.config("--global credential.helper",
"cache --timeout=43200")
# enables 'as' option for merge strategies -forces a conflict if two branches
# modify the same file
mergeVerifyPath = os.path.join(os.path.dirname(__file__), "..",
"merge-and-verify-driver")
if os.path.exists(mergeVerifyPath):
utility.printMsg(
"Enabling safe merges (triggers conflicts any time same file is modified),\n\t see 'as' option for grape m and grape md..."
)
git.config("merge.verify.name", "merge and verify driver")
git.config("merge.verify.driver",
"%s/merge-and-verify-driver %A %O %B")
else:
utility.printMsg(
"WARNING: merge and verify script not detected, safe merges ('as' option to grape m / md) will not work!"
)
# enables lg as an alias to print a pretty-font summary of
# key junctions in the history for this branch.
utility.printMsg("Setting lg as an alias for a pretty log call...")
git.config(
"alias.lg",
"log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit --date=relative --simplify-by-decoration"
)
# perform an update of the active subprojects if asked.
ask = not args["--nouv"]
updateView = ask and (args["--uv"] or utility.userInput(
"Do you want to edit your active subprojects?"
" (you can do this later using grape uv) [y/n]", "n"))
if updateView:
grapeMenu.menu().applyMenuChoice("uv", args["--uvArg"])
# configure git to use p4merge for conflict resolution
# and diffing
useP4Merge = not args["--nop4merge"] and (
args["--p4merge"] or utility.userInput(
"Would you like to use p4merge as your merge tool? [y/n]",
"y"))
# note that this relies on p4merge being in your path somewhere
if (useP4Merge):
git.config("merge.keepBackup", "false")
git.config("merge.tool", "p4merge")
git.config("mergetool.keepBackup", "false")
git.config(
"mergetool.p4merge.cmd",
'p4merge \"\$BASE\" \"\$LOCAL\" \"\$REMOTE\" \"\$MERGED\"')
git.config("mergetool.p4merge.keepTemporaries", "false")
git.config("mergetool.p4merge.trustExitCode", "false")
git.config("mergetool.p4merge.keepBackup", "false")
utility.printMsg(
"Configured repo to use p4merge for conflict resolution")
else:
git.config("merge.tool", "tkdiff")
useP4Diff = not args["--nop4diff"] and (
args["--p4diff"] or utility.userInput(
"Would you like to use p4merge as your diff tool? [y/n]", "y"))
# this relies on p4diff being defined as a custom bash script, with the following one-liner:
# [ $# -eq 7 ] && p4merge "$2" "$5"
if (useP4Diff):
p4diffScript = os.path.join(os.path.dirname(__file__), "..",
"p4diff")
if os.path.exists(p4diffScript):
git.config("diff.external", p4diffScript)
utility.printMsg(
"Configured repo to use p4merge for diff calls - p4merge must be in your path"
)
else:
utility.printMsg("Could not find p4diff script at %s" %
p4diffScript)
useGitP4 = args["--git-p4"]
if (useGitP4):
git.config("git-p4.useclientspec", "true")
# create p4 references to enable imports from p4
p4remotes = os.path.join(dotGit, "refs", "remotes", "p4", "")
utility.ensure_dir(p4remotes)
commit = utility.userInput(
"Please enter a descriptor (e.g. SHA, branch if tip, tag name) of the current git commit that mirrors the p4 repo",
"master")
sha = git.SHA(commit)
with open(os.path.join(p4remotes, "HEAD"), 'w') as f:
f.write(sha)
with open(os.path.join(p4remotes, "master"), 'w') as f:
f.write(sha)
# to enable exports to p4, a maindev client needs to be set up
haveCopied = False
while (not haveCopied):
p4settings = utility.userInput(
"Enter a path to a .p4settings file describing the maindev client you'd like to use for p4 updates",
".p4settings")
try:
shutil.copyfile(p4settings,
os.path.join(base, ".p4settings"))
haveCopied = True
except:
print(
"could not find p4settings file, please check your path and try again"
)
return False
# install hooks here and in all submodules
utility.printMsg("Installing hooks in all repos...")
cwd = git.baseDir()
grapeMenu.menu().applyMenuChoice("installHooks")
# ensure all public branches are available in all repos
submodules = git.getActiveSubmodules()
config = grapeConfig.grapeConfig()
publicBranches = config.getPublicBranchList()
submodulePublicBranches = set(
config.getMapping('workspace',
'submoduleTopicPrefixMappings').values())
for sub in submodules:
self.ensurePublicBranchesExist(grapeConfig.grapeRepoConfig(sub),
sub, submodulePublicBranches)
# reset config to the workspace grapeconfig, use that one for all nested projects' public branches.
wsDir = utility.workspaceDir()
config = grapeConfig.grapeRepoConfig(wsDir)
for proj in grapeConfig.GrapeConfigParser.getAllActiveNestedSubprojectPrefixes(
):
self.ensurePublicBranchesExist(config, os.path.join(wsDir, proj),
publicBranches)
self.ensurePublicBranchesExist(config, wsDir, publicBranches)
return True
def test_ensure_dir(self):
with temp_dir() as directory:
expected_dir = os.path.join(directory, 'new_dir')
ensure_dir(expected_dir)
self.assertTrue(os.path.isdir(expected_dir))
def test_ensure_dir_leaves_existing_files(self):
with temp_dir() as directory:
expected_file = os.path.join(directory, 'some_file')
open(expected_file, 'a').close()
ensure_dir(directory)
self.assertTrue(os.path.isfile(expected_file))
def test_ensure_dir_raises_when_existing(self):
with self.assertRaises(BaseException):
ensure_dir('/tmp/nofoo8765/new_dir')