def run(self, sorted_repos, versions_file):
self.log("`git checkout` repositories...")
versions = None
with open(versions_file, 'r') as f:
versions = json.load(f)
cd(self._basedir)
for repo in sorted_repos:
if repo not in versions:
self.log("skipping {0}, no version specified.".format(repo))
continue
where = versions[repo] # where to checkout
self.log("Checkout {0} -> {1}".format(repo, where))
with push_pop(repo):
git.fetch()
git.checkout("--quiet", where)
# just in case that we didn't just cloned but updated:
git.reset("--hard", where)
self.log("done checking out repos.")
def start(self, name, starting_point=None):
"""Start a new step (branch)."""
# make sure this is not a known checkpoint
if name in self.points():
raise TutException("Duplicate checkpoint.")
# make sure the repo is clean
if self._repo_dirty():
raise TutException("Dirty tree.")
# create the new branch
git.branch(name)
if starting_point is None:
args = ('-b', name)
else:
args = ('-b', name, starting_point, '--track')
# add the branch to config
config = self._config()
points = config['points']
if self.current():
points.insert(points.index(self.current()) + 1, name)
else:
points.append(name)
self._update_config(
config,
log='Adding new point %s' % name,
)
# checkout the new branch
git.checkout(name)
def initialize_git_repository() -> None:
"""Prepare the git repository."""
print(" * Initializing the project git repository")
try:
git.init()
git.remote(
"add",
"origin",
"[email protected]:{{ cookiecutter.github_user }}/"
"{{ cookiecutter.project_slug }}.git",
)
git.add(".")
git.commit("-m", "feat: create initial project structure")
git.checkout("-b", "gh-pages")
git.checkout("-b", "feat/initial_iteration")
print(" * Pushing initial changes to master")
git.push("--force", "--set-upstream", "origin", "master")
git.push("--force", "--set-upstream", "origin", "gh-pages")
git.push("--force", "--set-upstream", "origin",
"feat/initial_iteration")
git.push("--force")
except sh.ErrorReturnCode as error:
print("There was an error creating the Git repository.")
print(str(error.stderr, "utf8"))
sys.exit(1)
def _create_git(self):
"""create a book from a git repo"""
if not os.path.exists(self.path):
LOG.info(("Creating book {0} from {1}, branch: {2}" +
"").format(self.path, self.git, self.branch))
git.clone(self.git, self.path)
else:
LOG.info("Book {0} already exists".format(self.path))
cwd = os.getcwd()
os.chdir(self.path)
if self.skiprepourlcheck:
remote_match_found = False
for remote in git("remote", "-v"):
remote_parts = remote.split()
if Url(remote_parts[1]) == Url(self.git):
remote_match_found = True
if remote_match_found:
LOG.debug('Found {0} in the list of remotes for {1}'.format(self.git, self.path))
else:
LOG.error('ERROR: {0} wasn\'t found in the list of remotes for {1}'.format(self.git, self.path))
if not self._check_branch():
LOG.info("Switching {0} to branch {1}".format(self.path,
self.branch))
git.fetch
git.checkout(self.branch)
os.chdir(cwd)
def test_merge(self):
def cleanup_merge():
git.checkout("master")
# if something failed, the branch might still exist
if self.gd.branch_exists("to_merge_1"):
git.branch("-D", "to_merge_1")
git.branch("-D", "base_branch")
self.addCleanup(cleanup_merge)
git.checkout("-b", "to_merge_1")
git.checkout("-b", "base_branch")
file = open("foo.txt", "w")
file.write("ABC\n")
file.close()
git.add("foo.txt")
git.commit("-m", "Test commit")
new_sha = self.gd.merge("to_merge_1", "base_branch")
self.assertTrue(new_sha != "", "new_sha=%s is non-empty" % new_sha)
self.assertEqual(len(new_sha), 40, "SHA is 40 chars")
self.assertTrue(True, "Merge succeeded")
def create(version_number):
heading1("Creating new version based on Fedora " + version_number + "\n")
# Update git and create new version.
heading2("Updating master branch.")
print(git.checkout("master"))
print(git.pull()) # Bring branch current.
heading2("Creating new branch")
print(git.checkout("-b" + version_number)) # Create working branch.
# Get kickstart files.
heading2("Creating fedora-kickstarts directory\n")
mkdir("-p", (base_dir + "/fedora-kickstarts/"))
cd(base_dir + "/fedora-kickstarts/")
heading2("Downloading Fedora kickstart files.")
ks_base = "https://pagure.io/fedora-kickstarts/raw/f" \
+ version_number + "/f"
for file in ks_files:
file_path = ks_base + "/fedora-" + file
print ("Downloading " + file_path)
curl("-O", file_path)
def _create_git(self):
"""create a book from a git repo"""
if not os.path.exists(self.path):
LOG.info(("Creating book {0} from {1}, branch: {2}" +
"").format(self.path, self.git, self.branch))
git.clone(self.git, self.path)
else:
LOG.info("Book {0} already exists".format(self.path))
cwd = os.getcwd()
os.chdir(self.path)
if self.skiprepourlcheck:
remote_match_found = False
for remote in git("remote", "-v"):
remote_parts = remote.split()
if Url(remote_parts[1]) == Url(self.git):
remote_match_found = True
if remote_match_found:
LOG.debug('Found {0} in the list of remotes for {1}'.format(self.git, self.path))
else:
LOG.error('ERROR: {0} wasn\'t found in the list of remotes for {1}'.format(self.git, self.path))
if not self._check_branch():
LOG.info("Switching {0} to branch {1}".format(self.path,
self.branch))
git.fetch
git.checkout(self.branch)
os.chdir(cwd)
def newest_study_id(self):
"Return the numeric part of the newest study_id"
os.chdir(self.repo)
git.checkout("master")
dirs = []
# first we look for studies already in our master branch
for f in os.listdir("study/"):
if os.path.isdir("study/%s" % f):
# ignore alphabetic prefix, o = created by opentree API
if f[0].isalpha():
dirs.append(int(f[1:]))
else:
dirs.append(int(f))
# next we must look at local branch names for new studies
# without --no-color we get terminal color codes in the branch output
branches = git.branch("--no-color")
branches = [b.strip() for b in branches]
for b in branches:
mo = re.match(".+_o(\d+)", b)
if mo:
dirs.append(int(mo.group(1)))
dirs.sort()
return dirs[-1]
def commit(self, objects, message):
# validate commit message
if not message or not isinstance(message, basestring):
raise ValueError("Commit message should not be empty or not string")
env = os.environ.copy()
env.update({
'GIT_WORK_TREE': self.repo,
'GIT_DIR': '%s/.git' % self.repo,
})
git.gc("--prune", _env=env)
git.checkout("HEAD", _env=env)
# pull and push from and to the remote
git.pull("origin", "master", _env=env)
for obj in objects:
git.add("-A", obj, _env=env)
try:
git.commit("-m", message, _env=env)
except Exception:
pass
git.push(_env=env)
def cleanup_merge():
git.checkout("master")
# if something failed, the branch might still exist
if self.gd.branch_exists("to_merge_1"):
git.branch("-D", "to_merge_1")
git.branch("-D", "base_branch")
def newest_study_id(self):
"Return the numeric part of the newest study_id"
os.chdir(self.repo)
git.checkout("master")
dirs = []
# first we look for studies already in our master branch
for f in os.listdir("study/"):
if os.path.isdir("study/%s" % f):
# ignore alphabetic prefix, o = created by opentree API
if f[0].isalpha():
dirs.append(int(f[1:]))
else:
dirs.append(int(f))
# next we must look at local branch names for new studies
# without --no-color we get terminal color codes in the branch output
branches = git.branch("--no-color")
branches = [ b.strip() for b in branches ]
for b in branches:
mo = re.match(".+_o(\d+)",b)
if mo:
dirs.append(int(mo.group(1)))
dirs.sort()
return dirs[-1]
def test_merge(self):
def cleanup_merge():
git.checkout("master")
# if something failed, the branch might still exist
if self.gd.branch_exists("to_merge_1"):
git.branch("-D", "to_merge_1")
git.branch("-D", "base_branch")
self.addCleanup(cleanup_merge)
git.checkout("-b", "to_merge_1")
git.checkout("-b", "base_branch")
file = open("foo.txt", "w")
file.write("ABC\n")
file.close()
git.add("foo.txt")
git.commit("-m","Test commit")
new_sha = self.gd.merge("to_merge_1", "base_branch")
self.assertTrue( new_sha != "", "new_sha=%s is non-empty" % new_sha)
self.assertEqual(len(new_sha), 40, "SHA is 40 chars")
self.assertTrue(True, "Merge succeeded")
def start(self, name, starting_point=None):
"""Start a new step (branch)."""
# make sure this is not a known checkpoint
if name in self.points():
raise TutException("Duplicate checkpoint.")
# make sure the repo is clean
if self._repo_dirty():
raise TutException("Dirty tree.")
# create the new branch
git.branch(name)
if starting_point is None:
args = ('-b', name)
else:
args = ('-b', name, starting_point, '--track')
# add the branch to config
config = self._config()
points = config['points']
if self.current():
points.insert(points.index(self.current()) + 1, name)
else:
points.append(name)
self._update_config(
config,
log='Adding new point %s' % name,
)
# checkout the new branch
git.checkout(name)
def commit(self, objects, message):
# validate commit message
if not message or not isinstance(message, basestring):
raise ValueError(
"Commit message should not be empty or not string")
env = os.environ.copy()
env.update({
'GIT_WORK_TREE': self.repo,
'GIT_DIR': '%s/.git' % self.repo,
})
git.gc("--prune", _env=env)
git.checkout("HEAD", _env=env)
# pull and push from and to the remote
git.pull("origin", "master", _env=env)
for obj in objects:
git.add("-A", obj, _env=env)
try:
git.commit("-m", message, _env=env)
except Exception:
pass
git.push(_env=env)
def read_with_encoding(self, filename, document, codec_info, encoding):
f = None
try:
if 'branch' in self.options:
self.checkout = self.options.get('branch')
os.chdir(self.code_base)
print "\nChecking out branch:", self.checkout
git.checkout(self.checkout)
f = codecs.StreamReaderWriter(open(filename, 'rb'), codec_info[2],
codec_info[3], 'strict')
lines = f.readlines()
git.checkout(self.current_branch)
os.chdir(self.current_dir)
lines = dedent_lines(lines, self.options.get('dedent'))
return lines
except (IOError, OSError):
return [
document.reporter.warning(
'Include file %r not found or reading it failed' %
filename,
line=self.lineno)
]
except UnicodeError:
return [
document.reporter.warning(
'Encoding %r used for reading included file %r seems to '
'be wrong, try giving an :encoding: option' %
(encoding, filename))
]
finally:
if f is not None:
f.close()
def cleanup_merge():
git.checkout("master")
# if something failed, the branch might still exist
if self.gd.branch_exists("to_merge_1"):
git.branch("-D", "to_merge_1")
git.branch("-D", "base_branch")
def run(self):
global _DEFAULT_PATH
global _RESET_PATHS
if 'path' in self.options:
tut_path = self.options['path']
elif _DEFAULT_PATH is not None:
tut_path = _DEFAULT_PATH
else:
raise Exception("No tut path specified.")
# paths are relative to the project root
rel_path, tut_path = self.state.document.settings.env.relfn2path(
tut_path)
curdir = os.getcwd()
os.chdir(tut_path)
# if this is the first time visiting this repo
if tut_path not in _RESET_PATHS:
# record the current branch
_RESET_PATHS[tut_path] = \
git('name-rev', 'HEAD').strip().split()[-1]
git.checkout(self.arguments[0].strip().lower())
sphinx.pycode.ModuleAnalyzer.cache = {}
os.chdir(curdir)
return []
def publish_site(self):
tmp_branch = '__dynamo_deploy__'
detached_branch = None
git.checkout(self.target_branch)
self._add_and_commit_site('dist')
git.push('origin', self.target_branch)
git.checkout('@{-1}')
def checkout(branch):
print "Checking out "+branch
try:
git.checkout('-b',branch)
except:
# Branch already exists, check skip creation flag
git.checkout(branch)
pass
def test_current_returns_none_on_unknown_branch(self):
t = tut.model.Tut(self._testpath)
t.init()
t.start('step1')
git.checkout('master')
self.assertNotIn('master', t.points())
self.assertEqual(t.current(), None)
def __init__(self, settings):
self.settings = settings
# Into the context
if not os.path.exists(self.settings.local_dir):
git.clone(self.settings.repo_url, self.settings.local_dir)
cd(settings.local_dir)
# Make sure checkout the right branch
git.checkout(settings.branch)
self.__current_commit = ""
def __init__(self, settings):
self.settings = settings
# Into the context
if not os.path.exists(self.settings.local_dir):
git.clone(self.settings.repo_url, self.settings.local_dir)
cd(settings.local_dir)
# Make sure checkout the right branch
git.checkout(settings.branch)
self.__current_commit = ""
def need_update(p: str) -> Union[bool, int]:
exists = os.path.exists(join(args.root, p))
k = {'_cwd': args.root}
git.checkout('i18n/i18n', '--', p, **k)
git.reset('HEAD', '--', p, **k)
if not exists:
return True
else:
return self.file_is_modified(join(args.root, p))
def git_ch_branch(file_path, branch, remote=None, create=True):
'''Change current branch'''
args = []
if create:
args.append('-b')
if remote:
branch = remote + '/' + branch
args.append(branch)
git.checkout(*args, _cwd=dir_path(file_path))
show_msg('Now at ' + branch, 'Info')
def git_ch_branch(file_path, branch, remote=None, create=True):
'''Change current branch'''
args = []
if create:
args.append('-b')
if remote:
branch = remote + '/' + branch
args.append(branch)
git.checkout(*args, _cwd=dir_path(file_path))
show_msg('Now at ' + branch, 'Info')
def main():
git.checkout("-b", "task_1")
f = open("file1.txt", "w")
f.write("here is a file")
f.close()
git.commit("-am", "commited new text file1")
git.push("--set-upstream", "origin", "task1")
git.checkout("-")
def install_addon(self, addon):
addonPath = os.path.join(self.get_addon_path(), addon.name)
if os.path.isdir(addonPath):
# Folder already exists
if not os.path.isdir(os.path.join(addonPath, '.git')):
raise Exception('WARNING',
'ALREADY_INSTALLED_UNLINKED',
'Addon %s is already installed, but it\'s state is unknown.\nPlease reinstall with:\n' % addon.name + note('ofx reinstall %s' % addon.name))
else:
os.chdir(addonPath)
currentSha = sh.git('rev-parse', 'HEAD').rstrip()
remoteSha = addon.latest_commit['sha']
if addon.version is not None:
try:
git.checkout(addon.version)
except Exception:
raise Exception('ERROR',
'UNKNOWN_VERSION',
'No version named %s found for %s' % (addon.version, addon.name))
print ok() + addon.name + ": %s checked out" % addon.version
else:
if currentSha == remoteSha:
raise Exception('OK',
'ALREADY_INSTALLED',
'Addon %s is already installed and up to date' % addon.name)
else:
raise Exception('WARNING',
'ALREADY_INSTALLED_OUTDATED',
'Addon %s is already installed but is not up to date. Consider running:\n' % addon.name + note("ofx update %s" % addon.name))
else:
print '>> Cloning %s' % addon.clone_url
os.chdir(self.get_addon_path())
git.clone(addon.clone_url)
os.chdir(addonPath)
sha = sh.git('rev-parse', 'HEAD').rstrip()
if not sha:
raise Exception('ERROR', 'CLONE_ERROR', "Something went wrong while cloning from "+addon.clone_url)
else:
print ok()+addon.name + ": Clone done"
if addon.version is not None:
try:
git.checkout(addon.version)
except Exception:
raise Exception('ERROR',
'UNKNOWN_VERSION',
'No version named %s found for %s' % (addon.version, addon.name))
print ok() + addon.name + ": %s checked out" % addon.version
self.install_dependencies(addon)
return True
def cleanup(app, exception):
global _RESET_PATHS
curdir = os.getcwd()
try:
for path in _RESET_PATHS:
os.chdir(path)
git.checkout(_RESET_PATHS[path])
finally:
os.chdir(curdir)
def cleanup(app, exception):
global _RESET_PATHS
curdir = os.getcwd()
try:
for path in _RESET_PATHS:
os.chdir(path)
git.checkout(_RESET_PATHS[path])
finally:
os.chdir(curdir)
def edit(self, name):
"""Start editing the checkpoint point_name."""
# make sure this is a known checkpoint
if name not in self.points():
raise TutException("Unknown checkpoint.")
# make sure the repo is clean
if self._repo_dirty():
raise TutException("Dirty tree.")
git.checkout(name)
def setup(job_sources):
job_sources_by_name = {}
for job_source in job_sources:
if os.path.isdir(job_source):
name = os.path.basename(os.path.realpath(job_source))
js = {"commitish": None, "directory": job_source, "url": None}
else:
url, commitish = job_source.split(";", 1)
match = re.search(
r"/([a-zA-Z0-9-_]+)(?:.git)?",
url
)
name = match.group(1)
js = {"commitish": commitish, "directory": name, "url": url}
job_sources_by_name[name] = js
sys.stderr.write("Job Sources: {} \n".format(job_sources_by_name))
paths_by_source = {}
for name, job_source in job_sources_by_name.items():
directory = job_source["directory"]
if job_source["url"]:
try:
git.clone(
"--branch",
job_source["commitish"],
job_source["url"],
directory
)
except ErrorReturnCode_128 as e:
sys.stderr.write(
"Clone Failure: {}, "
"retrying checkout.\n".format(e))
git.checkout(
job_source["commitish"],
_env={"GIT_DIR": directory + "/.git"}
)
try:
metadata_file = "{d}/component_metadata.yml".format(d=directory)
with open(metadata_file) as f:
metadata = yaml.load(f)
except IOError as e:
sys.stderr.write("Failed to load component metadata file: "
"{}, error: {}".format(metadata_file, e))
else:
paths = metadata.get("jenkins", {}).get("jjb_paths", [])
paths_by_source[name] = [
os.path.join(directory, path)
for path in paths
]
all_paths = chain(*paths_by_source.values())
click.echo(":".join(all_paths))
def edit(self, name):
"""Start editing the checkpoint point_name."""
# make sure this is a known checkpoint
if name not in self.points():
raise TutException("Unknown checkpoint.")
# make sure the repo is clean
if self._repo_dirty():
raise TutException("Dirty tree.")
git.checkout(name)
def start(self, name):
"""Start a new step (branch)."""
# make sure this is not a known checkpoint
if name in self.points():
raise TutException("Duplicate checkpoint.")
# make sure the repo is clean
if self._repo_dirty():
raise TutException("Dirty tree.")
git.checkout('-b', name)
def main():
options = json.load(sys.stdin)
github_remote_url = f"{get_github_url()}/{options['target_remote']}.git"
if len(list(pathlib.Path('.').iterdir())) > 0:
print("Found existing files in work directory", file=sys.stderr)
assert pathlib.Path('.git').exists(
), "if files are present in the work dir, it must be a git work tree"
remote = "origin"
if options["source_remote_name"] == remote:
remote = remote + "2"
add_or_set_git_remote(remote, github_remote_url)
print(f"Fetching from {github_remote_url}", file=sys.stderr)
git.fetch(remote)
print(
f"Checking out {options['target_branch']} from {remote}/{options['target_branch']}",
file=sys.stderr)
git.checkout("-B", options['target_branch'],
f"{remote}/{options['target_branch']}")
print(f"Cleaning work tree", file=sys.stderr)
git.reset("--hard", "HEAD")
git.clean("-fdx")
else:
print(f"Cloning {options['target_branch']} from {github_remote_url}",
file=sys.stderr)
git.clone("--branch", options['target_branch'], github_remote_url, ".")
if options['target_remote'] != options['source_remote']:
source_remote_name = options['source_remote_name']
add_or_set_git_remote(
source_remote_name,
f"{get_github_url()}/{options['source_remote']}.git")
print(
f"Fetching from {get_github_url()}/{options['source_remote']}.git",
file=sys.stderr)
git.fetch(source_remote_name)
set_git_author_info(f"GitHub Action {os.environ['GITHUB_ACTION']}",
"action@localhost")
try:
git("cherry-pick", options['source_commits'])
print(
f"Source commits ({options['source_commits']}) were successfully cherry-picked "
f"onto {options['target_remote']}:{options['target_branch']}",
file=sys.stderr)
except sh.ErrorReturnCode:
print(
f"Source commits ({options['source_commits']}) could not be cherry-picked "
f"onto {options['target_remote']}:{options['target_branch']}",
file=sys.stderr)
raise
def need_update(path: str) -> bool:
"""
Check if the file is different in the i18n repo.
"""
exists = os.path.exists(join(args.root, path))
k = {"_cwd": args.root}
git.checkout(args.target, "--", path, **k)
git.reset("HEAD", "--", path, **k)
if not exists:
return True
return self.file_is_modified(join(args.root, path))
def next(self, merge=False):
current = self.current()
try:
switch_to = self.points()[self.points().index(current) + 1]
except IndexError:
# we've reached the end of the list; switch to master
switch_to = 'master'
git.checkout(switch_to)
if merge:
git.merge(current)
def test_merge_conflict(self):
def cleanup_merge_conflict():
git.checkout("master")
# if something failed, the branch might still exist
if self.gd.branch_exists("to_merge_1"):
git.branch("-D", "to_merge_1")
if self.gd.branch_exists("to_merge_2"):
git.branch("-D", "to_merge_2")
self.addCleanup(cleanup_merge_conflict)
git.checkout("master")
git.checkout("-b", "to_merge_1")
file = open("foo.txt", "w")
file.write("ABC\n")
file.close()
git.add("foo.txt")
git.commit("-m", "Test commit")
git.checkout("master")
git.checkout("-b", "to_merge_2")
file = open("foo.txt", "w")
file.write("XYZ\n")
file.close()
git.add("foo.txt")
git.commit("-m", "Test commit")
self.assertRaises(MergeException,
lambda: self.gd.merge("to_merge_1", "to_merge_2"))
def repository(namespace, name, branch='master'):
'''Returns a repository'''
with TemporaryDirectory() as download_path:
old_directory = str(pwd()).strip()
try:
git.clone('https://github.com/{0}/{1}.git'.format(namespace, name), download_path)
cd(download_path)
git.fetch('origin', branch)
git.checkout(branch)
yield (download_path, git('rev-parse', 'HEAD'), redis.Dict(key="{0}.{1}".format(namespace, name)))
except ErrorReturnCode_128:
mkdir(download_path)
yield (None, None, None)
cd(old_directory)
def test_merge_conflict(self):
def cleanup_merge_conflict():
git.checkout("master")
# if something failed, the branch might still exist
if self.gd.branch_exists("to_merge_1"):
git.branch("-D", "to_merge_1")
if self.gd.branch_exists("to_merge_2"):
git.branch("-D", "to_merge_2")
self.addCleanup(cleanup_merge_conflict)
git.checkout("master")
git.checkout("-b", "to_merge_1")
file = open("foo.txt", "w")
file.write("ABC\n")
file.close()
git.add("foo.txt")
git.commit("-m","Test commit")
git.checkout("master")
git.checkout("-b", "to_merge_2")
file = open("foo.txt", "w")
file.write("XYZ\n")
file.close()
git.add("foo.txt")
git.commit("-m","Test commit")
self.assertRaises(MergeException, lambda: self.gd.merge("to_merge_1", "to_merge_2") )
def add_to_blacklist(self, items_to_blacklist, username, code_permissions):
# Check if we're on master
if git("rev-parse", "--abbrev-ref", "HEAD").strip() != "master":
return (False, "Not currently on master.")
# Check that we're up-to-date with origin (GitHub)
git.remote.update()
if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
return (False, "HEAD isn't at tip of origin's master branch")
# Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
if "blacklisted_websites.txt" in git.status(): # Also ugly
return (False, "blacklisted_websites.txt modified locally. This is probably bad.")
# Store current commit hash
current_commit = git("rev-parse", "HEAD").strip()
# Add items to file
with open("blacklisted_websites.txt", "a+") as blacklisted_websites:
last_character = blacklisted_websites.read()[-1:]
if last_character != "\n":
blacklisted_websites.write("\n")
blacklisted_websites.write("\n".join(items_to_blacklist) + "\n")
# Checkout a new branch (mostly unnecessary, but may help if we create PRs in the future
branch = "auto-blacklist-{0}".format(str(time.time()))
git.checkout("-b", branch)
# Clear HEAD just in case
git.reset("HEAD")
git.add("blacklisted_websites.txt")
git.commit("-m", "Auto blacklist of {0} by {1} --autopull".format(", ".join(items_to_blacklist), username))
if code_permissions:
git.checkout("master")
git.merge(branch)
git.push()
else:
git.push("origin", branch)
git.checkout("master")
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "tell someone to set a GH password")
payload = {"title": "{0}: Blacklist {1}".format(username, ", ".join(items_to_blacklist)),
"body": "{0} requests blacklist of domains: \n\n - {1}".format(username, "\n - ".join(items_to_blacklist)),
"head": branch,
"base": "master"}
response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload))
print(response.json())
return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(response.json()["html_url"]))
git.checkout(current_commit) # Return to old commit to await CI. This will make Smokey think it's in reverted mode if it restarts
if not code_permissions:
return (False, "Unable to perform action due to lack of code-level permissions. [Branch pushed](https://github.com/Charcoal-SE/SmokeDetector/tree/{0}), PR at your leisure.".format(branch))
return (True, "Blacklisted {0} - the entry will be applied via autopull if CI succeeds.".format(", ".join(items_to_blacklist)))
def setUp(self):
super(TestFileResolve, self).setUp()
# Generate a conflict
git.checkout(b='branch')
utils_lib.write_file(FP_IN_CONFLICT, contents='branch')
utils_lib.write_file(DIR_FP_IN_CONFLICT, contents='branch')
git.add(FP_IN_CONFLICT, DIR_FP_IN_CONFLICT)
git.commit(FP_IN_CONFLICT, DIR_FP_IN_CONFLICT, m='branch')
git.checkout('master')
utils_lib.write_file(FP_IN_CONFLICT, contents='master')
utils_lib.write_file(DIR_FP_IN_CONFLICT, contents='master')
git.add(FP_IN_CONFLICT, DIR_FP_IN_CONFLICT)
git.commit(FP_IN_CONFLICT, DIR_FP_IN_CONFLICT, m='master')
git.merge('branch', _ok_code=[1])
def setUp(self):
super(TestFileResolve, self).setUp()
# Generate a conflict
git.checkout(b='branch')
utils_lib.write_file(FP_IN_CONFLICT, contents='branch')
utils_lib.write_file(DIR_FP_IN_CONFLICT, contents='branch')
git.add(FP_IN_CONFLICT, DIR_FP_IN_CONFLICT)
git.commit(FP_IN_CONFLICT, DIR_FP_IN_CONFLICT, m='branch')
git.checkout('master')
utils_lib.write_file(FP_IN_CONFLICT, contents='master')
utils_lib.write_file(DIR_FP_IN_CONFLICT, contents='master')
git.add(FP_IN_CONFLICT, DIR_FP_IN_CONFLICT)
git.commit(FP_IN_CONFLICT, DIR_FP_IN_CONFLICT, m='master')
git.merge('branch', _ok_code=[1])
def next(self, merge=False):
current = self.current()
try:
switch_to = self.points()[
self.points().index(current) + 1
]
except IndexError:
# we've reached the end of the list; switch to master
switch_to = 'master'
git.checkout(switch_to)
if merge:
git.merge(current)
def repository(namespace, name, branch='master'):
'''Returns a repository'''
with TemporaryDirectory() as download_path:
old_directory = str(pwd()).strip()
try:
git.clone('https://github.com/{0}/{1}.git'.format(namespace, name),
download_path)
cd(download_path)
git.fetch('origin', branch)
git.checkout(branch)
yield (download_path, git('rev-parse', 'HEAD'),
redis.Dict(key="{0}.{1}".format(namespace, name)))
except ErrorReturnCode_128:
mkdir(download_path)
yield (None, None, None)
cd(old_directory)
def _update_config(self, config, log=None):
branch = self._current_branch()
try:
git.checkout('tut')
yaml.dump(
config,
open('tut.cfg', 'w'),
default_flow_style=False,
)
git.add('tut.cfg')
git.commit(m=log or 'Update configuration.', )
finally:
git.checkout(branch)
def test_newest_study_id(self):
def cleanup_newest():
git.checkout("master")
git.branch("-D","leto_study_o9999")
self.addCleanup(cleanup_newest)
git.checkout("master")
newest_id = self.gd.newest_study_id()
self.assertGreaterEqual( newest_id, 2600)
git.checkout("-b", "leto_study_o9999")
newest_id = self.gd.newest_study_id()
self.assertGreaterEqual( newest_id, 9999 )
def test_newest_study_id(self):
def cleanup_newest():
git.checkout("master")
git.branch("-D", "leto_study_o9999")
self.addCleanup(cleanup_newest)
git.checkout("master")
newest_id = self.gd.newest_study_id()
self.assertGreaterEqual(newest_id, 2600)
git.checkout("-b", "leto_study_o9999")
newest_id = self.gd.newest_study_id()
self.assertGreaterEqual(newest_id, 9999)
def start_test(self, repo, ref):
base_repo = redis.get("source:" + repo).decode("utf-8")
l = redis.lock(base_repo, timeout=60 * 60)
log_key = "log:" + repo + "/" + ref
log_url = "https://rosie-ci.ngrok.io/log/" + repo + "/" + ref
print("grabbing lock " + base_repo)
# Retry the task in 10 seconds if the lock can't be grabbed.
if not l.acquire(blocking=False):
if self.request.retries == 24:
set_status(repo, ref, "error", log_url,
"Hit max retries. Please ping the owner.")
raise self.retry(countdown=30, max_retries=25)
print("Lock grabbed " + base_repo)
redis.set("owner-" + base_repo, log_url)
set_status(repo, ref, "pending", log_url, "Commencing Rosie test.")
repo_path = cwd + "/repos/" + base_repo
os.chdir(repo_path)
try:
redis.append(log_key, git.checkout(ref))
except sh.ErrorReturnCode_128 as e:
print("error 128")
redis.append(
log_key, e.full_cmd + "\n" + e.stdout.decode('utf-8') + "\n" +
e.stderr.decode('utf-8'))
final_status(repo, ref, "error", "Git error in Rosie.")
except sh.ErrorReturnCode_1 as e:
print("error 1")
redis.append(
log_key, e.full_cmd + "\n" + e.stdout.decode('utf-8') + "\n" +
e.stderr.decode('utf-8'))
final_status(repo, ref, "error", "Git checkout error in Rosie.")
print("test started " + log_url)
return l.local.token.decode("utf-8")
def run(self):
global _DEFAULT_PATH
global _RESET_PATHS
if 'path' in self.options:
tut_path = self.options['path']
elif _DEFAULT_PATH is not None:
tut_path = _DEFAULT_PATH
else:
raise Exception("No tut path specified.")
# paths are relative to the project root
rel_path, tut_path = self.state.document.settings.env.relfn2path(
tut_path)
curdir = os.getcwd()
os.chdir(tut_path)
# if this is the first time visiting this repo
if tut_path not in _RESET_PATHS:
# record the current branch
_RESET_PATHS[tut_path] = \
git('name-rev', 'HEAD').strip().split()[-1]
git_ref = self.arguments[0].strip().lower()
try:
git.checkout(git_ref)
except sh.ErrorReturnCode_1 as git_error:
if six.b(
"error: pathspec '%s' did not match any "
"file(s) known to git.\n" % (
git_ref,
)
) == git_error.stderr:
raise ValueError(
"git checkpoint '%s' does not exist." % (git_ref,)
)
finally:
sphinx.pycode.ModuleAnalyzer.cache = {}
os.chdir(curdir)
return []
def pull(url, path, remote=None):
here = str(pwd()).strip()
try:
cd(path)
except OSError:
print "path does not exist? {}".format(path)
cd(here)
return
try:
git.status()
except ErrorReturnCode_128:
print "{} is not a git repository!".format(path)
cd(here)
return
git.pull(remote or url, 'master')
git.checkout('-f')
cd(here)
def merge_and_check(base, head):
"""Merge <head> into <base>, then run some tests.
Only modifies the working tree---doesn't actually create a merge
commit. Resets and cleans the repo and leaves it in headless mode.
Raises sh.ErrorReturnCode if the merge or the tests fail.
"""
# Make sure we're up to date
git.fetch()
# Make sure we can do a clean checkout
git.reset(hard=True)
git.clean('-dfx')
git.checkout('origin/' + base)
# Merge the working tree, but don't modify the index
git.merge('origin/' + head, no_commit=True)
# Check the PR!
check()
def _update_config(self, config, log=None):
branch = self._current_branch()
try:
git.checkout('tut')
yaml.dump(
config,
open('tut.cfg', 'w'),
default_flow_style=False,
)
git.add('tut.cfg')
git.commit(
m=log or 'Update configuration.',
)
finally:
git.checkout(branch)
def merge_and_check(base, head):
"""Merge <head> into <base>, then run some tests.
Only modifies the working tree---doesn't actually create a merge
commit. Resets and cleans the repo and leaves it in headless mode.
Raises sh.ErrorReturnCode if the merge or the tests fail.
"""
# Make sure we're up to date
git.fetch()
# Make sure we can do a clean checkout
git.reset(hard=True)
git.clean('-dfx')
git.checkout('origin/' + base)
# Merge the working tree, but don't modify the index
git.merge('origin/' + head, no_commit=True)
# Check the PR!
check()
def setup_environment(exp):
"""
Do environment setup like downloading code and inputs.
"""
# Get the code, we only handle git repos.
code_urls = exp.code_urls.split(',')
for u in code_urls:
assert u[-4:] == '.git'
code_repos = [re.search('(\w+)\.git$', u).group(1) for u in code_urls]
code_hashes = exp.code_hashes.split(',')
assert len(code_urls) == len(code_hashes)
assert len(code_repos) == len(code_urls)
for c in code_urls:
git.clone(['--recursive', c])
orig_dir = os.getcwd()
for i, c in enumerate(code_hashes):
os.chdir(code_repos[i])
git.checkout(code_hashes[i])
git.submodule('update')
os.chdir(orig_dir)
data_urls = exp.data_urls.split(',')
data_hashes = exp.data_hashes.split(',')
for u in data_urls:
assert u[-7:] == '.tar.gz'
data_dirs = [re.search('(\w+)\.tar.gz$', u).group(1) for u in data_urls]
assert len(data_urls) == len(data_hashes)
assert len(data_dirs) == len(data_urls)
for u, d in zip(data_urls, data_dirs)
wget(d)
# Do build, then link in data.
if exp.codebase == 'MOM6':
mom6.build()
elif exp.codebase == 'MOM5':
mom5.build()
else:
assert False
return True
def run(self):
global _DEFAULT_PATH
global _RESET_PATHS
if 'path' in self.options:
tut_path = self.options['path']
elif _DEFAULT_PATH is not None:
tut_path = _DEFAULT_PATH
else:
raise Exception("No tut path specified.")
# paths are relative to the project root
rel_path, tut_path = self.state.document.settings.env.relfn2path(
tut_path)
curdir = os.getcwd()
os.chdir(tut_path)
# if this is the first time visiting this repo
if tut_path not in _RESET_PATHS:
# record the current branch
_RESET_PATHS[tut_path] = \
git('name-rev', 'HEAD').strip().split()[-1]
git_ref = self.arguments[0].strip().lower()
try:
git.checkout(git_ref)
except sh.ErrorReturnCode_1 as git_error:
if six.b("error: pathspec '%s' did not match any "
"file(s) known to git.\n" %
(git_ref, )) == git_error.stderr:
raise ValueError("git checkpoint '%s' does not exist." %
(git_ref, ))
finally:
sphinx.pycode.ModuleAnalyzer.cache = {}
os.chdir(curdir)
return []
def clone(self):
self.tmp_dir = tempfile.mkdtemp(suffix='facio')
try:
from sh import git
except ImportError:
raise Exception # TODO: Custom exception
try:
git = git.bake(_cwd=self.tmp_dir)
git.clone(self.repo, self.tmp_dir)
git.fetch('--all')
git.checkout('master') # TODO: Branch prompt to the user
except Exception:
raise Exception # TODO: Custom exception
rmtree(os.path.join(self.tmp_dir, '.git'))
with indent(4, quote=' >'):
puts(blue('Clone complete'))
def setUpClass(self):
conf = SafeConfigParser(allow_no_value=True)
if os.path.isfile("../private/localconfig"):
conf.read("../private/localconfig")
else:
conf.read("../private/config")
self.repo = conf.get("apis", "repo_path")
self.gd = GitData(repo=self.repo)
self.orig_cwd = os.getcwd()
# go into our data repo
os.chdir(self.repo)
self.testing_branch_name = "testing_%d" % int(time.time())
# create the branch
git.checkout("-b", self.testing_branch_name)
# start all tests on the master branch
git.checkout("master")
def setUpClass(self):
conf = SafeConfigParser(allow_no_value=True)
if os.path.isfile("../private/localconfig"):
conf.read("../private/localconfig")
else:
conf.read("../private/config")
self.repo = conf.get("apis","repo_path")
self.gd = GitData(repo=self.repo)
self.orig_cwd = os.getcwd()
# go into our data repo
os.chdir(self.repo)
self.testing_branch_name = "testing_%d" % int(time.time())
# create the branch
git.checkout("-b", self.testing_branch_name)
# start all tests on the master branch
git.checkout("master")
