def main():
usage = "Usage: %prog <prevrev> <currev>"
parser = OptionParser(usage=usage)
(options, args) = parser.parse_args()
if len(args) != 2:
parser.error("incorrect number of arguments")
print "Repository at: " + REPO_DIR
repo = git.bake(_cwd=REPO_DIR, _tty_out=False)
# just to verify
repo.status()
print "Repository verified."
commits = repo.log(args[0] + '..' + args[1], '--pretty=format:%h',
'--no-color').splitlines()
print "Number of revisions: " + ` len(commits) `
count = 0
for revision in commits:
diffs = repo.show(revision, '--oneline').splitlines()
# skip first 5 lines
for line in diffs[5:]:
if line.startswith("-") or line.startswith("+"):
count += 1
print "Lines changed: " + ` count `
def rollback(self, paths, msg="Rolling-Back"):
"""
Rollback last commit to restore previous.
configuration and commit changes automatically
"""
for path in paths:
global git
wogit = git.bake("-C", "{0}".format(path))
if os.path.isdir(path):
if not os.path.isdir(path + "/.git"):
Log.error(
self, "Unable to find a git repository at {0}"
.format(path))
try:
Log.debug(
self, "WOGit: git stash --include-untracked at {0}"
.format(path))
wogit.stash("push", "--include-untracked", "-m {0}"
.format(msg))
except ErrorReturnCode as e:
Log.debug(self, "{0}".format(e))
Log.error(self, "Unable to git reset at {0} "
.format(path))
else:
Log.debug(self, "WOGit: Path {0} not present".format(path))
def find_repo_unmerged_branches(repository_path):
git_branches = git.bake('branch', '-a', '--no-color', '--no-merged',
'origin/master')
branches = imap(_clean_branch_name,
git_branches(_cwd=repository_path, _iter=True))
for branch in branches:
yield branch
def add(self, paths, msg="Intializating"):
"""
Initializes Directory as repository if not already git repo.
and adds uncommited changes automatically
"""
for path in paths:
global git
git = git.bake("--git-dir={0}/.git".format(path),
"--work-tree={0}".format(path))
if os.path.isdir(path):
if not os.path.isdir(path+"/.git"):
try:
Log.debug(self, "EEGit: git init at {0}"
.format(path))
git.init(path)
except ErrorReturnCode as e:
Log.debug(self, "{0}".format(e))
Log.error(self, "Unable to git init at {0}"
.format(path))
status = git.status("-s")
if len(status.splitlines()) > 0:
try:
Log.debug(self, "EEGit: git commit at {0}"
.format(path))
git.add("--all")
git.commit("-am {0}".format(msg))
except ErrorReturnCode as e:
Log.debug(self, "{0}".format(e))
Log.error(self, "Unable to git commit at {0} "
.format(path))
else:
Log.debug(self, "EEGit: Path {0} not present".format(path))
def find_repo_merged_branches(repository_path):
git_branches = git.bake('branch', '-a', '--no-color', '--merged',
'origin/master')
for raw_branch in git_branches(_cwd=repository_path, _iter=True):
if 'master' in raw_branch:
continue
yield _clean_branch_name(raw_branch)
def add(self, paths, msg="Intializating"):
"""
Initializes Directory as repository if not already git repo.
and adds uncommited changes automatically
"""
for path in paths:
global git
git = git.bake("--git-dir={0}/.git".format(path),
"--work-tree={0}".format(path))
if os.path.isdir(path):
if not os.path.isdir(path + "/.git"):
try:
Log.debug(self, "EEGit: git init at {0}".format(path))
git.init(path)
except ErrorReturnCode as e:
Log.debug(self, "{0}".format(e))
Log.error(self,
"Unable to git init at {0}".format(path))
status = git.status("-s")
if len(status.splitlines()) > 0:
try:
Log.debug(self,
"EEGit: git commit at {0}".format(path))
git.add("--all")
git.commit("-am {0}".format(msg))
except ErrorReturnCode as e:
Log.debug(self, "{0}".format(e))
Log.error(self,
"Unable to git commit at {0} ".format(path))
else:
Log.debug(self, "EEGit: Path {0} not present".format(path))
def main():
usage = "Usage: %prog <prevrev> <currev>"
parser = OptionParser(usage=usage)
(options, args) = parser.parse_args()
if len(args) != 2:
parser.error("incorrect number of arguments")
print "Repository at: " + REPO_DIR
repo = git.bake(_cwd=REPO_DIR, _tty_out=False)
# just to verify
repo.status()
print "Repository verified."
commits = repo.log(args[0] + '..' + args[1], '--pretty=format:%h', '--no-color').splitlines()
print "Number of revisions: " + `len(commits)`
count = 0
for revision in commits:
diffs = repo.show(revision, '--oneline').splitlines()
# skip first 5 lines
for line in diffs[5:]:
if line.startswith("-") or line.startswith("+"):
count += 1
print "Lines changed: " + `count`
def fetch_merged_branches(branches, base_branch, repo_path):
print(f" Branches merged into {base_branch}:")
merged_branch_count = 0
gitsh = git.bake(_cwd=repo_path, _tty_out=False)
for branch, commitId in branches.items():
# skip when master == master, for example
if branch == base_branch:
log.debug(f"Skipping {branch} == {base_branch}")
continue
buf = StringIO()
gitsh.branch("-a", "--contains", commitId, _out=buf)
lines = buf.getvalue()
buf.close()
if base_branch in lines:
print(f" {branch:20} has been merged into {base_branch}")
merged_branch_count += 1
else:
log.debug(
f" {branch:20} has not yet been merged into {base_branch}"
)
if merged_branch_count == 0:
print(f" No merged branches found")
def generate_csv(output_directory='./'):
# Set up root data dir
root_filename = ROOT + '/cfb-data/automated/wiki/' + ROOT + '/'
# Clone data repository for updates - continue if already exists
from sh import git
try:
shutil.rmtree(ROOT + '/cfb-data/')
except:
pass
git.clone(
'https://' + os.getenv('MACHINE_AUTH') +
'@github.com/coffenbacher/cfb-data.git', ROOT + '/cfb-data/')
# Create our root if required
if not os.path.exists(root_filename):
os.makedirs(root_filename)
# Set up logging
logging.basicConfig(
level='WARNING',
#format='%(asctime)s %(levelname)-8s %(message)s',
#datefmt='%a, %d %b %Y %H:%M:%S',
filename=root_filename + ROOT + '.log',
filemode='w')
# Extract all current names from Wiki
data = FN()
# Write everything to csv
with open(root_filename + ROOT + '.csv', 'w') as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=FIELDNAMES,
extrasaction='ignore')
for d in data:
asci = dict([(unidecode(k), unidecode(unicode(v)))
for k, v in d.items()])
writer.writerow(asci)
# [unidecode(unicode(d.get(field))) for field in FIELDNAMES]
# Write everything to json
with open(root_filename + ROOT + '.json', 'w') as jsonfile:
relevant = [{f: d.get(f, None) for f in FIELDNAMES} for d in data]
jsonfile.write(json.dumps(relevant))
with open(root_filename + ROOT + '_meta.json', 'w') as metafile:
d = {
'created': datetime.now().strftime('%x %X'),
'rows': len(data),
'headers': ','.join(FIELDNAMES),
}
metafile.write(json.dumps(d))
git = git.bake(**{
'git-dir': ROOT + '/cfb-data/.git/',
'work-tree': ROOT + '/cfb-data'
})
git.commit(m='Auto updating %s data' % ROOT, a=True)
git.push('origin', 'master')
def git_is_repo_dir(file_path):
'''Return git root dir name'''
rev_parse = git.bake('rev-parse')
try:
roots = rev_parse('--show-toplevel', _cwd=dir_path(file_path))
return bool(next(fil_clean_output(roots)))
except Exception as e:
return False
def git_is_repo_dir(file_path):
'''Return git root dir name'''
rev_parse = git.bake('rev-parse')
try:
roots = rev_parse('--show-toplevel', _cwd=dir_path(file_path))
return bool(fil_clean_output(roots).next())
except Exception as e:
return False
def main():
server.git = git.bake(_cwd=server.settings.repo)
if options.listen_localhost_only:
server.listen(options.port, 'localhost')
else:
server.listen(options.port)
autoreload.start()
autoreload.watch(config_path.web)
IOLoop.instance().start()
def _update_repo(self, git_repo):
git_repo_dir = self.env_path('source', git_repo)
if not os.path.isdir(git_repo_dir):
logger.warning('%s does not exist. Cloning.', git_repo)
repos.git_clone_to_source(self._env_dir, git_repo)
return
try:
logger.info('Updating %s', git_repo)
git.bake(C=git_repo_dir).pull(_out=sys.stdout,
_err=sys.stderr,
_fg=True)
except ErrorReturnCode as e:
raise CommandError(e)
# Success!
logger.info('Updated %s', git_repo)
def setUp(self):
super(Test, self).setUp()
self.git = git.bake(_cwd=self.path, _tty_out=False)
self.notebook_name = "git_tests"
# make notebook
if not path.exists(path.join(self.path, self.notebook_name)):
res = self.post("/" + self.notebook_name)
def setUp(self):
super(Test, self).setUp()
self.git = git.bake(_cwd=self.path)
self.notebook_name = 'git_tests'
# make notebook
if not path.exists(path.join(self.path, self.notebook_name)):
res = self.post('/' + self.notebook_name)
def fetch_first_commit_id(branch, base_branch, commitId):
"""
From https://stackoverflow.com/questions/18407526/git-how-to-find-first-commit-of-specific-branch/32870852#32870852
A-B-C-D-E (base_branch)
\
F-G-H (branch)
We are looking for commit F.
git log base_branch..branch --oneline | tail -1
"dot-dot gives you all of the commits that the branch has that base_branch
doesn't have." These commits are listed in reverse topological order, so
the last one in the list is the first commit on the branch.
The downside of this approach is that we must specify the base_branch
from which 'branch' was branched. Ideally, base_branch wouldn't be
required.
This technique is good enough to start. Other options exist if this doesn't
hold up.
"""
first_commit_id = None
buf = StringIO()
gitsh = git.bake(_cwd=repo_path, _tty_out=False)
gitsh.log(f"{base_branch}..{commitId}", "--format=%h %s", _out=buf)
""" Output looks like:
$ git log master..20180905-foo --oneline
124e842 2nd commit on 20180905-foo
cdb7133 1st commit on 20180905-foo
"""
lines = buf.getvalue().splitlines()
if len(lines) <= 0:
log.debug(
f"Found no commits on {branch} that are not on {base_branch}")
else:
# The first commit on the branch is listed last in the output, so index
# the output by [-1] to find it.
# Split the line into only 2 parts: the commitId and the commit subject.
fields = lines[-1].split(' ', 1)
if len(fields) < 2:
log.warning(
f"Unexpected number of fields = {len(fields)}. Skipping...")
else:
log.debug(
f"First commit on {branch} is {fields[0]} w/ subject '{fields[1]}'"
)
first_commit_id = fields[0]
buf.close()
return first_commit_id
def checkfilestatus(self, repo, filepath):
"""
Checks status of file, If its tracked or untracked.
"""
global git
git = git.bake("--git-dir={0}/.git".format(repo), "--work-tree={0}".format(repo))
status = git.status("-s", "{0}".format(filepath))
if len(status.splitlines()) > 0:
return True
else:
return False
def _git_info(path):
from sh import git
git = git.bake('--no-pager')
rev_parse = getattr(git, 'rev-parse')
R = {'branch': rev_parse('--abbrev-ref', 'HEAD', _cwd=path).strip(),
'revision': rev_parse('HEAD', _cwd=path).strip(),
'untracked': getattr(git, 'ls-files')('--others', '--exclude-standard', '--directory', _cwd=path).strip(),
'status': git.status('-s', _cwd=path).strip(),
'diff': git.diff(_cwd=path).strip()}
R['clean'] = len(R['diff']) == 0
return R
def setUpClass(cls):
super(BasicSanityTest, cls).setUpClass()
# ONLY FOR LOCAL
if settings.ENVIRON == HWCentralEnv.LOCAL:
# switch the cabinet to the sanity test branch
user = getpass.getuser()
git_cabinet = git.bake(_cwd='/home/%s/hwcentral-cabinet' % user)
git_cabinet.stash('save',
'-u') # stash changes, including untracked files
git_cabinet.fetch()
git_cabinet.checkout('sanity_test')
def _git_info(path):
from sh import git
git = git.bake('--no-pager')
rev_parse = getattr(git, 'rev-parse')
R = {'branch': rev_parse('--abbrev-ref', 'HEAD', _cwd=path).strip(),
'revision': rev_parse('HEAD', _cwd=path).strip(),
'untracked': getattr(git, 'ls-files')('--others', '--exclude-standard', '--directory', _cwd=path).strip(),
'status': git.status('-s', _cwd=path).strip(),
'diff': git.diff(_cwd=path).strip()}
R['clean'] = len(R['diff']) == 0
return R
def checkfilestatus(self, repo, filepath):
"""
Checks status of file, If its tracked or untracked.
"""
global git
wogit = git.bake("-C", "{0}".format(repo))
status = wogit.status("-s", "{0}".format(filepath))
if len(status.splitlines()) > 0:
return True
else:
return False
def make_app(config=None):
root = path.dirname(__file__)
static_path = path.join(root, 'static')
template_path = path.join(root, 'template')
app_config = dict(static_path=static_path,
template_path=template_path)
define('port', default='8080', type=int)
define('address', default='localhost', type=str)
define('testing', default=False, type=bool)
define('repo', default=None, type=str)
define('username', default=None, type=str)
define('pwdhash', default=None, type=str)
define('autosave', default=False, type=bool)
define('prefix', default='/', type=str)
define('autosave_interval', default='5', type=int)
define('wysiwyg', default=False, type=bool)
define('theme', default=None, type=str)
if config is not None:
# This should only ever be used for testing
parse_config_file(config)
else:
parse_config_file(config_path.web)
if options.testing:
app_config.update(debug=True)
if not options.prefix.endswith('/'):
options.prefix += '/'
urls = get_urls(options.prefix)
app_config.update(
login_url=options.prefix + 'login',
static_url_prefix=options.prefix + 'static/')
server = Application(urls, **app_config)
server.settings = AttrDict(server.settings)
server.settings.repo = options.repo
server.settings.username = options.username
server.settings.pwdhash = options.pwdhash
server.settings.session = _rand_str()
server.settings.config_path = config_path
server.settings.autosave = options.autosave
server.settings.prefix = options.prefix
server.settings.autosave_interval = options.autosave_interval
server.settings.wysiwyg = options.wysiwyg
server.settings.theme = options.theme
server.git = git.bake(_cwd=server.settings.repo)
return server
def checkfilestatus(self, repo, filepath):
"""
Checks status of file, If its tracked or untracked.
"""
global git
git = git.bake("--git-dir={0}/.git".format(repo),
"--work-tree={0}".format(repo))
status = git.status("-s", "{0}".format(filepath))
if len(status.splitlines()) > 0:
return True
else:
return False
def tearDownClass(cls):
super(BasicSanityTest, cls).setUpClass()
# ONLY FOR LOCAL
if settings.ENVIRON == HWCentralEnv.LOCAL:
# switch the cabinet back to the master branch
user = getpass.getuser()
git_cabinet = git.bake(_cwd='/home/%s/hwcentral-cabinet' % user)
# cleaning up any changes made
git_cabinet.clean('-df')
git_cabinet.reset('--hard', 'HEAD')
git_cabinet.checkout('master')
git_cabinet.stash('apply')
def __init__(self, plugin_manager, config):
threading.Thread.__init__(self)
self.queue = Queue.Queue()
mustdoinit = False
self.repopath = config.get("repo_directory", "./repo_submissions")
if not os.path.exists(self.repopath):
mustdoinit = True
os.mkdir(self.repopath)
self.git = git.bake('--work-tree=' + self.repopath, '--git-dir=' + os.path.join(self.repopath, '.git'))
if mustdoinit:
self.git.init()
plugin_manager.add_hook('submission_done', self.add)
print "SubmissionGitSaver started"
def fetch_branch_authors(branches, repo_path):
print(" Branch authors:")
gitsh = git.bake(_cwd=repo_path, _tty_out=False)
for ref, commitId in branches.items():
buf = StringIO()
gitsh.log("-1", "--pretty=format:%an", commitId, _out=buf)
author = buf.getvalue()
buf.close()
print(f" {ref:20} most recent author is {author}")
def _update_img_sources(self):
"""
Update the S2E image repositories.
"""
git_repos = CONSTANTS['repos']['images'].values()
for git_repo in git_repos:
git_repo_dir = self.env_path('source', git_repo)
if not os.path.isdir(git_repo_dir):
logger.warning('%s does not exist. Skipping', git_repo)
continue
try:
logger.info('Updating %s', git_repo)
git.bake(C=git_repo_dir).pull(_out=sys.stdout,
_err=sys.stderr,
_fg=True)
except ErrorReturnCode as e:
raise CommandError(e)
# Success!
logger.info('Updated %s', git_repo)
def __init__(self, plugin_manager, config):
threading.Thread.__init__(self)
self._logger = logging.getLogger("inginious.webapp.plugins.SubmissionGitSaver")
self.queue = queue.Queue()
mustdoinit = False
self.repopath = config.get("repo_directory", "./repo_submissions")
if not os.path.exists(self.repopath):
mustdoinit = True
os.mkdir(self.repopath)
self.git = git.bake('--work-tree=' + self.repopath, '--git-dir=' + os.path.join(self.repopath, '.git'))
if mustdoinit:
self.git.init()
plugin_manager.add_hook('submission_done', self.add)
self._logger.info("SubmissionGitSaver started")
def fetch_commit_date(commit_id, repo_path):
commit_date = None
buf = StringIO()
gitsh = git.bake(_cwd=repo_path, _tty_out=False)
gitsh.log("-n 1", "--format=%ci", commit_id, _out=buf)
lines = buf.getvalue().splitlines()
if len(lines) != 1:
log.warning(f"Unexpected git log output: {lines}. Skipping ...")
else:
log.debug(f"First commit was created at {lines[0]}")
commit_date = lines[0]
return commit_date
def pull_git_templates(self):
self._git["gitTemplateList"].pull()
for template_prefix, repo in self._get_git_template_list().items():
templateid = template_prefix + "_g1t"
if templateid not in self._git:
self._git[templateid] = git.bake(
'--work-tree=' +
os.path.join(self._filesystem.prefix, templateid),
'--git-dir=' +
os.path.join(self._filesystem.prefix, templateid, '.git'))
path = os.path.join(self._filesystem.prefix, templateid)
ensure_git_exist(repo, path)
self._git[templateid].pull()
def _add_and_commit_site(self, path):
git_wt = git.bake(work_tree = path)
git_wt.add('.',ignore_removal=True)
status = git.st('-s', '-uno', '--porcelain')
if status:
try:
message = "Published {0} to Github.".format(self.target_branch)
git_wt.commit(m=message)
except sh.ErrorReturnCode_1:
print("Can't commit.")
else:
print('no changes to publish')
git.reset(hard=True)
def __init__(self,
filesystem: FileSystemProvider,
task_factory,
template_repo,
hook_manager,
course_class=Template):
super().__init__(filesystem, task_factory, hook_manager, course_class)
git_list_path = os.path.join(self._filesystem.prefix,
"gitTemplateList")
ensure_git_exist(template_repo, git_list_path)
self._git = {
"gitTemplateList":
git.bake('--work-tree=' + git_list_path,
'--git-dir=' + os.path.join(git_list_path, '.git'))
}
self.pull_git_templates()
def make_app(config=None):
root = path.dirname(__file__)
static_path = path.join(root, 'static')
template_path = path.join(root, 'template')
app_config = dict(static_path=static_path,
template_path=template_path,
login_url='/login')
define('port', default='8080', type=int)
define('address', default='localhost', type=str)
define('testing', default=False, type=bool)
define('repo', default=None, type=str)
define('username', default=None, type=str)
define('pwdhash', default=None, type=str)
define('autosave', default=False, type=bool)
define('autosave_interval', default='5', type=int)
define('wysiwyg', default=False, type=bool)
if config is not None:
# This should only ever be used for testing
parse_config_file(config)
else:
parse_config_file(config_path.web)
if options.testing:
app_config.update(debug=True)
server = Application(urls, **app_config)
server.settings = AttrDict(server.settings)
server.settings.repo = options.repo
server.settings.username = options.username
server.settings.pwdhash = options.pwdhash
server.settings.session = _rand_str()
server.settings.config_path = config_path
server.settings.autosave = options.autosave
server.settings.autosave_interval = options.autosave_interval
server.settings.wysiwyg = options.wysiwyg
server.git = git.bake(_cwd=server.settings.repo)
return server
def clone(self):
self.tmp_dir = tempfile.mkdtemp(suffix='facio')
try:
from sh import git
except ImportError:
raise Exception # TODO: Custom exception
try:
git = git.bake(_cwd=self.tmp_dir)
git.clone(self.repo, self.tmp_dir)
git.fetch('--all')
git.checkout('master') # TODO: Branch prompt to the user
except Exception:
raise Exception # TODO: Custom exception
rmtree(os.path.join(self.tmp_dir, '.git'))
with indent(4, quote=' >'):
puts(blue('Clone complete'))
def __init__(self,
path,
repo_name,
clone_url=None,
branch="master",
bare=False):
"""Creates a new repository in the directory @path/@repo_name.
If @clone_url is not None, it creates it by cloning it
from @clone_url. Otherwise it initializes a new repo.
The repository will be bare if @bare is True."""
self._workdir = os.path.join(path, repo_name)
if not os.path.isdir(self._workdir):
self._remote = clone_url
if self._remote:
if branch is not None:
git.clone("--branch",
branch,
self._remote,
self._workdir,
bare=bare)
else:
git.clone(self._remote, self._workdir, bare=bare)
else:
git.init(self._workdir, bare=bare)
else:
if os.path.isfile(os.path.join(self._workdir, '.git/config')):
cfg = os.path.join(self._workdir, '.git/config')
else:
cfg = os.path.join(self._workdir, 'config')
config = open(cfg).read()
m = re.match('^.*\s*url\s*=\s*(?P<url>[^\s]*)\s.*$', config,
re.IGNORECASE | re.DOTALL)
if m:
self._remote = m.groupdict()['url']
else:
self._remote = None
self._git = git.bake(_cwd=self._workdir, _tty_out=False)
def make_app(config=None):
root = path.dirname(__file__)
static_path = path.join(root, 'static')
template_path = path.join(root, 'template')
app_config = dict(static_path=static_path,
template_path=template_path,
login_url='/login')
define('port', default='8080', type=int)
define('address', default='localhost', type=str)
define('testing', default=False, type=bool)
define('repo', default=None, type=str)
define('username', default=None, type=str)
define('pwdhash', default=None, type=str)
define('autosave', default=False, type=bool)
if config is not None:
# This should only ever be used for testing
parse_config_file(config)
else:
parse_config_file(config_path.web)
if options.testing:
app_config.update(debug=True)
server = Application(urls, **app_config)
server.settings = AttrDict(server.settings)
server.settings.repo = options.repo
server.settings.username = options.username
server.settings.pwdhash = options.pwdhash
server.settings.session = _rand_str()
server.settings.config_path = config_path
server.settings.autosave = options.autosave
server.git = git.bake(_cwd=server.settings.repo)
return server
def auto_update() -> bool:
"""Updates the application from a github repo."""
add_remote = git.bake('remote', 'add', '-f', '--tags', '-m', 'master',
'-t', 'master', 'origin',
'https://github.com/threadreaper/autodomme.git')
if '.git' not in os.listdir(os.getcwd()):
git('init')
add_remote()
git.branch('--set-upstream-to', 'origin/master')
elif git('remote').exit_code != 0:
add_remote()
git.branch('--set-upstream-to', 'origin/master')
files = [
line for line in git('--no-pager', 'diff', '--name-only', _iter=True)
]
if files:
filenames = [filter_filename(file) for file in files[:-1]]
update_popup(filenames)
return True
else:
return False
def clone(self):
""" Clone the git repository into a temporary directory. """
try:
from sh import git
except ImportError:
raise FacioException('Git must be installed to use git+ '
'template paths')
temp_diretory = self.get_temp_directory()
self.out('Git Cloning {0} to {1}'.format(self.path, temp_diretory))
try:
git = git.bake(_cwd=temp_diretory)
git.clone(self.path, temp_diretory)
git.fetch('--all')
git.checkout('master')
except:
raise FacioException('Failed to clone git repository '
'at {0}'.format(self.path))
return temp_diretory
def rollback(self, paths, msg="Rolling-Back"):
"""
Rollback last commit to restore previous.
configuration and commit changes automatically
"""
for path in paths:
global git
git = git.bake("--git-dir={0}/.git".format(path),
"--work-tree={0}".format(path))
if os.path.isdir(path):
if not os.path.isdir(path + "/.git"):
Log.error(
self,
"Unable to find a git repository at {0}".format(path))
try:
Log.debug(self,
"WOGit: git reset HEAD~ at {0}".format(path))
git.reset("HEAD~", "--hard")
except ErrorReturnCode as e:
Log.debug(self, "{0}".format(e))
Log.error(self, "Unable to git reset at {0} ".format(path))
else:
Log.debug(self, "WOGit: Path {0} not present".format(path))
def find_repo_unmerged_branches(repository_path):
git_branches = git.bake('branch', '-a', '--no-color', '--no-merged', 'origin/master')
branches = imap(_clean_branch_name, git_branches(_cwd=repository_path, _iter=True))
for branch in branches:
yield branch
def switch_current_branch(self, dst_b, move_over=False):
"""Switches to the given branch.
Args:
dst_b: the destination branch.
move_over: if True, then uncommited changes made in the current branch are
moved to the destination branch (defaults to False).
"""
if dst_b.is_current:
raise ValueError(
'You are already on branch {0}. No need to switch.'.format(
dst_b.branch_name))
INFO_SEP = '|'
ANCESTOR = 'ancestor'
THEIRS = 'theirs'
OURS = 'ours'
REF_INFO = 'ref_info'
CONF_INFO = 'conf_info'
MSG_INFO = 'msg_info'
git_repo = self.git_repo
au_fp = lambda b: os.path.join(
self.path, 'GL_AU_{0}'.format(b.branch_name.replace('/', '_')))
update_index = git.bake('update-index', _cwd=self.root)
def save(b):
msg = _stash_msg(b.branch_name)
# Save assumed unchanged info
au_fps = ' '.join(b._au_files())
if au_fps:
with io.open(au_fp(b), mode='w', encoding=ENCODING) as f:
f.write(au_fps)
update_index('--no-assume-unchanged', au_fps)
if b.merge_in_progress or b.fuse_in_progress:
body = {}
if move_over:
raise GlError(
'Changes can\'t be moved over with a fuse or merge in progress')
# Save msg info
merge_msg_fp = os.path.join(self.path, 'MERGE_MSG')
with io.open(merge_msg_fp, 'r', encoding=ENCODING) as f:
merge_msg = f.read()
os.remove(merge_msg_fp)
body[MSG_INFO] = merge_msg
# Save conflict info
conf_info = {}
index = git_repo.index
index.read()
if index.conflicts:
extract = lambda e: {'mode': e.mode, 'id': str(e.id), 'path': e.path}
for ancestor, ours, theirs in index.conflicts:
if ancestor:
path = ancestor.path
ancestor = extract(ancestor)
if theirs:
path = theirs.path
theirs = extract(theirs)
if ours:
path = ours.path
ours = extract(ours)
conf_info[path] = {ANCESTOR: ancestor, THEIRS: theirs, OURS: ours}
index.add(path)
index.write()
body[CONF_INFO] = conf_info
# Save ref info
if b.merge_in_progress:
ref_info = {'MERGE_HEAD': str(self._ref_target('MERGE_HEAD'))}
self._ref_rm('MERGE_HEAD')
else:
ref_info = {
'HEAD': str(git_repo.head.target),
'GL_FUSE_ORIG_HEAD': str(self._ref_target('GL_FUSE_ORIG_HEAD')),
'CHERRY_PICK_HEAD': str(self._ref_target('CHERRY_PICK_HEAD'))
}
self._ref_rm('GL_FUSE_ORIG_HEAD')
self._ref_rm('CHERRY_PICK_HEAD')
body[REF_INFO] = ref_info
msg += INFO_SEP + json.dumps(body)
if not move_over:
# Stash
git.stash.save('--all', '--', msg)
def restore(b):
s_id, msg = _stash(_stash_msg(b.branch_name))
if not s_id:
return
def restore_au_info():
au = au_fp(b)
if os.path.exists(au):
with io.open(au, mode='r', encoding=ENCODING) as f:
au_fps = f.read()
update_index('--assume-unchanged', au_fps)
os.remove(au)
split_msg = msg.split(INFO_SEP)
if len(split_msg) == 1: # No op to restore
# Pop
git.stash.pop(s_id)
# Restore assumed unchanged info
restore_au_info()
else: # Restore op
body = json.loads(split_msg[1])
# Restore ref info
ref_info = body[REF_INFO]
if 'GL_FUSE_ORIG_HEAD' in ref_info: # fuse
head = git_repo[ref_info['HEAD']]
git_repo.set_head(head.id)
git_repo.reset(head.id, pygit2.GIT_RESET_HARD)
self._ref_create('CHERRY_PICK_HEAD', ref_info['CHERRY_PICK_HEAD'])
self._ref_create('GL_FUSE_ORIG_HEAD', ref_info['GL_FUSE_ORIG_HEAD'])
else: # merge
self._ref_create('MERGE_HEAD', ref_info['MERGE_HEAD'])
# Pop
git.stash.pop(s_id)
# Restore conflict info
conf_info = body[CONF_INFO]
rm_sentinel = lambda path: '0 {0}\t{1}'.format('0' * 40, path)
build_entry = (
lambda e, num: '{mode:o} {id} {0}\t{path}'.format(num, **e))
index_info = []
for path, index_e in conf_info.items():
index_info.append(rm_sentinel(path))
if index_e[ANCESTOR]:
index_info.append(build_entry(index_e[ANCESTOR], 1))
if index_e[OURS]:
index_info.append(build_entry(index_e[OURS], 2))
if index_e[THEIRS]:
index_info.append(build_entry(index_e[THEIRS], 3))
update_index('--unresolve', _in=' '.join(conf_info.keys()))
update_index('--index-info', _in='\n'.join(index_info))
# Restore msg info
merge_msg_fp = os.path.join(self.path, 'MERGE_MSG')
with io.open(merge_msg_fp, 'w', encoding=ENCODING) as f:
f.write(body[MSG_INFO])
# Restore assumed unchanged info
restore_au_info()
save(self.current_branch)
git_repo.checkout(dst_b.git_branch)
restore(dst_b)
#!/usr/bin/env python
import collections
import datetime
import fnmatch
import json
import os
import os.path
import re
from os.path import expanduser
from configobj import ConfigObj
from sh import git
git = git.bake('--no-pager')
# this regex is intended to match strings like this:
# 4f994a3ff5dc62e01a5967f48c5883e60a53890f (remotes/svn/foobar~1)
# the first match should be the hash ref
# the second match should be the ref name without the trailing tilde and
# digits:
HASH_NAME_RE = re.compile(r'\b([0-9a-f]{40})\s\((.*?)(~\d+)?\)')
def list_of_local_repos(dirs):
git_repos = list()
for dir in dirs:
for root, dirnames, filenames in os.walk(os.path.expanduser(dir)):
for filename in fnmatch.filter(dirnames, '.git'):
git_repos.append(os.path.join(root, filename))
return git_repos
# -*- coding: utf-8 -*-
import os
import shutil
from sh import git
from mock import patch, PropertyMock
from django.test import TestCase
from django.core.urlresolvers import reverse
from waliki.models import Page
from waliki.git.models import Git
from waliki.settings import WALIKI_DATA_DIR, WALIKI_COMMITTER_EMAIL, WALIKI_COMMITTER_NAME
from .factories import PageFactory
git = git.bake("--no-pager", _tty_out=False)
class TestGit(TestCase):
def setUp(self):
self.page = PageFactory()
self.edit_url = reverse('waliki_edit', args=(self.page.slug, ))
def test_init_git_create_repo(self):
git_dir = os.path.join(WALIKI_DATA_DIR, '.git')
shutil.rmtree(git_dir)
Git()
self.assertTrue(os.path.isdir(git_dir))
self.assertEqual(
git.config('user.name').stdout.decode('utf8')[:-1],
WALIKI_COMMITTER_NAME)
self.assertEqual(
git.config('user.email').stdout.decode('utf8')[:-1],
WALIKI_COMMITTER_EMAIL)
def git_repo(repopath=repopath):
return git.bake(_cwd=repopath)
def find_repo_merged_branches(repository_path):
git_branches = git.bake('branch', '-a', '--no-color', '--merged', 'origin/master')
for raw_branch in git_branches(_cwd=repository_path, _iter=True):
if 'master' in raw_branch:
continue
yield _clean_branch_name(raw_branch)
from sh import git, ssh
branch = git.bake('branch')
myssh = ssh.bake('username@hostname')
def __init__(self, wiki_path):
self.path = wiki_path
self.git = git.bake(_cwd=self.path)
self.git.init()
self._set_config()
self._bootstrap()
from sh import git as _git, ErrorReturnCode # pylint: disable=no-name-in-module
import requests
from flask import current_app, request
from .. import URL
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
DATA = os.path.join(ROOT, "data")
GITHUB_BASE = "https://raw.githubusercontent.com/jacebrowning/coverage-space/master/"
CONTRIBUTING_URL = GITHUB_BASE + "CONTRIBUTING.md"
CHANGELOG_URL = GITHUB_BASE + "CHANGELOG.md"
log = logging.getLogger(__name__)
git = _git.bake(git_dir=os.path.join(DATA, ".git"), work_tree=DATA)
def sync(model, *, push=True):
"""Store all changes in version control."""
log.info("Saving changes...")
git.checkout('master')
git.add(all=True)
try:
git.commit(message=str(model))
except ErrorReturnCode:
log.info("No changes to save")
log.info("Pulling changes...")
try:
git.pull(rebase=True)
def _fetch_repo(repository_path):
git_fetch = git.bake('fetch', '-p')
git_fetch(_cwd=repository_path)
def git_root(file_path):
'''Return git root dir name'''
rev_parse = git.bake('rev-parse')
roots = rev_parse('--show-toplevel', _cwd=dir_path(file_path))
return fil_clean_output(roots).next()
import os
import re
import json
from django.contrib.auth.models import User
from django.utils import six
from sh import git
git = git.bake("--no-pager")
class Git(object):
__shared_state = {} # it's a Borg
def __init__(self):
self.__dict__ = self.__shared_state
from waliki.settings import WALIKI_DATA_DIR
self.content_dir = WALIKI_DATA_DIR
os.chdir(self.content_dir)
if not os.path.isdir(os.path.join(self.content_dir, '.git')):
git.init()
self.commit('.', 'initial commit')
def commit(self, path, message='', author=None):
kwargs = {}
if isinstance(author, User) and author.is_authenticated():
kwargs['author'] = "% <%s>" % (author.get_full_name() or author.username)
elif isinstance(author, six.string_types):
kwargs['author'] = author
try:
git.add(path)
git.commit(path, m=message or 'Update %s' % path, **kwargs)
def switch_current_branch(self, dst_b, move_over=False):
"""Switches to the given branch.
Args:
dst_b: the destination branch.
move_over: if True, then uncommitted changes made in the current branch are
moved to the destination branch (defaults to False).
"""
if dst_b.is_current:
raise ValueError(
'You are already on branch {0}. No need to switch.'.format(
dst_b.branch_name))
INFO_SEP = '|'
ANCESTOR = 'ancestor'
THEIRS = 'theirs'
OURS = 'ours'
REF_INFO = 'ref_info'
CONF_INFO = 'conf_info'
MSG_INFO = 'msg_info'
git_repo = self.git_repo
au_fp = lambda b: os.path.join(
self.path, 'GL_AU_{0}'.format(b.branch_name.replace('/', '_')))
update_index = git.bake('update-index', _cwd=self.root)
def save(b):
msg = _stash_msg(b.branch_name)
# Save assumed unchanged info
au_fps = ' '.join(b._au_files())
if au_fps:
with io.open(au_fp(b), mode='w', encoding=ENCODING) as f:
f.write(au_fps)
update_index('--no-assume-unchanged', au_fps)
if b.merge_in_progress or b.fuse_in_progress:
body = {}
if move_over:
raise GlError(
'Changes can\'t be moved over with a fuse or merge in progress'
)
# Save msg info
merge_msg_fp = os.path.join(self.path, 'MERGE_MSG')
with io.open(merge_msg_fp, 'r', encoding=ENCODING) as f:
merge_msg = f.read()
os.remove(merge_msg_fp)
body[MSG_INFO] = merge_msg
# Save conflict info
conf_info = {}
index = git_repo.index
index.read()
if index.conflicts:
extract = lambda e: {
'mode': e.mode,
'id': str(e.id),
'path': e.path
}
for ancestor, ours, theirs in index.conflicts:
if ancestor:
path = ancestor.path
ancestor = extract(ancestor)
if theirs:
path = theirs.path
theirs = extract(theirs)
if ours:
path = ours.path
ours = extract(ours)
conf_info[path] = {
ANCESTOR: ancestor,
THEIRS: theirs,
OURS: ours
}
index.add(path)
index.write()
body[CONF_INFO] = conf_info
# Save ref info
if b.merge_in_progress:
ref_info = {
'MERGE_HEAD': str(self._ref_target('MERGE_HEAD'))
}
self._ref_rm('MERGE_HEAD')
else:
ref_info = {
'HEAD':
str(git_repo.head.target),
'GL_FUSE_ORIG_HEAD':
str(self._ref_target('GL_FUSE_ORIG_HEAD')),
'CHERRY_PICK_HEAD':
str(self._ref_target('CHERRY_PICK_HEAD'))
}
self._ref_rm('GL_FUSE_ORIG_HEAD')
self._ref_rm('CHERRY_PICK_HEAD')
body[REF_INFO] = ref_info
msg += INFO_SEP + json.dumps(body)
if not move_over:
# Stash
git.stash.save('--all', '--', msg)
def restore(b):
s_id, msg = _stash(_stash_msg(b.branch_name))
if not s_id:
return
def restore_au_info():
au = au_fp(b)
if os.path.exists(au):
with io.open(au, mode='r', encoding=ENCODING) as f:
au_fps = f.read()
update_index('--assume-unchanged', au_fps)
os.remove(au)
split_msg = msg.split(INFO_SEP)
if len(split_msg) == 1: # No op to restore
# Pop
git.stash.pop(s_id)
# Restore assumed unchanged info
restore_au_info()
else: # Restore op
body = json.loads(split_msg[1])
# Restore ref info
ref_info = body[REF_INFO]
if 'GL_FUSE_ORIG_HEAD' in ref_info: # fuse
head = git_repo[ref_info['HEAD']]
git_repo.set_head(head.id)
git_repo.reset(head.id, pygit2.GIT_RESET_HARD)
self._ref_create('CHERRY_PICK_HEAD',
ref_info['CHERRY_PICK_HEAD'])
self._ref_create('GL_FUSE_ORIG_HEAD',
ref_info['GL_FUSE_ORIG_HEAD'])
else: # merge
self._ref_create('MERGE_HEAD', ref_info['MERGE_HEAD'])
# Pop
git.stash.pop(s_id)
# Restore conflict info
conf_info = body[CONF_INFO]
rm_sentinel = lambda path: '0 {0}\t{1}'.format('0' * 40, path)
build_entry = (lambda e, num: '{mode:o} {id} {0}\t{path}'.
format(num, **e))
index_info = []
for path, index_e in conf_info.items():
index_info.append(rm_sentinel(path))
if index_e[ANCESTOR]:
index_info.append(build_entry(index_e[ANCESTOR], 1))
if index_e[OURS]:
index_info.append(build_entry(index_e[OURS], 2))
if index_e[THEIRS]:
index_info.append(build_entry(index_e[THEIRS], 3))
update_index('--unresolve', _in=' '.join(conf_info.keys()))
update_index('--index-info', _in='\n'.join(index_info))
# Restore msg info
merge_msg_fp = os.path.join(self.path, 'MERGE_MSG')
with io.open(merge_msg_fp, 'w', encoding=ENCODING) as f:
f.write(body[MSG_INFO])
# Restore assumed unchanged info
restore_au_info()
save(self.current_branch)
git_repo.checkout(dst_b.git_branch)
restore(dst_b)
def _handle_new_code(self, id_, obj=None, code=None):
if code is None:
code = master.models.Code.objects(id=id_)[0]
if not code.type.startswith("new_"):
return
code.type = code.type.replace("new_", "")
self._log.info("creating new code from template ({}, {})".format(code.name, code.type))
# TODO this should be in some central setting somewhere,
# e.g. master.settings.TALUS_GIT or something
tmpdir = tempfile.mkdtemp()
git.clone(TALUS_GIT, tmpdir)
self._log.info("cloned code into {}".format(tmpdir))
if code.type == "tool":
src_dir = os.path.join(tmpdir, "talus", "tools", "tool_template")
dest_path = os.path.join(tmpdir, "talus", "tools", self._camel_to_under(code.name))
replace_name = "ToolTemplate"
elif code.type == "component":
src_dir = os.path.join(tmpdir, "talus", "components", "component_template")
dest_path = os.path.join(tmpdir, "talus", "components", self._camel_to_under(code.name))
replace_name = "ComponentTemplate"
self._log.info("copying template to {}".format(dest_path))
shutil.copytree(src_dir, dest_path)
files = ["__init__.py", "requirements.txt", "run_local.py"]
for filename in files:
filepath = os.path.join(dest_path, filename)
if not os.path.exists(filepath):
continue
f = open(filepath, "rb+")
file_data = f.read()
f.seek(0)
f.truncate()
file_data = file_data.replace(replace_name, code.name)
f.write(file_data)
f.close()
git_ = git.bake("--git-dir", os.path.join(tmpdir, ".git"), "--work-tree", tmpdir, _tty_out=False)
git_.config("--local", "user.email", "master@talus")
git_.config("--local", "user.name", "Talus Master")
git_.add(dest_path)
git_.commit(message="auto created from template by TALUS")
git_.config("--local", "push.default", "matching")
try:
git_.push("origin")
except Exception as e:
self._log.error("ERROR COMMITTING CODE")
self._log.error(e)
self._log.error(e.stderr)
# the git commit will have saved it!!!
code.delete()
def __init__(self, addon): self.addon = addon self.source = addon['metadata']['source'] self.dir = os.path.join(CACHE_DIR, addon['id']) self.git = git.bake(_cwd=self.dir) self.reset()
#!/usr/bin/env python
# encoding: utf-8
import base64
import json
import os
from sh import git as GIT
from sh import ls
from django.http import HttpResponse
from django.shortcuts import render
code_path = "/code_cache/code"
git = GIT.bake("--git-dir", os.path.join(code_path, ".git"), "--work-tree", code_path, _tty_out=False)
def git_info(request, ref, path):
path = path
data = {"ref": ref, "path": path}
res = {}
res["filename"] = path
if path.startswith("talus/pypi"):
file_path = os.path.join(code_path, path)
if os.path.isdir(file_path):
res["type"] = "listing"
items = []
for item in os.listdir(file_path):
if os.path.isdir(os.path.join(file_path, item)):
item += "/"
items.append(item)
import os
import re
import json
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _
from django.utils import six
from waliki.models import Page
from waliki import settings
from sh import git, ErrorReturnCode, Command
from collections import namedtuple
git = git.bake("--no-pager", _tty_out=False)
Commit = namedtuple(
"Commit", ["hash", "author_name", "author_email", "subject", "date", "date_relative", "paths", "diff"]
)
class Git(object):
__shared_state = {} # it's a Borg
def __init__(self):
self.__dict__ = self.__shared_state
from waliki.settings import WALIKI_DATA_DIR
self.content_dir = WALIKI_DATA_DIR
os.chdir(self.content_dir)
if not os.path.isdir(os.path.join(self.content_dir, ".git")):
git.init()
git.config("user.email", settings.WALIKI_COMMITTER_EMAIL)
git.config("user.name", settings.WALIKI_COMMITTER_NAME)
move_shell_to
from aeriscloud.ansible import get_organization_list, get_env_path, \
organization_path
from aeriscloud.config import has_github_integration, aeriscloud_path
from aeriscloud.github import Github
@click.group()
def cli():
"""
Manage organizations
"""
pass
vgit = git.bake(_out=sys.stdout, _out_bufsize=0, _err_to_out=True)
def run_galaxy_install(organization):
roles_path = os.path.join(get_env_path(organization), 'roles')
dependencies_paths = [os.path.join(roles_path, '%s.%s' % (filename, ext))
for (filename, ext)
in product(['requirements', 'dependencies'],
['yml', 'txt'])]
for path in dependencies_paths:
if os.path.exists(path):
dependencies_path = path
break
else:
return
