def commit(options, debug, info):
path = getcwd()
git = Git(path)
debug(git.commit('-m', message(options['m'], 'commit')))
info('Committed with message "{message}".'.format(message=options['m']))
def gitPush(self, path, comment=None):
# Announce changes to git
cmd = Git(path)
changed = False
for line in cmd.status("--porcelain").splitlines():
changed = True
line = line.lstrip()
self.log.info("staging changes for %s" % line.split(" ", 1)[1:])
mode, file_path = line.split(" ", 1)
file_path = os.path.join(*file_path.split("/")[0:])
# Remove data?
if mode == "D":
cmd.rm("-r", file_path)
# Add data...
else:
cmd.add(file_path)
# No need to deal...
if not changed:
return False
# Commit changes
if not comment:
comment = "Change made with no comment"
self.log.info("committing changes for module %s" % path)
cmd.commit("-m", comment)
cmd.push("origin")
return True
def init(cls, path=None, mkdir=True, odbt=DefaultDBType, **kwargs):
"""Initialize a git repository at the given path if specified
:param path:
is the full path to the repo (traditionally ends with /<name>.git)
or None in which case the repository will be created in the current
working directory
:parm mkdir:
if specified will create the repository directory if it doesn't
already exists. Creates the directory with a mode=0755.
Only effective if a path is explicitly given
:param odbt:
Object DataBase type - a type which is constructed by providing
the directory containing the database objects, i.e. .git/objects.
It will be used to access all object data
:parm kwargs:
keyword arguments serving as additional options to the git-init command
:return: ``git.Repo`` (the newly created repo)"""
if path:
path = _expand_path(path)
if mkdir and path and not os.path.exists(path):
os.makedirs(path, 0o755)
# git command automatically chdir into the directory
git = Git(path)
git.init(**kwargs)
return cls(path, odbt=odbt)
def get_days_since_last_update(path):
"""
:return: The days since the last update of any of the files in path. If the
path points to a filename, the last update of that filename is
returned.
"""
git = Git(".")
cmd_str = "git log -1 --format=%%cd %s" % path
cmd = shlex.split(cmd_str)
try:
date_str = git.execute(command=cmd, with_extended_output=False)
except GitCommandError:
raise ValueError('"%s" is not in tracked by this repository.' % path)
# The date_str is in the following format: Sat Jun 21 10:20:31 2014 -0300
# We need to parse it, and then do some date math to return the result
#
# We ignore the UTC offset because it was "hard to parse" and we don't care
last_commit_time = datetime.strptime(date_str[:-6], "%a %b %d %H:%M:%S %Y")
last_commit_date = last_commit_time.date()
today_date = date.today()
time_delta = today_date - last_commit_date
return time_delta.days
def init(cls, path=None, mkdir=True, **kwargs): """Initialize a git repository at the given path if specified :param path: is the full path to the repo (traditionally ends with /<name>.git) or None in which case the repository will be created in the current working directory :parm mkdir: if specified will create the repository directory if it doesn't already exists. Creates the directory with a mode=0755. Only effective if a path is explicitly given :parm kwargs: keyword arguments serving as additional options to the git-init command :return: ``git.Repo`` (the newly created repo)""" if mkdir and path and not os.path.exists(path): os.makedirs(path, 0755) # git command automatically chdir into the directory git = Git(path) output = git.init(**kwargs) return Repo(path)
def fork(self, new_body):
git_path = mkdtemp(prefix=settings.GIT_ROOT_PATH)
git = Git(git_path)
git.clone(os.path.abspath(self.git_path), git_path)
new_fork = Fork.objects.create(body=new_body, parent=self, git_path=git_path)
return new_fork
def removeItem(self, release, path, comment=None):
result = None
session = None
try:
session = self._manager.getSession()
item = self._get_item(release, path)
item = session.merge(item)
target_path, target_name = self.__get_target(release, path)
module = self._supportedItems[item.item_type]['module'](target_path, target_name)
module.delete()
# Commit changes
if not comment:
comment = "Change made with no comment"
self.log.info("commiting changes for module %s" % target_name)
cmd = Git(target_path)
try:
cmd.commit("-a", "-m", comment)
cmd.push("origin")
except GitCommandError as e:
self.log.debug("no commit for %s: %s" % (target_name, str(e)))
session.commit()
result = True
except:
session.rollback()
raise
finally:
session.close()
result &= super(PuppetInstallMethod, self).removeItem(release, path)
return result
def init_remotes():
path = getcwd()
git = Git(path)
has_upstream = raw_input('Is this a fork? (y/N) ')
if has_upstream.lower() == 'y':
upstream = raw_input('What is the url of the upstream project? ')
git.remote('add', 'upstream', upstream)
def merge(self, remoteref, wd=None, **kwargs):
if wd is None:
git = self.git
else:
git = Git(wd)
git.extra = dict(self.git.extra)
git.extra["env"]["GIT_WORK_TREE"] = wd
git.merge(remoteref)
def command(self, command):
"""
Runs the Git command in self.repo
"""
args = split(command)
cmd = Git(self.repodir)
cmd.execute(args)
def merge_index(self, *args, **kwargs):
wd = kwargs.get("work_tree")
if wd is None:
git = self.git
else:
git = Git(wd)
git.extra = dict(self.git.extra)
git.extra["env"]["GIT_WORK_TREE"] = wd
return git.merge_index(*args, **kwargs)
def init_branches(options, error, info, debug):
path = getcwd()
git = Git(path)
create = options.release_tool_settings['branches']
for branch in create:
debug('creating branch "%s"...' % branch)
debug(git.branch(branch))
info('Created branches ' + ', '.join(create))
def __init__(self, options, build_type='job'):
self.options = options
repository, project, version = self.parse_repository()
self.repository = repository
self.project = project
self.version = version
self.cache_dir = Workspace().get_vcs_space('git')
self.build_type = build_type
self.workspace = self.cwd
self.git = Git(working_dir=self.cwd)
self.refs = None
self.logger = get_logger('console')
def is_cached(self):
try:
cache_path = self.cache_dir + '/' + self.project
is_dir = os.path.isdir(cache_path)
if is_dir:
result = Git(cache_path).execute(
['git', 'rev-parse', '--git-dir'], )
if result.startswith('.'):
return True
return False
except Exception:
return False
def get_contibutors(self) -> list[str]:
"""Return list of Git contributors for given path using `git shortlog`
:path: Path of which to get contributors
:flags: Additional flags passed to `git shortlog`
"""
docfile_path = Path(self.get_source_info()[0]).resolve()
docfile_name = docfile_path.name
docdir_path = docfile_path.parent
min_commits = self.optn_over_conf("min_commits",
"scm_contribs_min_commits")
limit_authors = self.optn_over_conf("limit_authors",
"scm_contribs_limit_authors")
if limit_authors is not None and limit_authors < 1:
logger.warning(
"List of contributors limited to less than one entry. "
"Check '(scm_contribs_)limit_authors' option/config value")
flags: list[str] = []
contribs_email = directives.choice(
self.optn_over_conf("email", "scm_contribs_email"),
("true", "false"),
)
flags += ["-e"] if contribs_email == "true" else []
contribs_sort = directives.choice(
self.optn_over_conf("sort", "scm_contribs_sort"),
("name", "num"),
)
flags += ["-n"] if contribs_sort == "num" else []
contribs_type = directives.choice(
self.optn_over_conf("type", "scm_contribs_type"),
("author", "committer"),
)
flags += ["-c"] if contribs_type == "committer" else []
git_shortlog_options = ["-s", *flags, "--", docfile_name]
contributors = []
git_shortlog = Git(docdir_path).shortlog(*git_shortlog_options)
git_shortlog_items = git_shortlog.split("\n")
for item in git_shortlog_items:
if not item:
continue
num, contributor = item.split("\t")
if int(num) < min_commits:
continue
contributors += [contributor]
return contributors[:limit_authors]
def _clone(cls, git, url, path, odb_default_type, progress, **kwargs):
if progress is not None:
progress = to_progress_instance(progress)
odbt = kwargs.pop('odbt', odb_default_type)
## A bug win cygwin's Git, when `--bare` or `--separate-git-dir`
# it prepends the cwd or(?) the `url` into the `path, so::
# git clone --bare /cygwin/d/foo.git C:\\Work
# becomes::
# git clone --bare /cygwin/d/foo.git /cygwin/d/C:\\Work
#
clone_path = (Git.polish_url(path)
if Git.is_cygwin() and 'bare' in kwargs else path)
sep_dir = kwargs.get('separate_git_dir')
if sep_dir:
kwargs['separate_git_dir'] = Git.polish_url(sep_dir)
proc = git.clone(Git.polish_url(url),
clone_path,
with_extended_output=True,
as_process=True,
v=True,
**add_progress(kwargs, git, progress))
if progress:
handle_process_output(proc, None, progress.new_message_handler(),
finalize_process)
else:
(stdout, stderr) = proc.communicate()
log.debug("Cmd(%s)'s unused stdout: %s", getattr(proc, 'args', ''),
stdout)
finalize_process(proc, stderr=stderr)
# our git command could have a different working dir than our actual
# environment, hence we prepend its working dir if required
if not osp.isabs(path) and git.working_dir:
path = osp.join(git._working_dir, path)
repo = cls(path, odbt=odbt)
# retain env values that were passed to _clone()
repo.git.update_environment(**git.environment())
# adjust remotes - there may be operating systems which use backslashes,
# These might be given as initial paths, but when handling the config file
# that contains the remote from which we were clones, git stops liking it
# as it will escape the backslashes. Hence we undo the escaping just to be
# sure
if repo.remotes:
with repo.remotes[0].config_writer as writer:
writer.set_value('url', Git.polish_url(repo.remotes[0].url))
# END handle remote repo
return repo
def pull(repo_dir: str):
"""
Pull all the new files in the master in specified directory.
Directory should contain path where .git file is located.
Arguments:
:param repo_dir directory where .git file is located
"""
git = Git(repo_dir)
git.pull()
repo = Repo(repo_dir)
for submodule in repo.submodules:
submodule.update(recursive=True, init=True)
def clone_from(cls, url, to_path, progress=None, env=None, **kwargs):
"""Create a clone from the given URL
:param url: valid git url, see http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS
:param to_path: Path to which the repository should be cloned to
:param progress: See 'git.remote.Remote.push'.
:param env: Optional dictionary containing the desired environment variables.
:param kwargs: see the ``clone`` method
:return: Repo instance pointing to the cloned directory"""
git = Git(os.getcwd())
if env is not None:
git.update_environment(**env)
return cls._clone(git, url, to_path, GitCmdObjectDB, progress, **kwargs)
def _clone(cls, git, url, path, odb_default_type, progress, **kwargs):
if progress is not None:
progress = to_progress_instance(progress)
odbt = kwargs.pop('odbt', odb_default_type)
# when pathlib.Path or other classbased path is passed
if not isinstance(path, str):
path = str(path)
## A bug win cygwin's Git, when `--bare` or `--separate-git-dir`
# it prepends the cwd or(?) the `url` into the `path, so::
# git clone --bare /cygwin/d/foo.git C:\\Work
# becomes::
# git clone --bare /cygwin/d/foo.git /cygwin/d/C:\\Work
#
clone_path = (Git.polish_url(path)
if Git.is_cygwin() and 'bare' in kwargs
else path)
sep_dir = kwargs.get('separate_git_dir')
if sep_dir:
kwargs['separate_git_dir'] = Git.polish_url(sep_dir)
proc = git.clone(Git.polish_url(url), clone_path, with_extended_output=True, as_process=True,
v=True, universal_newlines=True, **add_progress(kwargs, git, progress))
if progress:
handle_process_output(proc, None, progress.new_message_handler(), finalize_process, decode_streams=False)
else:
(stdout, stderr) = proc.communicate()
log.debug("Cmd(%s)'s unused stdout: %s", getattr(proc, 'args', ''), stdout)
finalize_process(proc, stderr=stderr)
# our git command could have a different working dir than our actual
# environment, hence we prepend its working dir if required
if not osp.isabs(path) and git.working_dir:
path = osp.join(git._working_dir, path)
repo = cls(path, odbt=odbt)
# retain env values that were passed to _clone()
repo.git.update_environment(**git.environment())
# adjust remotes - there may be operating systems which use backslashes,
# These might be given as initial paths, but when handling the config file
# that contains the remote from which we were clones, git stops liking it
# as it will escape the backslashes. Hence we undo the escaping just to be
# sure
if repo.remotes:
with repo.remotes[0].config_writer as writer:
writer.set_value('url', Git.polish_url(repo.remotes[0].url))
# END handle remote repo
return repo
def __init__(self, *args, **kwargs):
if not kwargs.get('parent', None):
# Initialize a new repo
git_path = mkdtemp(prefix=settings.GIT_ROOT_PATH)
git = Git(git_path)
git.init()
body_path = os.path.join(git_path, 'BODY')
self.message = "created new"
kwargs['git_path'] = git_path
super(Fork, self).__init__(*args, **kwargs)
def checkout(self, wd=None, **kwargs):
if wd is None:
git = self.git
wd = self.repo.wd
else:
git = Git(wd)
git.extra = dict(self.git.extra)
git.extra["env"]["GIT_WORK_TREE"] = wd
try:
makedirs(wd)
except OSError, exc:
if exc.errno != EEXIST:
raise
def save(self, *args, **kwargs):
message = kwargs.pop('message', self._next_message() or 'update')
super(Fork, self).save(*args, **kwargs)
with open(os.path.join(self.git_path, "BODY"), 'w') as f:
f.write(self.body)
git = Git(self.git_path)
git.add('BODY')
try:
git.commit(message=message)
except GitCommandError, e:
if e.status != 1:
raise
def checkout(c, name=None):
_platform = __platform()
_c_name, _c_platform = f'[hl]{name}[/hl]', f'[hl]{_platform}[/hl]'
_h_name, _h_platform = f'<b>{name}</b>', f'<b>{_platform}</b>'
_msg = f'Checkout of :{cfg.icon.component}: %s %s completed @ :{cfg.icon.platform}: %s.'
try:
_git = Git(cfg.git.get(name))
_git.checkout()
log.success(_msg % (_c_name, '[ok]correctly[/ok]', _c_platform))
telegrot.success(_msg % (_h_name, '<u>correctly</u>', _h_platform))
exit(0)
except Exception:
log.error(_msg % (_c_name, '[err]not[/err]', _c_platform))
telegrot.error(_msg % (_h_name, '<u>not</u>', _h_platform))
exit(1)
def test_diff_with_staged_file(self, rw_dir):
# SETUP INDEX WITH MULTIPLE STAGES
r = Repo.init(rw_dir)
fp = osp.join(rw_dir, 'hello.txt')
with open(fp, 'w') as fs:
fs.write("hello world")
r.git.add(Git.polish_url(fp))
r.git.commit(message="init")
with open(fp, 'w') as fs:
fs.write("Hola Mundo")
r.git.commit(all=True, message="change on master")
r.git.checkout('HEAD~1', b='topic')
with open(fp, 'w') as fs:
fs.write("Hallo Welt")
r.git.commit(all=True, message="change on topic branch")
# there must be a merge-conflict
self.failUnlessRaises(GitCommandError, r.git.cherry_pick, 'master')
# Now do the actual testing - this should just work
self.assertEqual(len(r.index.diff(None)), 2)
self.assertEqual(len(r.index.diff(None, create_patch=True)), 0,
"This should work, but doesn't right now ... it's OK")
def __init__(self,
options,
title=None,
default_index=0,
options_map_func=None):
self.KEYS_ENTER = (curses.KEY_ENTER, ord('\n'), ord('\r'))
self.KEYS_UP = (curses.KEY_UP, ord('k'))
self.KEYS_DOWN = (curses.KEY_DOWN, ord('j'))
self.KEYS_SELECT = (curses.KEY_RIGHT, ord(' '))
self.options = options
self.title = title
self.indicator = "=>"
self.options_map_func = options_map_func
self.commit_type = [
"feat", "fix", "docs", "style", "refactor", "test", "chore", "exit"
]
self.git = Git(os.getcwd())
self.emojis = {
"feat": "sparkles",
"fix": "bug",
"docs": "memo",
"style": "art",
"refactor": "recycle",
"test": "white_check_mark",
"chore": "building_construction",
}
if default_index >= len(options):
raise ValueError(
'default_index should be less than the length of options')
self.index = default_index
async def pull(self, ctx):
"""Update the bot from github"""
g = Git()
try:
await ctx.send(f"Probably pulled.\n```\n{g.pull()}```")
except git.exc.GitCommandError as e:
await ctx.send(f"An error has occured when pulling```\n{e}```")
def __set_path(config: Config, attr: str):
msg = {
'root_path': "path of the repository to work on",
}
while True:
if not hasattr(config, attr) or not getattr(config, attr):
path = prompt_enter_value(msg[attr])
else:
path = getattr(config, attr) # set by script param
if not __check_path(path):
setattr(config, attr, "")
continue
if (attr == "root_path") & (os.path.realpath(__file__).startswith(
os.path.abspath(path))):
log_error(
"Please copy and run the create release script in another place outside of the repository and execute again."
)
sys.exit()
try:
Git(path)
except InvalidGitRepositoryError:
log_error("Path is not a git repository.")
setattr(config, attr, path)
info = {
"root_path": "Executing release in path '",
}
log_info(info[attr] + str(getattr(config, attr)) + "'")
break
def test_git_submodules_and_add_sm_with_new_commit(self, rwdir):
parent = git.Repo.init(osp.join(rwdir, 'parent'))
parent.git.submodule('add', self._small_repo_url(), 'module')
parent.index.commit("added submodule")
assert len(parent.submodules) == 1
sm = parent.submodules[0]
assert sm.exists() and sm.module_exists()
clone = git.Repo.clone_from(
self._small_repo_url(),
osp.join(parent.working_tree_dir, 'existing-subrepository'))
sm2 = parent.create_submodule('nongit-file-submodule',
clone.working_tree_dir)
assert len(parent.submodules) == 2
for _ in range(2):
for init in (False, True):
sm.update(init=init)
sm2.update(init=init)
# end for each init state
# end for each iteration
sm.move(sm.path + '_moved')
sm2.move(sm2.path + '_moved')
parent.index.commit("moved submodules")
with sm.config_writer() as writer:
writer.set_value('user.email', '*****@*****.**')
writer.set_value('user.name', 'me')
smm = sm.module()
fp = osp.join(smm.working_tree_dir, 'empty-file')
with open(fp, 'w'):
pass
smm.git.add(Git.polish_url(fp))
smm.git.commit(m="new file added")
# submodules are retrieved from the current commit's tree, therefore we can't really get a new submodule
# object pointing to the new submodule commit
sm_too = parent.submodules['module_moved']
assert parent.head.commit.tree[sm.path].binsha == sm.binsha
assert sm_too.binsha == sm.binsha, "cached submodule should point to the same commit as updated one"
added_bies = parent.index.add([sm]) # addded base-index-entries
assert len(added_bies) == 1
parent.index.commit("add same submodule entry")
commit_sm = parent.head.commit.tree[sm.path]
assert commit_sm.binsha == added_bies[0].binsha
assert commit_sm.binsha == sm.binsha
sm_too.binsha = sm_too.module().head.commit.binsha
added_bies = parent.index.add([sm_too])
assert len(added_bies) == 1
parent.index.commit("add new submodule entry")
commit_sm = parent.head.commit.tree[sm.path]
assert commit_sm.binsha == added_bies[0].binsha
assert commit_sm.binsha == sm_too.binsha
assert sm_too.binsha != sm.binsha
def urls(self):
""":return: Iterator yielding all configured URL targets on a remote as strings"""
try:
remote_details = self.repo.git.remote("get-url", "--all", self.name)
for line in remote_details.split('\n'):
yield line
except GitCommandError as ex:
## We are on git < 2.7 (i.e TravisCI as of Oct-2016),
# so `get-utl` command does not exist yet!
# see: https://github.com/gitpython-developers/GitPython/pull/528#issuecomment-252976319
# and: http://stackoverflow.com/a/32991784/548792
#
if 'Unknown subcommand: get-url' in str(ex):
try:
remote_details = self.repo.git.remote("show", self.name)
for line in remote_details.split('\n'):
if ' Push URL:' in line:
yield line.split(': ')[-1]
except GitCommandError as ex:
if any([msg in str(ex) for msg in ['correct access rights',
'cannot run ssh']]):
# If ssh is not setup to access this repository, see issue 694
result = Git().execute(['git', 'config', '--get',
'remote.%s.url' % self.name])
yield result
else:
raise ex
else:
raise ex
def test_diff_with_staged_file(self, rw_dir):
# SETUP INDEX WITH MULTIPLE STAGES
r = Repo.init(rw_dir)
fp = osp.join(rw_dir, 'hello.txt')
with open(fp, 'w') as fs:
fs.write("hello world")
r.git.add(Git.polish_url(fp))
r.git.commit(message="init")
with open(fp, 'w') as fs:
fs.write("Hola Mundo")
r.git.commit(all=True, message="change on master")
r.git.checkout('HEAD~1', b='topic')
with open(fp, 'w') as fs:
fs.write("Hallo Welt")
r.git.commit(all=True, message="change on topic branch")
# there must be a merge-conflict
self.failUnlessRaises(GitCommandError, r.git.cherry_pick, 'master')
# Now do the actual testing - this should just work
self.assertEqual(len(r.index.diff(None)), 2)
self.assertEqual(
len(r.index.diff(None, create_patch=True)), 0,
"This should work, but doesn't right now ... it's OK")
def clone(self, url):
"""Clones a dotfile repository.
Args:
url: The URL of the repository to clone.
"""
print('Cloning {} into {}'.format(url, self.path))
_exec_raw(lambda: Git().clone(url, self.path))
def clone_from(cls, url, to_path, **kwargs):
"""Create a clone from the given URL
:param url: valid git url, see http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS
:param to_path: Path to which the repository should be cloned to
:param **kwargs: see the ``clone`` method
:return: Repo instance pointing to the cloned directory"""
return cls._clone(Git(os.getcwd()), url, to_path, GitCmdObjectDB,
**kwargs)
def graph(self):
"""
Prints a graph of the git log.
This is used for testing and debugging only.
"""
sys.stdout.write(
Git(self.repodir).execute(["git", "log", "--graph", "--oneline"]))
def sync_mirror(self):
dirname = self.cache_dir + '/' + self.project
if not self.is_cached():
Workspace.mkdir(dirname)
git = Git(dirname)
git.execute(['git', 'clone', '--mirror', self.repository, dirname])
command = ['git', 'remote', 'add', 'eclogue', self.repository]
git.execute(command)
self.logger.info(' '.join(command))
command = [
'git', 'remote', 'set-url', '--push', 'origin', self.repository
]
git.execute(command)
self.logger.info(' '.join(command))
command = ['git', 'remote', 'update', '--prune', 'origin']
git.execute(command)
self.logger.info(' '.join(command))
def graph(self):
"""
Prints a graph of the git log.
This is used for testing and debugging only.
"""
sys.stdout.write(
Git(self.repodir).execute(['git', 'log', '--graph', '--oneline']))
def test_git_submodules_and_add_sm_with_new_commit(self, rwdir):
parent = git.Repo.init(osp.join(rwdir, 'parent'))
parent.git.submodule('add', self._small_repo_url(), 'module')
parent.index.commit("added submodule")
assert len(parent.submodules) == 1
sm = parent.submodules[0]
assert sm.exists() and sm.module_exists()
clone = git.Repo.clone_from(self._small_repo_url(),
osp.join(parent.working_tree_dir, 'existing-subrepository'))
sm2 = parent.create_submodule('nongit-file-submodule', clone.working_tree_dir)
assert len(parent.submodules) == 2
for _ in range(2):
for init in (False, True):
sm.update(init=init)
sm2.update(init=init)
# end for each init state
# end for each iteration
sm.move(sm.path + '_moved')
sm2.move(sm2.path + '_moved')
parent.index.commit("moved submodules")
with sm.config_writer() as writer:
writer.set_value('user.email', '*****@*****.**')
writer.set_value('user.name', 'me')
smm = sm.module()
fp = osp.join(smm.working_tree_dir, 'empty-file')
with open(fp, 'w'):
pass
smm.git.add(Git.polish_url(fp))
smm.git.commit(m="new file added")
# submodules are retrieved from the current commit's tree, therefore we can't really get a new submodule
# object pointing to the new submodule commit
sm_too = parent.submodules['module_moved']
assert parent.head.commit.tree[sm.path].binsha == sm.binsha
assert sm_too.binsha == sm.binsha, "cached submodule should point to the same commit as updated one"
added_bies = parent.index.add([sm]) # addded base-index-entries
assert len(added_bies) == 1
parent.index.commit("add same submodule entry")
commit_sm = parent.head.commit.tree[sm.path]
assert commit_sm.binsha == added_bies[0].binsha
assert commit_sm.binsha == sm.binsha
sm_too.binsha = sm_too.module().head.commit.binsha
added_bies = parent.index.add([sm_too])
assert len(added_bies) == 1
parent.index.commit("add new submodule entry")
commit_sm = parent.head.commit.tree[sm.path]
assert commit_sm.binsha == added_bies[0].binsha
assert commit_sm.binsha == sm_too.binsha
assert sm_too.binsha != sm.binsha
def removeRelease(self, name, recursive=False):
super(PuppetInstallMethod, self).removeRelease(name, recursive)
with puppet_lock:
# Move to concrete directory name
name = name.replace("/", "@")
# Sort by length and remove relevant releases
for fname in [f for f in sorted(
os.listdir(self.__work_path), lambda a, b: cmp(b, a), len)
if (recursive and f.startswith(name)) or (not recursive and f == name)]:
current_dir = os.path.join(self.__work_path, fname)
cmd = Git(current_dir)
cmd.push("origin", ":" + fname)
shutil.rmtree(current_dir)
return True
def get_repo(repo_url: str, repo_path: Path):
if repo_path.exists():
repo = Repo(repo_path)
heads = Git().ls_remote(repo_url)
git_hash_length = 40
remote_last_commit = heads.split("\n")[0][:git_hash_length]
if remote_last_commit != repo.head.commit.hexsha:
update_repo = questionary.confirm(
"The remote repository has been updated."
"Do you want to update it?"
).ask()
if update_repo:
repo.remotes.origin.pull()
else:
# pyright: reportUnknownMemberType=false
repo = Repo.clone_from(repo_url, repo_path)
return repo
def get_remote_url(self, remote='origin'):
r = self.get_remote(remote)
try:
url = list(r.urls)[0]
except GitCommandError as ex:
if 'correct access rights' in str(ex):
# If ssh is not setup to access this repository
cmd = ['git', 'config', '--get', 'remote.%s.url' % r.name]
url = Git().execute(cmd)
else:
raise ex
except AttributeError:
url = None
if url is not None and url.startswith('git@'):
domain = url.split('@')[1].split(':')[0]
path = url.split(':')[1]
url = "http://%s/%s" % (domain, path)
return url
def get_remote_url(self, remote='origin'):
r = self.get_remote(remote)
try:
url = list(r.urls)[0]
except GitCommandError as ex:
if 'correct access rights' in str(ex):
# If ssh is not setup to access this repository
cmd = ['git','config','--get','remote.%s.url' % r.name]
url = Git().execute(cmd)
else:
raise ex
except AttributeError:
url = None
if url is not None and url.startswith('git@'):
domain = url.split('@')[1].split(':')[0]
path = url.split(':')[1]
url = "http://%s/%s" % (domain,path)
return url
def init_logging(gn_env: GNEnvironment) -> None:
if len(gn_env.config) == 0 or gn_env.config.get(ConfigKeys.TESTING, False):
# assume we're testing
return
logging_type = gn_env.config.get(
ConfigKeys.TYPE, domain=ConfigKeys.LOGGING, default="logger"
)
if (
logging_type is None
or len(logging_type.strip()) == 0
or logging_type in ["logger", "default", "mock"]
):
return
if logging_type != "sentry":
raise RuntimeError(f"unknown logging type {logging_type}")
dsn = gn_env.config.get(ConfigKeys.DSN, domain=ConfigKeys.LOGGING, default="")
if dsn is None or len(dsn.strip()) == 0:
logger.warning(
"sentry logging selected but no DSN supplied, not configuring sentry"
)
return
import socket
from git.cmd import Git
home_dir = os.environ.get("DINO_HOME", default=None)
if home_dir is None:
home_dir = "."
tag_name = Git(home_dir).describe()
import sentry_sdk
from sentry_sdk import capture_exception as sentry_capture_exception
from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
sentry_sdk.init(
dsn=dsn,
environment=os.getenv("ENVIRONMENT"), # TODO: fix DINO_ENVIRONMENT / ENVIRONMENT discrepancy
server_name=socket.gethostname(),
release=tag_name,
integrations=[
SqlalchemyIntegration(),
RedisIntegration()
],
)
def capture_wrapper(e_info) -> None:
try:
sentry_capture_exception(e_info)
except Exception as e2:
logger.exception(e_info)
logger.error(f"could not capture exception with sentry: {str(e2)}")
gn_env.capture_exception = capture_wrapper
def init_releaserc(debug, info):
path = getcwd()
git = Git(path)
another_env = 'y'
n_th = 'first'
branches = []
secrets = {}
commands = {
"test": "/usr/bin/true"
}
# get branch names and passwords
info('\nLet\'s setup your roll process.')
info('-------------------------------')
while another_env.lower() == 'y' or another_env.lower() == '':
env_name = raw_input('What\'s the name of the ' + n_th + ' environment? ')
branches.append(env_name)
password_required = raw_input('Should a password be required to roll here? (y/N) ')
password = None
if password_required.lower() == 'y':
password = sha.new(getpass()).hexdigest()
secrets[env_name] = password
another_env = raw_input('Is there another environment? (Y/n) ')
n_th = 'next'
info('')
info('Your roll process is:\nmaster -> ' + ' -> '.join(branches))
info('')
info('')
fd = open('.releaserc', 'w')
json.dump({
"branches": branches,
"commands": commands,
"secrets": secrets
}, fd, indent=4, separators=(',', ': '))
debug('adding .releaserc...')
debug(git.add('.releaserc'))
def init(error, info, debug):
path = getcwd()
git = Git(path)
debug('running git init...')
debug(git.init())
welcome_package()
init_releaserc()
call_task('commit', options={
'm': 'Intializing repository'
})
init_branches()
debug('')
info('Git repository created successfully.')
def clone_from(cls, url, to_path, progress=None, env=None, multi_options=None, **kwargs):
"""Create a clone from the given URL
:param url: valid git url, see http://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS
:param to_path: Path to which the repository should be cloned to
:param progress: See 'git.remote.Remote.push'.
:param env: Optional dictionary containing the desired environment variables.
Note: Provided variables will be used to update the execution
environment for `git`. If some variable is not specified in `env`
and is defined in `os.environ`, value from `os.environ` will be used.
If you want to unset some variable, consider providing empty string
as its value.
:param multi_options: See ``clone`` method
:param kwargs: see the ``clone`` method
:return: Repo instance pointing to the cloned directory"""
git = Git(os.getcwd())
if env is not None:
git.update_environment(**env)
return cls._clone(git, url, to_path, GitCmdObjectDB, progress, multi_options, **kwargs)
def refresh(path=None):
"""Convenience method for setting the git executable path."""
global GIT_OK
GIT_OK = False
if not Git.refresh(path=path):
return
if not FetchInfo.refresh():
return
GIT_OK = True
def refresh(path=None):
"""Convenience method for setting the git executable path."""
global GIT_OK
GIT_OK = False
if not Git.refresh(path=path):
return
if not FetchInfo.refresh():
return
GIT_OK = True
def refresh(path='C:\Program Files\Git\bin'):
"""Convenience method for setting the git executable path."""
global GIT_OK
GIT_OK = False
if not Git.refresh(path=path):
return
if not FetchInfo.refresh():
return
GIT_OK = True
def update_repo(project, mirror_dir, gerrit_url):
"""Utility function to mirror git repo
Arguments:
:arg str project: Git project to mirror
:arg str mirror_dir: Path to directory containing repos.
:arg str gerrit_url: URL to the Gerrit server. Used for cloning repos.
"""
log.debug("Updating repo for {0}".format(project))
project_dir = os.path.join(mirror_dir, '{0}.git'.format(project))
if os.path.exists(project_dir):
args = ['git', 'fetch']
else:
os.makedirs(project_dir)
args = ['git', 'clone', '--mirror',
'{0}/{1}'.format(gerrit_url, project), '.']
project_repo = Git(project_dir)
project_repo.execute(args)
class Index(object):
def __init__(self, repo, path=None):
if path is None:
path = join(repo.path, "index")
self.git = repo.git
else:
self.git = Git(repo.wd)
env = {
"GIT_INDEX_FILE": path,
}
if not repo.bare:
env["GIT_WORK_TREE"] = repo.wd
self.git.extra["env"] = env
self.path = path
self.repo = repo
def data_update(self, path, data, mode="0644", **kwargs):
path = str(path)
if isabs(path):
path = path[1:]
hash = self.repo.git.hash_object(stdin=True, t="blob", w=True,
input=data, path=path)
self.git.update_index("--cacheinfo", mode, hash, path, **kwargs)
def checkout(self, wd=None, **kwargs):
if wd is None:
git = self.git
wd = self.repo.wd
else:
git = Git(wd)
git.extra = dict(self.git.extra)
git.extra["env"]["GIT_WORK_TREE"] = wd
try:
makedirs(wd)
except OSError, exc:
if exc.errno != EEXIST:
raise
return git.checkout_index(**kwargs)
def init(cls,
path=None,
mkdir=True,
odbt=DefaultDBType,
expand_vars=True,
**kwargs):
"""Initialize a git repository at the given path if specified
:param path:
is the full path to the repo (traditionally ends with /<name>.git)
or None in which case the repository will be created in the current
working directory
:parm mkdir:
if specified will create the repository directory if it doesn't
already exists. Creates the directory with a mode=0755.
Only effective if a path is explicitly given
:param odbt:
Object DataBase type - a type which is constructed by providing
the directory containing the database objects, i.e. .git/objects.
It will be used to access all object data
:param expand_vars:
if specified, environment variables will not be escaped. This
can lead to information disclosure, allowing attackers to
access the contents of environment variables
:parm kwargs:
keyword arguments serving as additional options to the git-init command
:return: ``git.Repo`` (the newly created repo)"""
if path:
path = expand_path(path, expand_vars)
if mkdir and path and not osp.exists(path):
os.makedirs(path, 0o755)
# git command automatically chdir into the directory
git = Git(path)
git.init(**kwargs)
return cls(path, odbt=odbt)
def __init__(
self,
root=None,
folders=None,
name=None,
url=None,
):
self.DEFAULT_BRANCH = 'master'
if root is None:
self.root = tempfile.mkdtemp(prefix="bugle_" + name + "_")
self.is_root_tmp = True
else:
self.root = root
self.is_root_tmp = False
if name is None:
self.name = os.path.basename(root)
else:
self.name = name
self.url = url
# Make sure folder is a git repo
if self.is_root_tmp and self.url is None:
raise Exception("New folder with no repo URL")
self.git = Git(self.root)
self.git_clone()
self.repo = Repo(self.root)
if folders is None:
self.folders = ['.']
else:
self.folders = folders
self.files = {}
self.branches = {}
self.total_count = {}
self.branch_count = {}
print "creating repo %s under %s, folders %s" % (name, self.root,
self.folders)
def install_or_update(
self, parent_repo, address, target_dir, select=[], fname=None
):
from git.cmd import Git
if not self.is_in_root():
raise DvcException(
"This command can be run only from a repository root"
)
if not os.path.exists(self.MODULES_DIR):
logger.debug("Creating modules dir {}".format(self.MODULES_DIR))
os.makedirs(self.MODULES_DIR)
parent_repo.scm.ignore(os.path.abspath(self.MODULES_DIR))
module_name = Git.polish_url(address).strip("/").split("/")[-1]
if not module_name:
raise DvcException(
"Package address error: unable to extract package name"
)
with TempGitRepo(
address, module_name, Package.MODULES_DIR
) as tmp_repo:
outputs_to_copy = tmp_repo.outs
if select:
outputs_to_copy = list(
filter(lambda out: out.dvc_path in select, outputs_to_copy)
)
fetched_stage_files = set(
map(lambda o: o.stage.path, outputs_to_copy)
)
tmp_repo.fetch(fetched_stage_files)
module_dir = self.create_module_dir(module_name)
tmp_repo.persist_to(module_dir, parent_repo)
dvc_file = self.get_dvc_file_name(fname, target_dir, module_name)
try:
self.persist_stage_and_scm_state(
parent_repo, outputs_to_copy, target_dir, dvc_file
)
except Exception as ex:
raise DvcException(
"Package '{}' was installed "
"but stage file '{}' "
"was not created properly: {}".format(
address, dvc_file, ex
)
)
parent_repo.checkout(dvc_file)
def create(cls, repo, name, url, **kwargs):
"""Create a new remote to the given repository
:param repo: Repository instance that is to receive the new remote
:param name: Desired name of the remote
:param url: URL which corresponds to the remote's name
:param kwargs: Additional arguments to be passed to the git-remote add command
:return: New Remote instance
:raise GitCommandError: in case an origin with that name already exists"""
scmd = 'add'
kwargs['insert_kwargs_after'] = scmd
repo.git.remote(scmd, name, Git.polish_url(url), **kwargs)
return cls(repo, name)
def create(cls, repo, name, url, **kwargs):
"""Create a new remote to the given repository
:param repo: Repository instance that is to receive the new remote
:param name: Desired name of the remote
:param url: URL which corresponds to the remote's name
:param kwargs: Additional arguments to be passed to the git-remote add command
:return: New Remote instance
:raise GitCommandError: in case an origin with that name already exists"""
scmd = 'add'
kwargs['insert_kwargs_after'] = scmd
repo.git.remote(scmd, name, Git.polish_url(url), **kwargs)
return cls(repo, name)
def get():
parser = reqparse.RequestParser()
parser.add_argument('repo', location='args', required=True)
parser.add_argument('port', location='args', required=True)
data = parser.parse_args()
repo = data['repo']
port = data['port']
try:
response = requests.get(repo)
if response.status_code != 200:
return jsonify({
'status': 401,
'error': '{} is not a valid URL'.format(repo)
})
except:
return jsonify({
'status': 401,
'error': '{} is not a valid URL'.format(repo)
})
gitcmd = Git('.')
gitcmd.init()
name = repo.split('/')
repo_name = name[-1]
repo_name = repo_name.replace('.git', '')
gitcmd.clone(repo, 'repos/' + repo_name)
subprocess.Popen(
'.flaskapp/bin/uwsgi --http :%s --file repos/{}/wsgi.py --callable app'
.format(port, repo_name))
return jsonify({
'status':
200,
'message':
'Deployed to Raspberry Pi! {} is now running!'.format(repo_name)
})
def update_repo(project, mirror_dir, gerrit_url):
"""Utility function to mirror git repo
Arguments:
:arg str project: Git project to mirror
:arg str mirror_dir: Path to directory containing repos.
:arg str gerrit_url: URL to the Gerrit server. Used for cloning repos.
"""
log.debug("Updating repo for {0}".format(project))
project_dir = os.path.join(mirror_dir, '{0}.git'.format(project))
if os.path.exists(project_dir):
args = ['git', 'fetch']
else:
os.makedirs(project_dir)
args = [
'git', 'clone', '--mirror', '{0}/{1}'.format(gerrit_url, project),
'.'
]
project_repo = Git(project_dir)
project_repo.execute(args)
def get_remote_url(self, remote='origin', cached=True):
"""Get a git remote URL for this instance."""
if hasattr(self.__class__, '_remote_url') and cached:
url = self.__class__._remote_url
else:
r = self.get_remote(remote)
try:
url = list(r.urls)[0]
except GitCommandError as ex:
if 'correct access rights' in str(ex):
# If ssh is not setup to access this repository
cmd = ['git', 'config', '--get', 'remote.%s.url' % r.name]
url = Git().execute(cmd)
else:
raise ex
except AttributeError:
url = None
if url is not None and url.startswith('git@'):
domain = url.split('@')[1].split(':')[0]
path = url.split(':')[1]
url = "http://%s/%s" % (domain, path)
self.__class__._remote_url = url
return url
def roll(options, debug, info, error):
to_arg = options.args.index('to')
path = getcwd()
git = Git(path)
repo = Repo(path)
dest = None
source = None
active = repo.active_branch
settings = options.release_tool_settings
if to_arg == 0:
source = active
dest = options.args[1]
#update()
elif to_arg == 1:
source = options.args[0]
dest = options.args[2]
else:
return
secret = settings['secrets'][str(dest)]
if secret:
info('Special permission is required to roll to this branch.')
if not secret or has_permission(if_matches=secret):
debug('rebasing {source} onto {dest}...'.format(
source=source,
dest=dest
))
if secret:
debug(git.rebase(source, dest))
git.tag('-m', message('Roll from {source} to {dest} with special \
permission by {name} <{email}>.'.format(
source=source,
dest=dest,
name=Actor.author().name,
email=Actor.author().email
), 'tag'),
'-s',
'roll-from-' +
source + '-to-' +
dest + '-' +
datetime.utcnow().isoformat().replace(':', '.'),
output_stream=stdout)
else:
debug(git.rebase(source, dest))
debug('checking out {source}...'.format(source=source))
debug(git.checkout(active))
info('Rolled {source} to {dest}.'.format(source=source, dest=dest))
else:
error('You do not have permission to roll to this branch')
def welcome_package(debug, info):
path = getcwd()
git = Git(path)
fd = open('welcome.txt', 'w')
fd.write('''
Welcome to {tool_name}
=======================
This tool is a helper to be used along side git.
$ {tool_command} init # you just ran this!
$ {tool_command} update
$ {tool_command} roll to dev
$ {tool_command} roll qa to prod
'''.format(tool_name=tool_name, tool_command=tool_command))
debug('adding welcome.txt...')
debug(git.add('welcome.txt'))
info('Welcome to {tool_name}'.format(tool_name=tool_name))
