def add_binary_git_repo(folder, filename):
""" Create a fake image file for the specified git repo. """
repo, newfolder, parents = _clone_and_top_commits(folder, 'master')
content = b"""\x00\x00\x01\x00\x01\x00\x18\x18\x00\x00\x01\x00 \x00\x88
\t\x00\x00\x16\x00\x00\x00(\x00\x00\x00\x18\x00x00\x00\x01\x00 \x00\x00\x00
\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00
00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa7lM\x01\xa6kM\t\xa6kM\x01
\xa4fF\x04\xa2dE\x95\xa2cD8\xa1a
"""
# Create a file in that git repo
with open(os.path.join(newfolder, filename), 'wb') as stream:
stream.write(content)
repo.index.add(filename)
repo.index.write()
# Commits the files added
tree = repo.index.write_tree()
author = pygit2.Signature('Alice Author', '*****@*****.**')
committer = pygit2.Signature('Cecil Committer', '*****@*****.**')
repo.create_commit(
'refs/heads/master', # the name of the reference to update
author,
committer,
'Add a fake image file',
# binary string representing the tree object ID
tree,
# list of binary strings representing parents of the new commit
parents)
# Push to origin
ori_remote = repo.remotes[0]
master_ref = repo.lookup_reference('HEAD').resolve()
refname = '%s:%s' % (master_ref.name, master_ref.name)
PagureRepo.push(ori_remote, refname)
shutil.rmtree(newfolder)
def test_merge_base(self):
commit = self.repo.merge_base(
'5ebeeebb320790caf276b9fc8b24546d63316533',
'4ec4389a8068641da2d6578db0419484972284c8')
assert commit.hex == 'acecd5ea2924a4b900e7e149496e1f4b57976e51'
# Create a commit without any merge base to any other
sig = pygit2.Signature("me", "*****@*****.**")
indep = self.repo.create_commit(None, sig, sig, "a new root commit",
self.repo[commit].peel(pygit2.Tree).id,
[])
assert self.repo.merge_base(indep, commit) is None
def test_stash(self):
# some changes to working dir
with open(os.path.join(self.repo.workdir, 'hello.txt'), 'w') as f:
f.write('new content')
sig = pygit2.Signature('Stasher', '*****@*****.**')
self.repo.stash(sig, include_untracked=True)
assert 'hello.txt' not in self.repo.status()
self.repo.stash_apply()
assert 'hello.txt' in self.repo.status()
self.repo.stash_drop()
with pytest.raises(KeyError):
self.repo.stash_pop()
def create_commit(self, message, user):
self.ensure_source_folder_exists()
index = self.repo.index
index.add_all()
treeId = index.write_tree(self.repo)
author = pygit2.Signature(user.username, user.email)
parents = [] if self.repo.head_is_unborn else [self.repo.head.target]
self.repo.create_commit('HEAD', author, author, message, treeId,
parents)
def push_project_to_github(project_name, user_name, user_password):
base64_bytes = user_password.encode('ascii')
user_password_bytes = base64.b64decode(base64_bytes)
decoded_user_password = user_password_bytes.decode('ascii')
g = Github(user_name, decoded_user_password)
user = g.get_user()
repo = user.create_repo(project_name)
repoClone = pygit2.clone_repository(repo.git_url,
'{}'.format(project_name))
repoClone.remotes.set_url('origin', repo.clone_url)
os.system('cp -r work/ {}/'.format(project_name))
index = repoClone.index
index.add_all()
index.write()
author = pygit2.Signature('Desislava', '*****@*****.**')
committer = pygit2.Signature('Desislava', '*****@*****.**')
tree = index.write_tree()
repoClone.create_commit(
'HEAD', author, committer,
'Pushed project directory to GitHub through STACKn studio.', tree,
[repoClone.head.target])
remote = repoClone.remotes['origin']
credentials = pygit2.UserPass(user_name, decoded_user_password)
remote.credentials = credentials
callbacks = pygit2.RemoteCallbacks(credentials)
remote.push(['refs/heads/master'], callbacks)
os.system('rm -rf {}/'.format(project_name))
return json.dumps({'status': 'OK', 'clone_url': repo.clone_url})
def test_read_tag(self):
repo = self.repo
tag = repo[TAG_SHA]
target = repo[tag.target]
assert isinstance(tag, pygit2.Tag)
assert pygit2.GIT_OBJ_TAG == tag.type
assert pygit2.GIT_OBJ_COMMIT == target.type
assert 'root' == tag.name
assert 'Tagged root commit.\n' == tag.message
assert 'Initial test data commit.\n' == target.message
self.assertEqualSignature(
tag.tagger,
pygit2.Signature('Dave Borowitz', '*****@*****.**',
1288724692, -420))
def make_signature(name, email, tz=None):
"""Generate a Signature object with the given user name and e-mail.
The time is set to now() in Django's current timezone. User name is encoded to UTF-8.
@param name: user full name
@param email: user e-mail
@return: pygit2.Signature()
"""
now = datetime.datetime.utcnow()
timestamp = int((now - EPOCH).total_seconds()) # XXX Python 2.7
# Offset in minutes
offset = int(tz.utcoffset(now).total_seconds() // 60) if tz else 0
return pygit2.Signature(name, email, timestamp, offset)
def set_author(self, user):
"""
Set the current author for the following commits (through save() and delete().
:param user: User model (or compatible) instance
This state is stored not to modify save() and delete() signatures. This also means you must remember to
set another author as required.
Designed with a repository reopened at each web request in mind.
"""
self.author_signature = pygit2.Signature(user.get_full_name(),
user.email,
encoding='utf8')
def test_read_tag(self):
repo = self.repo
tag = repo[TAG_SHA]
target = repo[tag.target]
self.assertTrue(isinstance(tag, pygit2.Tag))
self.assertEqual(pygit2.GIT_OBJ_TAG, tag.type)
self.assertEqual(pygit2.GIT_OBJ_COMMIT, target.type)
self.assertEqual('root', tag.name)
self.assertEqual('Tagged root commit.\n', tag.message)
self.assertEqual('Initial test data commit.\n', target.message)
self.assertEqualSignature(
tag.tagger,
pygit2.Signature('Dave Borowitz', '*****@*****.**',
1288724692, -420))
def setUp(self):
os.environ["XDG_CACHE_HOME"] = tempfile.mkdtemp(prefix="gs_cache")
# setup stub linux repository
os.environ["LINUX_GIT"] = tempfile.mkdtemp(prefix="gs_repo")
self.repo = pygit2.init_repository(os.environ["LINUX_GIT"])
author = pygit2.Signature('Alice Author', '*****@*****.**')
committer = pygit2.Signature('Cecil Committer', '*****@*****.**')
tree = self.repo.TreeBuilder().write()
parent = []
commits = []
for i in range(3):
subject = "mainline %d" % (i, )
cid = self.repo.create_commit("refs/heads/master", author,
committer, "%s\n\nlog" % (subject, ),
tree, parent)
parent = [cid]
commits.append(cid)
self.commits = commits
self.repo.remotes.create(
"origin",
"git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git")
self.repo.references.create("refs/remotes/origin/master", commits[-1])
self.index = git_sort.SortIndex(self.repo)
# setup stub kernel-source content
self.ks_dir = tempfile.mkdtemp(prefix="gs_ks")
patch_dir = os.path.join(self.ks_dir, "patches.suse")
os.mkdir(patch_dir)
os.chdir(patch_dir)
for commit in commits:
tests.support.format_patch(self.repo.get(commit),
mainline="v3.45-rc6")
def apply_changes(repo, version, contents, path, delete=False):
# Apply the requested change to the git repository
branch_name = BRANCHES[version.channel]
git_repo = repo.git_repository
blob_id = git_repo.create_blob(contents)
# Initialize the index from the tree structure of the most
# recent commit in `branch`
tree = git_repo.revparse_single(branch_name).tree
index = git_repo.index
index.read_tree(tree)
# Add / update the index
path = os.path.join(EXTRACTED_PREFIX, path)
if delete:
index.remove(path)
else:
entry = pygit2.IndexEntry(path, blob_id, pygit2.GIT_FILEMODE_BLOB)
index.add(entry)
tree = index.write_tree()
# Now apply a new commit
author = pygit2.Signature('test', '*****@*****.**')
committer = pygit2.Signature('test', '*****@*****.**')
branch = git_repo.branches.get(branch_name)
# Create commit and properly update branch and reflog
oid = git_repo.create_commit(
None, author, committer, '...', tree, [branch.target])
commit = git_repo.get(oid)
branch.set_target(commit.hex)
# To make sure the serializer makes use of the new commit we'll have
# to update the `git_hash` values on the version object.
version.update(git_hash=commit.hex)
def add_version(self, rep, version):
if len(rep.diff()) == 0:
try:
_ = rep.head
raise Exception("No new changes")
except pygit2.GitError:
new = True
else:
new = False
metadata = self._get_metadata(rep)
rep.index.read()
base = dirname(rep.path)
for path, dirs, files in os.walk(dirname(rep.path)):
if os.path.basename(path) == ".git":
del dirs[:]
continue
for file in files:
rep.index.add(os.path.relpath(os.path.join(path, file), base))
author = metadata["author"].strip()
email = re.search(r"(?<=<)[^@>]+@[^@>]+(?=>$)", author)
if email is not None:
email = email.group()
author = author[:-(len(email) + 2)].strip()
rep.create_commit('HEAD', pygit2.Signature(author, email or ""),
pygit2.Signature("VelesForge", ""),
version, rep.index.write_tree(),
[rep.head.peel().oid] if not new else [])
rep.index.write()
if version != "master":
try:
rep.lookup_reference("refs/tags/" + version)
raise Exception("Tag %s already exists for %s" % (
version, metadata["name"]))
except:
rep.create_tag(
version, rep.head.peel().oid, pygit2.GIT_OBJ_COMMIT,
pygit2.Signature("VelesForge", ""), "")
def create_commit(repo, ref, parents, text):
# For now we don't do merge commits.
assert len(parents) <= 1
toid = add_blob(repo, parents, text)
author = pygit2.Signature('Alice Author', '*****@*****.**')
committer = author
return repo.create_commit(
ref, # the name of the reference to update
author, committer, f'Say {text}\n\n{text} ',
toid, # binary string representing the tree object ID
parents # list of binary strings representing parents of the new commit
)
def on_post(self, req, resp, dataset):
ds_path = self.store.get_dataset_path(dataset)
if (os.path.isdir(ds_path)):
resp.media = {'error': 'dataset already exists'}
resp.status = falcon.HTTP_CONFLICT
else:
# Record if this was done on behalf of a user
name, email = get_user_info(req)
if name and email:
author = pygit2.Signature(name, email)
else:
author = None
hexsha = create_dataset(self.store, dataset, author)
resp.media = {'hexsha': hexsha}
resp.status = falcon.HTTP_OK
def put(self, build):
assert isinstance(build, Build)
branch = self._get_or_create_branch(build.branch)
tree_id = self._create_tree(build.config_files())
# creating the new commit
timestamp = int(time.time())
name = next(self.repo.config.get_multivar('user.name'))
email = next(self.repo.config.get_multivar('user.email'))
author = pygit2.Signature('crossbow', '*****@*****.**',
int(timestamp))
committer = pygit2.Signature(name, email, int(timestamp))
message = build.description
reference = 'refs/heads/{}'.format(branch.branch_name)
commit_id = self.repo.create_commit(reference, author, committer,
message, tree_id, [branch.target])
logging.info('{} created on {}'.format(
commit_id, branch.branch_name))
self.updated_branches.append(branch)
def add_repo_tag(git_dir, repo, tags, repo_name):
""" Use a list to create multiple tags on a git repo """
for tag in reversed(tags):
time.sleep(1)
tests.add_commit_git_repo(os.path.join(git_dir, "repos", repo_name),
ncommits=1)
first_commit = repo.revparse_single("HEAD")
tagger = pygit2.Signature("Alice Doe", "*****@*****.**", 12347, 0)
repo.create_tag(
tag,
first_commit.oid.hex,
pygit2.GIT_OBJ_COMMIT,
tagger,
"Release " + tag,
)
def post_repo(repo):
repo = os.path.join(SERVER_REPO_PATH, repo)
if not os.path.isdir(repo):
return make_response('/%s: Unknown repo.' % repo, 404)
git = pygit2.Repository(repo)
headw = git.lookup_reference("refs/heads/working").target
commitw = git[headw]
head = git.lookup_reference("refs/heads/master").target
if commitw.tree != git[head].tree:
commit = git.create_commit(
'refs/heads/master',
pygit2.Signature("POST " + request.full_path,
request.remote_addr + "@hoster"),
pygit2.Signature("hoster", "@hoster"), '', commitw.tree.id, [head])
if git.branches['working'].target != head:
head = git.lookup_reference("refs/heads/master").target
git.branches['working'].delete()
git.branches.local.create("working", git[head])
return make_response("OK")
def move_files_into_repo(dataset_id, dataset_path, upload_path, name, email,
cookies):
repo = pygit2.Repository(dataset_path)
unlock_files = [
os.path.relpath(filename, start=upload_path)
for filename in pathlib.Path(upload_path).glob('**/*')
if os.path.islink(
os.path.join(dataset_path,
os.path.relpath(filename, start=upload_path)))
]
gevent.sleep()
move_files(upload_path, dataset_path)
author = pygit2.Signature(name, email)
hexsha = git_commit(repo, unlock_files, author).hex
update_head(dataset_id, dataset_path, hexsha, cookies)
def set_up_git_repo(self, name='test'):
""" Set up the git repo to play with. """
# Create a git repo to play with
gitrepo = os.path.join(self.path, 'repos', '%s.git' % name)
repo = pygit2.init_repository(gitrepo, bare=True)
newpath = tempfile.mkdtemp(prefix='pagure-other-test')
repopath = os.path.join(newpath, 'test')
clone_repo = pygit2.clone_repository(gitrepo, repopath)
# Create a file in that git repo
with open(os.path.join(repopath, 'sources'), 'w') as stream:
stream.write('foo\n bar')
clone_repo.index.add('sources')
clone_repo.index.write()
# Commits the files added
tree = clone_repo.index.write_tree()
author = pygit2.Signature(
'Alice Author', '*****@*****.**')
committer = pygit2.Signature(
'Cecil Committer', '*****@*****.**')
clone_repo.create_commit(
'refs/heads/master', # the name of the reference to update
author,
committer,
'Add sources file for testing',
# binary string representing the tree object ID
tree,
# list of binary strings representing parents of the new commit
[]
)
refname = 'refs/heads/master'
ori_remote = clone_repo.remotes[0]
PagureRepo.push(ori_remote, refname)
def setUp(self):
os.environ["XDG_CACHE_HOME"] = tempfile.mkdtemp(prefix="gs_cache")
self.repo_dir = tempfile.mkdtemp(prefix="gs_repo")
self.repo = pygit2.init_repository(self.repo_dir)
author = pygit2.Signature('Alice Author', '*****@*****.**')
committer = pygit2.Signature('Cecil Committer', '*****@*****.**')
tree = self.repo.TreeBuilder().write()
parent = []
commits = []
for i in range(3):
subject = "commit %d" % (i, )
cid = self.repo.create_commit("refs/heads/master", author,
committer, "%s\n\nlog" % (subject, ),
tree, parent)
parent = [cid]
commits.append((
str(cid),
subject,
))
self.commits = commits
self.index = git_sort.SortIndex(self.repo)
class Branch:
author = git.Signature('Lord Buckethead', '*****@*****.**')
committer = author
def __init__(self, repo, *head, name='refs/heads/master'):
self.repo = repo
self.head = head
self.name = name
def commit(self, title, tree, *merge):
self.head = (self.repo.create_commit(self.name, self.author,
self.committer, title,
create_tree(self.repo, tree),
list(self.head + merge)), )
def add_tags(self, n=2):
""" Add a tag to the laste `n` commit in the test repo.
"""
self.add_commits()
git_repo_path = os.path.join(self.gitroot, 'test_repo')
repo = pygit2.Repository(git_repo_path)
author = pygit2.Signature('Alice Author', '*****@*****.**')
# Create the local branches
for i in range(n):
commitid = repo.revparse_single('HEAD~%s' % i).oid.hex
repo.create_tag(
'v%s' % i, commitid, pygit2.GIT_OBJ_COMMIT, author,
'tag v%s' % i)
def make_commit_from_index(self, repo, index, message):
tree = index.write_tree(repo.pygit)
sig = pygit2.Signature("Wagtail Localize",
"*****@*****.**")
repo.pygit.create_commit(
f"refs/heads/{DEFAULT_BRANCH}",
sig,
sig,
message,
tree,
[repo.pygit.head.target],
)
return repo.pygit.head.target.hex
def read_configfile(config_filename):
""" Read and parse given configfile """
global TEMP_PATH
global ZABBIX_URL, ZABBIX_USER, ZABBIX_PASSWORD, ZABBIX_TEMPLATE_EXPORT_GROUP, \
ZABBIX_TEMPLATE_ROOT_GROUP
global GIT_URL, GIT_BRANCH, GIT_CALLBACKS, GIT_SIGNATURE
# Find and read configfile
config = configparser.ConfigParser()
configfile = '%s.conf' % config_filename
print(f'Reading configfile {configfile}')
for loc in os.path.join(os.curdir, configfile), \
os.path.join(os.path.expanduser('~'), configfile), \
resource_filename(__name__,
os.path.join("../../../../etc/zbxtpltools/",
configfile)):
try:
with open(loc) as source:
config.read_file(source)
break
except IOError:
pass
if not config.has_option('zabbix', 'url'):
raise Exception('Could not read configfile ' + loc)
TEMP_PATH = config['general']['temp_path']
ZABBIX_URL = config['zabbix']['url']
ZABBIX_USER = config['zabbix']['user']
ZABBIX_PASSWORD = config['zabbix']['password']
LOGGER.info("Zabbix URL: %s - API user: %s", ZABBIX_URL, ZABBIX_USER)
ZABBIX_TEMPLATE_EXPORT_GROUP = config['zabbix']['template_export_group']
ZABBIX_TEMPLATE_ROOT_GROUP = config['zabbix']['template_root_group']
GIT_URL = config['git']['url']
GIT_BRANCH = config['git']['branch']
GIT_USER = re.search(r'(?:https?|ssh)://([^@?/:]+)', GIT_URL).group(1)
GIT_CALLBACKS = GitRemoteCallbacks(
credentials=pygit2.Keypair(GIT_USER,
find_file(config['git']['ssh_pubkey']),
find_file(config['git']['ssh_privkey']),
''))
GIT_SIGNATURE = pygit2.Signature(config['git']['author_name'], config['git']['author_email'])
LOGGER.info("Git URL: %s - branch: %s", GIT_URL, GIT_BRANCH)
def git_commit(self):
"""Commit the current index to the git repository."""
author = pygit2.Signature(
'Git-Annex-Adapter Tester',
'*****@*****.**',
1500000000, # Date: 2017-07-14 02:40:00
)
try:
parents = [self.repo.head.get_object().hex]
except pygit2.GitError:
parents = []
return self.repo.create_commit(
'HEAD', author, author, 'Test commit',
self.repo.index.write_tree(), parents,
)
def setUp(self):
self.maxDiff = None # pylint: disable=invalid-name
self.tmp_dir = tempfile.mkdtemp()
self.remote_source_repo_path = os.path.join(self.tmp_dir, 'source_repo')
# Initialise fake source_repo.
repo = pygit2.init_repository(self.remote_source_repo_path, True)
tree = repo.TreeBuilder().write()
author = pygit2.Signature('OSV', '*****@*****.**')
repo.create_commit('HEAD', author, author, 'Initial commit', tree, [])
osv.SourceRepository(
id='oss-fuzz',
name='oss-fuzz',
repo_url='file://' + self.remote_source_repo_path,
repo_username='').put()
osv.Bug(
id='2017-134',
affected=['FILE5_29', 'FILE5_30'],
affected_fuzzy=['5-29', '5-30'],
confidence=100,
details=(
'OSS-Fuzz report: '
'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1064\n\n'
'Crash type: Heap-buffer-overflow READ 1\n'
'Crash state:\ncdf_file_property_info\ncdf_file_summary_info\n'
'cdf_check_summary_info\n'),
ecosystem='',
fixed='19ccebafb7663c422c714e0c67fa4775abf91c43',
has_affected=True,
issue_id='1064',
project='file',
public=True,
reference_urls=[
'https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1064'
],
regressed='17ee4cf670c363de8d2ea4a4897d7a699837873f',
repo_url='https://github.com/file/file.git',
search_indices=['file', '2017-134', '2017', '134'],
severity='MEDIUM',
sort_key='2017-0000134',
source_id='oss-fuzz:5417710252982272',
status=1,
summary='Heap-buffer-overflow in cdf_file_property_info',
timestamp=datetime.datetime(2021, 1, 15, 0, 0, 24, 559102)).put()
def create_new_commit(repo, ref, template, state):
values = ""
for entry in state:
values += "{} = {}\n".format(entry, state[entry])
data = template.decode().replace("#{OVERRIDE_VALUES_HERE}\n", values)
commit_msg = "{},{},{},{}".format(state['perf'], state['variance'],
state['build_broken'], state['exec_broken'])
data_oid = repo.create_blob(data)
tip = repo.revparse_single(ref)
tb = repo.TreeBuilder(tip.tree)
tb.insert('perf.py', data_oid, pygit2.GIT_FILEMODE_BLOB_EXECUTABLE)
new_tree = tb.write()
author = pygit2.Signature('EzBench unit test', '*****@*****.**')
return repo.create_commit(ref, author, author, commit_msg, new_tree, [tip.id])
def test_view_history_file_on_tag(self):
""" Test the view_history_file endpoint """
# set a tag on the head's parent commit
repo_obj = pygit2.Repository(
os.path.join(self.path, "repos", "test.git"))
commit = repo_obj[repo_obj.head.target]
parent = commit.parents[0].oid.hex
tagger = pygit2.Signature("Alice Doe", "*****@*****.**", 12347, 0)
repo_obj.create_tag("v1.0", parent, pygit2.GIT_OBJ_COMMIT, tagger,
"Release v1.0")
output = self.app.get("/test/history/sources?identifier=v1.0")
self.assertEqual(output.status_code, 200)
output_text = output.get_data(as_text=True)
self.assertIn("<strong>initial commit</strong>", output_text)
data = self.regex.findall(output_text)
self.assertEqual(len(data), 1)
def git_commit_index(repo,
author=None,
message="[OpenNeuro] Recorded changes",
parents=None):
"""Commit any existing index changes."""
committer = pygit2.Signature(COMMITTER_NAME, COMMITTER_EMAIL)
if not author:
author = committer
if parents is None:
parent_commits = [repo.head.target.hex]
else:
parent_commits = parents
tree = repo.index.write_tree()
commit = repo.create_commit('refs/heads/master', author, committer,
message, tree, parent_commits)
repo.head.set_target(commit)
return commit
def migrate(function, _s_old, _s_new):
assert function(_s_old) == _s_new
folder = '/tmp/migrate_script_iemldb'
if os.path.isdir(folder):
shutil.rmtree(folder)
# os.mkdir(folder)
git_address = "https://github.com/IEMLdev/ieml-language.git"
credentials = pygit2.Keypair('ogrergo', '~/.ssh/id_rsa.pub',
'~/.ssh/id_rsa', None)
gitdb = GitInterface(origin=git_address,
credentials=credentials,
folder=folder)
signature = pygit2.Signature("Louis van Beurden",
"*****@*****.**")
db = IEMLDatabase(folder=folder, use_cache=False)
to_migrate = {}
desc = db.get_descriptors()
struct = db.get_structure()
for s in db.get_dictionary().scripts:
s2 = function(s)
if s2 != s:
to_migrate[s] = s2
print(to_migrate)
with gitdb.commit(
signature,
"[Translate script] Translate paradigm from '{}' to '{}".format(
str(_s_old), str(_s_new))):
for s_old, s_new in to_migrate.items():
db.remove_structure(s_old)
for (_, key), values in struct.get_values_partial(s_old).items():
for v in values:
db.add_structure(s_new, key, v)
db.remove_descriptor(s_old)
for (_, lang, d), values in desc.get_values_partial(s_old).items():
for v in values:
db.add_descriptor(s_new, lang, d, v)