def __call__(self, data_event):
from mcdp_hdb.disk_map_disk_events_from_data_events import disk_events_from_data_event
from mcdp_hdb.disk_events import apply_disk_event_to_filesystem
s = yaml_dump(data_event)
logger.debug('Event #%d:\n%s' % (len(self.data_events), indent(s, '> ')) )
self.data_events.append(data_event)
disk_events = disk_events_from_data_event(disk_map=self.disk_map,
schema=self.view._schema,
data_rep=self.view._data,
data_event=data_event)
for disk_event in disk_events:
logger.debug('Disk event:\n%s' % yaml_dump(disk_event))
wd = self.repo.working_dir
apply_disk_event_to_filesystem(wd, disk_event, repo=self.repo)
message = yaml_dump(data_event)
who = data_event['who']
if who is not None:
actor = who['actor']
host = who['host']
instance = who['instance']
else:
actor = 'system'
host = host_name()
instance = 'unspecified'
author = Actor(actor, '%s@%s' % (actor, instance))
committer = Actor(instance, '%s@%s' % (instance, host))
_commit = self.repo.index.commit(message, author=author, committer=committer)
def test_actor_from_string(self):
self.assertEqual(Actor._from_string("name"), Actor("name", None))
self.assertEqual(Actor._from_string("name <>"), Actor("name", ""))
self.assertEqual(
Actor._from_string(
"name last another <*****@*****.**>"),
Actor("name last another", "*****@*****.**"))
def check_translation_diskrep_to_gitrep(disk_rep0, disk_events, disk_rep1, out): # @UnusedVariable
if not disk_events:
raise ValueError('no disk events')
repo = gitrep_from_diskrep(disk_rep0)
wd = repo.working_tree_dir
readback = diskrep_from_gitrep(repo)
assert_diskreps_same(disk_rep0, readback, 'original', 'written back')
logger.debug(wd)
logger.debug('\n'+indent(readback.tree(), 'read back |'))
logger.debug('\n'+indent(yaml_dump(disk_events), 'disk_events|'))
commits = []
for disk_event in disk_events:
logger.debug(indent(yaml_dump(disk_event), 'disk_event | '))
apply_disk_event_to_filesystem(wd, disk_event, repo=repo)
if repo.untracked_files:
logger.debug('adding untracked file %r' % repo.untracked_files)
repo.index.add(repo.untracked_files)
message = yaml_dump(disk_event)
who = disk_event['who']
logger.info('who: %s' % who)
actor = who['actor']
instance = who.get('instance', None)
host = who.get('host', None)
author = Actor(actor, instance)
committer = Actor(instance, host)
commit = repo.index.commit(message, author=author, committer=committer)
commits.append(commit)
res = {}
res['repo'] = repo
return res
def make_commit(self, freezer=None):
"""
Makes a random commit in the current branch.
"""
freezer.start() if freezer else None
from datetime import datetime
fragment = uuid().hex[:8]
filename = join(self.repodir, fragment)
with open(filename, "w") as fh:
fh.write(uuid().hex)
self.repo.index.add([basename(filename)])
str_date = str(datetime.utcnow())
if "." in str_date:
str_date, _ = str_date.split(".")
str_date = str_date.replace(" ", "T")
if not freezer:
str_date, _ = str(datetime.utcnow().isoformat()).split(".")
msg = "Adding {0}".format(basename(filename))
self.repo.index.commit(
author=Actor("Peter", "*****@*****.**"),
author_date=str_date,
committer=Actor("Paul", "*****@*****.**"),
commit_date=str_date,
message=msg,
)
freezer.stop() if freezer else None
def commit(self):
# Calc tree hash
treedata = b''
for entry in sorted(os.listdir(self.repo_path)):
if entry == '.git': continue
with open(os.path.join(self.repo_path, entry)) as f:
data_hash = git_object_hash('blob', f.read().encode()).digest()
treedata += b'100644 ' + entry.encode() + b'\x00' + data_hash
tree_hash = git_object_hash('tree', treedata).hexdigest()
# Serialize commit
commit_data = b'tree ' + tree_hash.encode() + b'\n'
commit_data += b'parent ' + self.repo.head.commit.hexsha.encode(
) + b'\n'
# Random author
author = Actor(get_full_name(), get_email_address())
# Calc timestamp
unix_time = int(time())
is_dst = daylight and localtime().tm_isdst > 0
offset = altzone if is_dst else timezone
datetime = str(unix_time) + ' ' + altz_to_utctz_str(offset)
commit_data += b'author ' + author.name.encode(
) + b' <' + author.email.encode() + b'> ' + datetime.encode() + b'\n'
commit_data += b'committer ' + author.name.encode(
) + b' <' + author.email.encode() + b'> ' + datetime.encode() + b'\n'
commit_data += b'\n'
# Randomize commit message to feature proof-of-work
msg = calc_pow(commit_data)
return self.repo.index.commit(msg,
author=author,
committer=author,
author_date=datetime,
commit_date=datetime)
class GitStorage(WikiABC):
AUTHOR = Actor('tw5_server', '')
@cached_property
def repo(self):
return Repo(self.ROOT / self.name)
@classmethod
def _init_storage(cls, root):
if not root.exists():
os.makedirs(root)
try:
repo = Repo.init(root)
filepath = root / cls.FILENAME
shutil.copy(CONFIG['TW5_TEMPLATE_FILE'], filepath)
repo.index.add(cls.FILENAME)
repo.index.commit('automatic create', author=cls.AUTHOR)
except Exception as err:
if root.exists():
shutil.rmtree(root)
raise err
def update(self, file_obj):
with open(self.file, 'wb') as f:
shutil.copyfileobj(file_obj, f)
self.repo.index.add(self.FILENAME)
self.repo.index.commit('automatic commit', author=self.AUTHOR)
def setUp(self):
"""
Sets up the Git repository for testing.
The following will be available after :py:method`setUp()` runs.
self.repodir
The absolute filename of the Git repository
self.repo
A ``git.Repo`` object for self.repodir
This will create the root commit in the test repository automaticall.
"""
freezer = freeze_time("2001-01-01T00:00:00")
freezer.start()
from datetime import datetime
super(GitSweepTestCase, self).setUp()
repodir = mkdtemp()
self.repodir = repodir
self.repo = Repo.init(repodir)
rootcommit_filename = join(repodir, "rootcommit")
with open(rootcommit_filename, "w") as fh:
fh.write("")
self.repo.index.add([basename(rootcommit_filename)])
str_date = str(datetime.utcnow())
self.repo.index.commit(
author=Actor("Peter", "*****@*****.**"),
author_date=str_date,
committer=Actor("Paul", "*****@*****.**"),
commit_date=str_date,
message="Root commit",
)
# Cache the remote per test
self._remote = None
# Keep track of cloned repositories that track self.repo
self._clone_dirs = []
freezer.stop()
def repo_commit_all_changes(repo, message=None, author=None):
if author is None:
author = Actor("system", "system")
if message is None:
message = ""
if repo.untracked_files:
repo.index.add(repo.untracked_files)
modified_files = repo.index.diff(None)
for m in modified_files:
repo.index.add([m.b_path])
commit = repo.index.commit(message, author=author)
return commit
def test_reflogentry(self):
nullhexsha = IndexObject.NULL_HEX_SHA
hexsha = 'F' * 40
actor = Actor('name', 'email')
msg = "message"
self.assertRaises(ValueError, RefLogEntry.new, nullhexsha, hexsha, 'noactor', 0, 0, "")
e = RefLogEntry.new(nullhexsha, hexsha, actor, 0, 1, msg)
assert e.oldhexsha == nullhexsha
assert e.newhexsha == hexsha
assert e.actor == actor
assert e.time[0] == 0
assert e.time[1] == 1
assert e.message == msg
# check representation (roughly)
assert repr(e).startswith(nullhexsha)
def test_index_mutation(self, rw_repo):
index = rw_repo.index
num_entries = len(index.entries)
cur_head = rw_repo.head
uname = u"Thomas Müller"
umail = "*****@*****.**"
with rw_repo.config_writer() as writer:
writer.set_value("user", "name", uname)
writer.set_value("user", "email", umail)
self.assertEqual(writer.get_value("user", "name"), uname)
# remove all of the files, provide a wild mix of paths, BaseIndexEntries,
# IndexEntries
def mixed_iterator():
count = 0
for entry in index.entries.values():
type_id = count % 4
if type_id == 0: # path
yield entry.path
elif type_id == 1: # blob
yield Blob(rw_repo, entry.binsha, entry.mode, entry.path)
elif type_id == 2: # BaseIndexEntry
yield BaseIndexEntry(entry[:4])
elif type_id == 3: # IndexEntry
yield entry
else:
raise AssertionError("Invalid Type")
count += 1
# END for each entry
# END mixed iterator
deleted_files = index.remove(mixed_iterator(), working_tree=False)
assert deleted_files
self.assertEqual(self._count_existing(rw_repo, deleted_files),
len(deleted_files))
self.assertEqual(len(index.entries), 0)
# reset the index to undo our changes
index.reset()
self.assertEqual(len(index.entries), num_entries)
# remove with working copy
deleted_files = index.remove(mixed_iterator(), working_tree=True)
assert deleted_files
self.assertEqual(self._count_existing(rw_repo, deleted_files), 0)
# reset everything
index.reset(working_tree=True)
self.assertEqual(self._count_existing(rw_repo, deleted_files),
len(deleted_files))
# invalid type
self.failUnlessRaises(TypeError, index.remove, [1])
# absolute path
deleted_files = index.remove(
[osp.join(rw_repo.working_tree_dir, "lib")], r=True)
assert len(deleted_files) > 1
self.failUnlessRaises(ValueError, index.remove, ["/doesnt/exists"])
# TEST COMMITTING
# commit changed index
cur_commit = cur_head.commit
commit_message = u"commit default head by Frèderic Çaufl€"
new_commit = index.commit(commit_message, head=False)
assert cur_commit != new_commit
self.assertEqual(new_commit.author.name, uname)
self.assertEqual(new_commit.author.email, umail)
self.assertEqual(new_commit.committer.name, uname)
self.assertEqual(new_commit.committer.email, umail)
self.assertEqual(new_commit.message, commit_message)
self.assertEqual(new_commit.parents[0], cur_commit)
self.assertEqual(len(new_commit.parents), 1)
self.assertEqual(cur_head.commit, cur_commit)
# commit with other actor
cur_commit = cur_head.commit
my_author = Actor(u"Frèderic Çaufl€", "*****@*****.**")
my_committer = Actor(u"Committing Frèderic Çaufl€",
"*****@*****.**")
commit_actor = index.commit(commit_message,
author=my_author,
committer=my_committer)
assert cur_commit != commit_actor
self.assertEqual(commit_actor.author.name, u"Frèderic Çaufl€")
self.assertEqual(commit_actor.author.email, "*****@*****.**")
self.assertEqual(commit_actor.committer.name,
u"Committing Frèderic Çaufl€")
self.assertEqual(commit_actor.committer.email, "*****@*****.**")
self.assertEqual(commit_actor.message, commit_message)
self.assertEqual(commit_actor.parents[0], cur_commit)
self.assertEqual(len(new_commit.parents), 1)
self.assertEqual(cur_head.commit, commit_actor)
self.assertEqual(cur_head.log()[-1].actor, my_committer)
# commit with author_date and commit_date
cur_commit = cur_head.commit
commit_message = u"commit with dates by Avinash Sajjanshetty"
new_commit = index.commit(commit_message,
author_date="2006-04-07T22:13:13",
commit_date="2005-04-07T22:13:13")
assert cur_commit != new_commit
print(new_commit.authored_date, new_commit.committed_date)
self.assertEqual(new_commit.message, commit_message)
self.assertEqual(new_commit.authored_date, 1144447993)
self.assertEqual(new_commit.committed_date, 1112911993)
# same index, no parents
commit_message = "index without parents"
commit_no_parents = index.commit(commit_message,
parent_commits=[],
head=True)
self.assertEqual(commit_no_parents.message, commit_message)
self.assertEqual(len(commit_no_parents.parents), 0)
self.assertEqual(cur_head.commit, commit_no_parents)
# same index, multiple parents
commit_message = "Index with multiple parents\n commit with another line"
commit_multi_parent = index.commit(commit_message,
parent_commits=(commit_no_parents,
new_commit))
self.assertEqual(commit_multi_parent.message, commit_message)
self.assertEqual(len(commit_multi_parent.parents), 2)
self.assertEqual(commit_multi_parent.parents[0], commit_no_parents)
self.assertEqual(commit_multi_parent.parents[1], new_commit)
self.assertEqual(cur_head.commit, commit_multi_parent)
# re-add all files in lib
# get the lib folder back on disk, but get an index without it
index.reset(new_commit.parents[0],
working_tree=True).reset(new_commit, working_tree=False)
lib_file_path = osp.join("lib", "git", "__init__.py")
assert (lib_file_path, 0) not in index.entries
assert osp.isfile(osp.join(rw_repo.working_tree_dir, lib_file_path))
# directory
entries = index.add(['lib'], fprogress=self._fprogress_add)
self._assert_entries(entries)
self._assert_fprogress(entries)
assert len(entries) > 1
# glob
entries = index.reset(new_commit).add([osp.join('lib', 'git', '*.py')],
fprogress=self._fprogress_add)
self._assert_entries(entries)
self._assert_fprogress(entries)
self.assertEqual(len(entries), 14)
# same file
entries = index.reset(new_commit).add(
[osp.join(rw_repo.working_tree_dir, 'lib', 'git', 'head.py')] * 2,
fprogress=self._fprogress_add)
self._assert_entries(entries)
self.assertEqual(entries[0].mode & 0o644, 0o644)
# would fail, test is too primitive to handle this case
# self._assert_fprogress(entries)
self._reset_progress()
self.assertEqual(len(entries), 2)
# missing path
self.failUnlessRaises(OSError,
index.reset(new_commit).add,
['doesnt/exist/must/raise'])
# blob from older revision overrides current index revision
old_blob = new_commit.parents[0].tree.blobs[0]
entries = index.reset(new_commit).add([old_blob],
fprogress=self._fprogress_add)
self._assert_entries(entries)
self._assert_fprogress(entries)
self.assertEqual(index.entries[(old_blob.path, 0)].hexsha,
old_blob.hexsha)
self.assertEqual(len(entries), 1)
# mode 0 not allowed
null_hex_sha = Diff.NULL_HEX_SHA
null_bin_sha = b"\0" * 20
self.failUnlessRaises(
ValueError,
index.reset(new_commit).add,
[BaseIndexEntry((0, null_bin_sha, 0, "doesntmatter"))])
# add new file
new_file_relapath = "my_new_file"
self._make_file(new_file_relapath, "hello world", rw_repo)
entries = index.reset(new_commit).add(
[BaseIndexEntry((0o10644, null_bin_sha, 0, new_file_relapath))],
fprogress=self._fprogress_add)
self._assert_entries(entries)
self._assert_fprogress(entries)
self.assertEqual(len(entries), 1)
self.assertNotEquals(entries[0].hexsha, null_hex_sha)
# add symlink
if not is_win:
for target in ('/etc/nonexisting', '/etc/passwd', '/etc'):
basename = "my_real_symlink"
link_file = osp.join(rw_repo.working_tree_dir, basename)
os.symlink(target, link_file)
entries = index.reset(new_commit).add(
[link_file], fprogress=self._fprogress_add)
self._assert_entries(entries)
self._assert_fprogress(entries)
self.assertEqual(len(entries), 1)
self.assertTrue(S_ISLNK(entries[0].mode))
self.assertTrue(
S_ISLNK(index.entries[index.entry_key(
"my_real_symlink", 0)].mode))
# we expect only the target to be written
self.assertEqual(
index.repo.odb.stream(
entries[0].binsha).read().decode('ascii'), target)
os.remove(link_file)
# end for each target
# END real symlink test
# add fake symlink and assure it checks-our as symlink
fake_symlink_relapath = "my_fake_symlink"
link_target = "/etc/that"
fake_symlink_path = self._make_file(fake_symlink_relapath, link_target,
rw_repo)
fake_entry = BaseIndexEntry(
(0o120000, null_bin_sha, 0, fake_symlink_relapath))
entries = index.reset(new_commit).add([fake_entry],
fprogress=self._fprogress_add)
self._assert_entries(entries)
self._assert_fprogress(entries)
assert entries[0].hexsha != null_hex_sha
self.assertEqual(len(entries), 1)
self.assertTrue(S_ISLNK(entries[0].mode))
# assure this also works with an alternate method
full_index_entry = IndexEntry.from_base(
BaseIndexEntry((0o120000, entries[0].binsha, 0, entries[0].path)))
entry_key = index.entry_key(full_index_entry)
index.reset(new_commit)
assert entry_key not in index.entries
index.entries[entry_key] = full_index_entry
index.write()
index.update() # force reread of entries
new_entry = index.entries[entry_key]
assert S_ISLNK(new_entry.mode)
# a tree created from this should contain the symlink
tree = index.write_tree()
assert fake_symlink_relapath in tree
index.write() # flush our changes for the checkout
# checkout the fakelink, should be a link then
assert not S_ISLNK(os.stat(fake_symlink_path)[ST_MODE])
os.remove(fake_symlink_path)
index.checkout(fake_symlink_path)
# on windows we will never get symlinks
if is_win:
# simlinks should contain the link as text ( which is what a
# symlink actually is )
with open(fake_symlink_path, 'rt') as fd:
self.assertEqual(fd.read(), link_target)
else:
self.assertTrue(S_ISLNK(os.lstat(fake_symlink_path)[ST_MODE]))
# TEST RENAMING
def assert_mv_rval(rval):
for source, dest in rval:
assert not osp.exists(source) and osp.exists(dest)
# END for each renamed item
# END move assertion utility
self.failUnlessRaises(ValueError, index.move, ['just_one_path'])
# file onto existing file
files = ['AUTHORS', 'LICENSE']
self.failUnlessRaises(GitCommandError, index.move, files)
# again, with force
assert_mv_rval(index.move(files, f=True))
# files into directory - dry run
paths = ['LICENSE', 'VERSION', 'doc']
rval = index.move(paths, dry_run=True)
self.assertEqual(len(rval), 2)
assert osp.exists(paths[0])
# again, no dry run
rval = index.move(paths)
assert_mv_rval(rval)
# dir into dir
rval = index.move(['doc', 'test'])
assert_mv_rval(rval)
# TEST PATH REWRITING
######################
count = [0]
def rewriter(entry):
rval = str(count[0])
count[0] += 1
return rval
# END rewriter
def make_paths():
# two existing ones, one new one
yield 'CHANGES'
yield 'ez_setup.py'
yield index.entries[index.entry_key('README', 0)]
yield index.entries[index.entry_key('.gitignore', 0)]
for fid in range(3):
fname = 'newfile%i' % fid
with open(fname, 'wb') as fd:
fd.write(b"abcd")
yield Blob(rw_repo, Blob.NULL_BIN_SHA, 0o100644, fname)
# END for each new file
# END path producer
paths = list(make_paths())
self._assert_entries(index.add(paths, path_rewriter=rewriter))
for filenum in range(len(paths)):
assert index.entry_key(str(filenum), 0) in index.entries
# TEST RESET ON PATHS
######################
arela = "aa"
brela = "bb"
afile = self._make_file(arela, "adata", rw_repo)
bfile = self._make_file(brela, "bdata", rw_repo)
akey = index.entry_key(arela, 0)
bkey = index.entry_key(brela, 0)
keys = (akey, bkey)
absfiles = (afile, bfile)
files = (arela, brela)
for fkey in keys:
assert fkey not in index.entries
index.add(files, write=True)
nc = index.commit("2 files committed", head=False)
for fkey in keys:
assert fkey in index.entries
# just the index
index.reset(paths=(arela, afile))
assert akey not in index.entries
assert bkey in index.entries
# now with working tree - files on disk as well as entries must be recreated
rw_repo.head.commit = nc
for absfile in absfiles:
os.remove(absfile)
index.reset(working_tree=True, paths=files)
for fkey in keys:
assert fkey in index.entries
for absfile in absfiles:
assert osp.isfile(absfile)
def blame_incremental(self, rev, file, **kwargs):
"""Iterator for blame information for the given file at the given revision.
Unlike .blame(), this does not return the actual file's contents, only
a stream of BlameEntry tuples.
:parm rev: revision specifier, see git-rev-parse for viable options.
:return: lazy iterator of BlameEntry tuples, where the commit
indicates the commit to blame for the line, and range
indicates a span of line numbers in the resulting file.
If you combine all line number ranges outputted by this command, you
should get a continuous range spanning all line numbers in the file.
"""
data = self.git.blame(rev,
'--',
file,
p=True,
incremental=True,
stdout_as_string=False,
**kwargs)
commits = dict()
stream = (line for line in data.split(b'\n') if line)
while True:
line = next(
stream
) # when exhausted, causes a StopIteration, terminating this function
hexsha, orig_lineno, lineno, num_lines = line.split()
lineno = int(lineno)
num_lines = int(num_lines)
orig_lineno = int(orig_lineno)
if hexsha not in commits:
# Now read the next few lines and build up a dict of properties
# for this commit
props = dict()
while True:
line = next(stream)
if line == b'boundary':
# "boundary" indicates a root commit and occurs
# instead of the "previous" tag
continue
tag, value = line.split(b' ', 1)
props[tag] = value
if tag == b'filename':
# "filename" formally terminates the entry for --incremental
orig_filename = value
break
c = Commit(
self,
hex_to_bin(hexsha),
author=Actor(
safe_decode(props[b'author']),
safe_decode(
props[b'author-mail'].lstrip(b'<').rstrip(b'>'))),
authored_date=int(props[b'author-time']),
committer=Actor(
safe_decode(props[b'committer']),
safe_decode(props[b'committer-mail'].lstrip(
b'<').rstrip(b'>'))),
committed_date=int(props[b'committer-time']))
commits[hexsha] = c
else:
# Discard all lines until we find "filename" which is
# guaranteed to be the last line
while True:
line = next(
stream) # will fail if we reach the EOF unexpectedly
tag, value = line.split(b' ', 1)
if tag == b'filename':
orig_filename = value
break
yield BlameEntry(commits[hexsha], range(lineno,
lineno + num_lines),
safe_decode(orig_filename),
range(orig_lineno, orig_lineno + num_lines))
def blame_incremental(self, rev, file, **kwargs):
"""Iterator for blame information for the given file at the given revision.
Unlike .blame(), this does not return the actual file's contents, only
a stream of (commit, range) tuples.
:parm rev: revision specifier, see git-rev-parse for viable options.
:return: lazy iterator of (git.Commit, range) tuples, where the commit
indicates the commit to blame for the line, and range
indicates a span of line numbers in the resulting file.
If you combine all line number ranges outputted by this command, you
should get a continuous range spanning all line numbers in the file.
"""
data = self.git.blame(rev,
'--',
file,
p=True,
incremental=True,
stdout_as_string=False,
**kwargs)
commits = dict()
stream = iter(data.splitlines())
while True:
line = next(
stream
) # when exhausted, casues a StopIteration, terminating this function
hexsha, _, lineno, num_lines = line.split()
lineno = int(lineno)
num_lines = int(num_lines)
if hexsha not in commits:
# Now read the next few lines and build up a dict of properties
# for this commit
props = dict()
while True:
line = next(stream)
if line == b'boundary':
# "boundary" indicates a root commit and occurs
# instead of the "previous" tag
continue
tag, value = line.split(b' ', 1)
props[tag] = value
if tag == b'filename':
# "filename" formally terminates the entry for --incremental
break
c = Commit(
self,
hex_to_bin(hexsha),
author=Actor(
safe_decode(props[b'author']),
safe_decode(
props[b'author-mail'].lstrip(b'<').rstrip(b'>'))),
authored_date=int(props[b'author-time']),
committer=Actor(
safe_decode(props[b'committer']),
safe_decode(props[b'committer-mail'].lstrip(
b'<').rstrip(b'>'))),
committed_date=int(props[b'committer-time']),
message=safe_decode(props[b'summary']))
commits[hexsha] = c
else:
# Discard the next line (it's a filename end tag)
line = next(stream)
assert line.startswith(
b'filename'), 'Unexpected git blame output'
yield commits[hexsha], range(lineno, lineno + num_lines)
def as_git_actor(self):
hostname = 'hostname' # XXX
email = '%s@%s' % (self.username, hostname)
author = Actor(self.username, email)
return author
