def html():
"""Build/make the HTML documentation.
"""
# Rebuild the static images.
if util.yes("Do you want to rebuild the static images (y/n)?"):
static()
# Update the download link.
try:
commit = git('rev-list', '--tags', '--max-count=1').stdout.rstrip()
lastversion = git.describe('--tags', commit).stdout.rstrip()
# This is simpler but doesn't always return the latest tag:
# lastversion = git.describe('--tag', abbrev=0).stdout.rstrip()
except ErrorReturnCode_128:
pass # No tags recorded; leave download link as is
else:
date = git.log('-1', lastversion,
date='short', format='%ad').stdout[8:18]
rpls = [(r'(ModelicaRes)-.*(\.tar)', r'\1-%s\2' % lastversion[1:]),
(r'(Latest version<br>\().*(\)</a>)',
r'\1%s, %s\2' % (lastversion, date)),
]
util.replace('_templates/download.html', rpls)
# Build the documentation.
make_dirs()
sphinx = sphinx_build.bake(b='html', d='build/doctrees')
print(sphinx('.', BUILD_DIR))
# Spellcheck.
if util.yes("Do you want to spellcheck the HTML documentation (y/n)?"):
spellcheck()
def stat(self, email):
'''
returns a statistic of a git author with the given e_mail.
:param email: name of the author
:rtype: a dict with the statistics
'''
sums = [0, 0, 0]
for line in git.log("--all", "--stat", '--author={0}'.format(email), _tty_in=True, _tty_out=False, _iter=True):
line = line[:-1]
if " files changed" in line:
line = line.replace(" insertions(+)", "")
line = line.replace(" insertion(+)", "")
line = line.replace(" deletion(-)", "")
line = line.replace(" deletions(-)", "")
line = line.replace(" files changed", "")
line = line.split(",")
data = [int(i) for i in line]
for index in range(0, len(data)):
sums[index] += data[index]
return {"email": email,
"fileschanged": sums[0],
"inserted": sums[1],
"deleted": sums[2],
"lineschanged": sums[1] + sums[2],
}
def emails(self, format_arg=None):
'''
returns the emails of the authors either as a text or as a dict. The
format is specified as an argument.
:param format_arg: if "dict" is specified a dict will be returned
:rtype: dict or arry of e-mails dependent on format_arg
'''
if format_arg is None:
format_string = "'%aN' <%cE>"
elif format_arg == 'dict':
format_string = "%aN\t%cE"
result = sort(git.log(
"--all", "--format=" + format_string,
_tty_in=True, _tty_out=False, _piped=True), "-u")
if format_arg is None:
return result
elif format_arg == "dict":
list = result.replace("\n", "\t").split("\t")[:-1]
it = iter(list)
emails = dict(zip(it, it))
for name in emails:
emails[name] = emails[name]
return emails
def test_concurrent_edition_with_conflict(self):
self.page.raw = "- item1"
Git().commit(self.page, message="original")
response1 = self.client.get(self.edit_url)
response2 = self.client.get(self.edit_url)
data1 = response1.context[0]['form'].initial
data1["raw"] = '- item2'
data1["message"] = "add item2"
data2 = response2.context[0]['form'].initial
data2["raw"] = '- item0'
data1["message"] = "add item0"
r = self.client.post(self.edit_url, data1)
self.assertRedirects(r, reverse('waliki_detail', args=(self.page.slug,)))
r = self.client.post(self.edit_url, data2)
self.assertRedirects(r, self.edit_url)
self.assertEqual('', git.status('--porcelain', self.page.path).stdout.decode('utf8'))
self.assertIn('Merged with conflict', git.log("--no-color", "--pretty=format:%s", "-n 1", self.page.path).stdout.decode('utf8'))
self.assertRegexpMatches(self.page.raw,'<<<<<<< HEAD\n- item2\n=======\n- item0\n>>>>>>> [0-9a-f]{7}\n')
# can edit in conflict
response = self.client.get(self.edit_url)
data1 = response.context[0]['form'].initial
data1["raw"] = '- item0\n- item2'
data1["message"] = "fixing :)"
response = self.client.post(self.edit_url, data1)
self.assertRedirects(response, reverse('waliki_detail', args=(self.page.slug,)))
self.assertEqual(self.page.raw, '- item0\n- item2')
def history(self, page):
data = [("commit", "%h"), ("author", "%an"), ("date", "%ad"),
("date_relative", "%ar"), ("message", "%s")]
format = "{%s}" % ','.join(
[""" \"%s\": \"%s\" """ % item for item in data])
output = git.log('--format=%s' % format, '-z', '--shortstat',
page.abspath)
output = output.replace('\x00', '').split('\n')[:-1]
history = []
for line in output:
if line.startswith('{'):
history.append(json.loads(line))
else:
insertion = re.match(r'.* (\d+) insertion', line)
deletion = re.match(r'.* (\d+) deletion', line)
history[-1]['insertion'] = int(
insertion.group(1)) if insertion else 0
history[-1]['deletion'] = int(
deletion.group(1)) if deletion else 0
max_changes = float(
max([(v['insertion'] + v['deletion']) for v in history])) or 1.0
for v in history:
v.update({
'insertion_relative': (v['insertion'] / max_changes) * 100,
'deletion_relative': (v['deletion'] / max_changes) * 100
})
return history
def whatchanged(self, skip=0, max_count=None, include_diff=False):
GIT_LOG_FORMAT = '%x1f'.join(['%an', '%ae', '%h', '%s', '%at'])
pages = []
args = ["--pretty=format:%s" % GIT_LOG_FORMAT, '--skip=%d' % skip]
if max_count:
args.append('--max-count=%d' % max_count)
raw_log = git.whatchanged(*args).stdout.decode('utf8')
logs = re.findall(
r'((.*)\x1f(.*)\x1f(.*)\x1f(.*)\x1f(.*))?\n:.*\t(.*)',
raw_log,
flags=re.MULTILINE | re.UNICODE)
for log in logs:
if log[0]:
log = list(log[1:])
log[-1] = [log[-1]] # pages
pages.append(list(log))
else:
pages[-1][-1].append(log[-1])
if include_diff:
args = ['--no-color', '-p', '--format="%x1f"', '--skip=%d' % skip]
if max_count:
args.append('--max-count=%d' % max_count)
diffs = git.log(*args).stdout.decode('utf8').split('\x1f')[1:]
return zip(pages, diffs)
return pages
def history(self, page):
GIT_COMMIT_FIELDS = [
'commit', 'author', 'date', 'date_relative', 'date_parsed',
'message'
]
GIT_LOG_FORMAT = '%x1f'.join(['%h', '%an', '%ad', '%ar', '%ai', '%s'
]) + '%x1e'
output = git.log('--format=%s' % GIT_LOG_FORMAT, '--follow', '-z',
'--shortstat', page.abspath)
output = output.split('\n')
history = []
for line in output:
if '\x1f' in line:
log = line.strip('\x1e\x00').split('\x1f')
history.append(dict(zip(GIT_COMMIT_FIELDS, log)))
else:
insertion = re.match(r'.* (\d+) insertion', line)
deletion = re.match(r'.* (\d+) deletion', line)
history[-1]['insertion'] = int(
insertion.group(1)) if insertion else 0
history[-1]['deletion'] = int(
deletion.group(1)) if deletion else 0
max_changes = float(
max([(v['insertion'] + v['deletion']) for v in history])) or 1.0
for v in history:
v.update({
'insertion_relative':
str((v['insertion'] / max_changes) * 100),
'deletion_relative':
str((v['deletion'] / max_changes) * 100),
'date_parsed':
parse_datetime('%s %s%s' % tuple(v['date_parsed'].split()))
})
return history
def whatchanged(self, skip=0, max_count=None, include_diff=False):
GIT_LOG_FORMAT = '%x1f'.join(['%an', '%ae', '%h', '%s', '%at'])
pages = []
args = ["--pretty=format:%s" % GIT_LOG_FORMAT, '--skip=%d' % skip]
if max_count:
args.append('--max-count=%d' % max_count)
raw_log = git.whatchanged(*args).stdout.decode('utf8')
logs = re.findall(r'((.*)\x1f(.*)\x1f(.*)\x1f(.*)\x1f(.*))?\n:.*\t(.*)', raw_log, flags=re.MULTILINE | re.UNICODE)
for log in logs:
if log[0]:
log = list(log[1:])
log[-1] = [log[-1]] # pages
pages.append(list(log))
else:
pages[-1][-1].append(log[-1])
if include_diff:
args = ['--no-color', '-p', '--format="%x1f"', '--skip=%d' % skip]
if max_count:
args.append('--max-count=%d' % max_count)
diffs = git.log(*args).stdout.decode('utf8').split('\x1f')[1:]
return zip(pages, diffs)
return pages
def whatchanged(self, skip=0, max_count=None, include_diff=False):
GIT_LOG_FORMAT = "%x1f".join(["%an", "%ae", "%h", "%s", "%at"])
pages = []
args = ["--pretty=format:%s" % GIT_LOG_FORMAT, "--skip=%d" % skip]
if max_count:
args.append("--max-count=%d" % max_count)
raw_log = git.whatchanged(*args).stdout.decode("utf8")
logs = re.findall(
r"((.*)\x1f(.*)\x1f(.*)\x1f(.*)\x1f(.*))?\n:.*\t(.*)", raw_log, flags=re.MULTILINE | re.UNICODE
)
for log in logs:
if log[0]:
log = list(log[1:])
log[-1] = [log[-1]] # pages
pages.append(list(log))
else:
pages[-1][-1].append(log[-1])
if include_diff:
args = ["--no-color", "-p", '--format="%x1f"', "--skip=%d" % skip]
if max_count:
args.append("--max-count=%d" % max_count)
diffs = git.log(*args).stdout.decode("utf8").split("\x1f")[1:]
return zip(pages, diffs)
return pages
def html():
"""Build/make the HTML documentation.
"""
# Rebuild the static images.
if util.yes("Do you want to rebuild the static images (y/n)?"):
static()
# Update the download link.
try:
commit = git('rev-list', '--tags', '--max-count=1').stdout.rstrip()
lastversion = git.describe('--tags', commit).stdout.rstrip()
# This is simpler but doesn't always return the latest tag:
# lastversion = git.describe('--tag', abbrev=0).stdout.rstrip()
except ErrorReturnCode_128:
pass # No tags recorded; leave download link as is
else:
date = git.log('-1', lastversion, date='short',
format='%ad').stdout[8:18]
rpls = [
(r'(ModelicaRes)-.*(\.tar)', r'\1-%s\2' % lastversion[1:]),
(r'(Latest version<br>\().*(\)</a>)',
r'\1%s, %s\2' % (lastversion, date)),
]
util.replace('_templates/download.html', rpls)
# Build the documentation.
make_dirs()
sphinx = sphinx_build.bake(b='html', d='build/doctrees')
print((sphinx('.', BUILD_DIR)))
# Spellcheck.
if util.yes("Do you want to spellcheck the HTML documentation (y/n)?"):
spellcheck()
def history(self, page):
GIT_COMMIT_FIELDS = ["commit", "author", "date", "date_relative", "message"]
GIT_LOG_FORMAT = "%x1f".join(["%h", "%an", "%ad", "%ar", "%s"]) + "%x1e"
output = git.log("--format=%s" % GIT_LOG_FORMAT, "--follow", "-z", "--shortstat", page.abspath)
output = output.split("\n")
history = []
for line in output:
if "\x1f" in line:
log = line.strip("\x1e\x00").split("\x1f")
history.append(dict(zip(GIT_COMMIT_FIELDS, log)))
else:
insertion = re.match(r".* (\d+) insertion", line)
deletion = re.match(r".* (\d+) deletion", line)
history[-1]["insertion"] = int(insertion.group(1)) if insertion else 0
history[-1]["deletion"] = int(deletion.group(1)) if deletion else 0
max_changes = float(max([(v["insertion"] + v["deletion"]) for v in history])) or 1.0
for v in history:
v.update(
{
"insertion_relative": str((v["insertion"] / max_changes) * 100),
"deletion_relative": str((v["deletion"] / max_changes) * 100),
}
)
return history
def test_commit_existent_page_with_no_previous_commits(self):
response = self.client.get(self.edit_url)
data = response.context[0]['form'].initial
data["raw"] = "test content"
data["message"] = "testing :)"
response = self.client.post(self.edit_url, data)
self.assertEqual(self.page.raw, "test content")
self.assertEqual(Git().version(self.page, 'HEAD'), "test content")
self.assertIn("testing :)", git.log('-s', '--format=%s', self.page.path))
def test_commit_existent_page_with_previous_commits(self):
self.page.raw = "lala"
Git().commit(self.page, message="previous commit")
assert "previous commit" in git.log('-s', '--format=%s', self.page.path)
response = self.client.get(self.edit_url)
data = response.context[0]['form'].initial
data["raw"] = "test content"
response = self.client.post(self.edit_url, data)
self.assertEqual(self.page.raw, "test content")
self.assertEqual(Git().version(self.page, 'HEAD'), "test content")
def test_concurrent_edition_no_existent_page(self):
assert not Page.objects.filter(slug='test2').exists()
url = reverse('waliki_edit', args=('test2', ))
response1 = self.client.post(url)
response2 = self.client.post(url)
page = Page.objects.get(slug='test2')
data1 = response1.context[0]['form'].initial
data1["raw"] = '- item2\n'
data1["message"] = "add item2"
data1["title"] = "a title"
data2 = response2.context[0]['form'].initial
data2["raw"] = '- item0\n'
data2["message"] = "add item0"
data2["title"] = "another title"
self.client.post(url, data1)
with patch('waliki.views.messages') as messages:
response = self.client.post(url, data2)
# there is a warning
self.assertTrue(messages.warning.called)
self.assertIsInstance(messages.warning.call_args[0][1],
Page.EditionConflict)
# redirect
self.assertRedirects(response, url)
# file committed with conflict
self.assertEqual(
'',
git.status('--porcelain', page.path).stdout.decode('utf8'))
self.assertIn(
'Merged with conflict',
git.log("--pretty=format:%s", "-n 1",
page.path).stdout.decode('utf8'))
self.assertRegexpMatches(
page.raw, r"""<<<<<<< HEAD\n- item2
=======\n- item0\n>>>>>>> [0-9a-f]{7}\n""")
page = Page.objects.get(slug='test2') # refresh
self.assertEqual(page.title, "another title")
# can edit in conflict
response = self.client.get(url)
data1 = response.context[0]['form'].initial
data1["raw"] = '- item0\n- item2\n'
data1["message"] = "fixing :)"
response = self.client.post(url, data1)
self.assertRedirects(response,
reverse('waliki_detail', args=(page.slug, )))
self.assertEqual(page.raw, '- item0\n- item2\n')
def install_site(repo_url):
source = source_path(repo_url)
destination = destination_path(repo_url)
try:
os.chdir(source)
jekyll.build('--destination', destination)
commit_sha = git.log("-n1", "--pretty='%H'")
logging.info("Installed site from {}".format(repo_url))
return get_raw_sha(commit_sha)
except Exception as ex:
logging.error("Failed to build the Jekyll file. Exception follows")
logging.exception(ex)
return None
def test_concurrent_edition_no_existent_page(self):
assert not Page.objects.filter(slug="test2").exists()
url = reverse("waliki_edit", args=("test2",))
response1 = self.client.post(url)
response2 = self.client.post(url)
page = Page.objects.get(slug="test2")
data1 = response1.context[0]["form"].initial
data1["raw"] = "- item2\n"
data1["message"] = "add item2"
data1["title"] = "a title"
data2 = response2.context[0]["form"].initial
data2["raw"] = "- item0\n"
data2["message"] = "add item0"
data2["title"] = "another title"
self.client.post(url, data1)
with patch("waliki.views.messages") as messages:
response = self.client.post(url, data2)
# there is a warning
self.assertTrue(messages.warning.called)
self.assertIsInstance(messages.warning.call_args[0][1], Page.EditionConflict)
# redirect
self.assertRedirects(response, url)
# file committed with conflict
self.assertEqual("", git.status("--porcelain", page.path).stdout.decode("utf8"))
self.assertIn("Merged with conflict", git.log("--pretty=format:%s", "-n 1", page.path).stdout.decode("utf8"))
self.assertRegexpMatches(
page.raw,
r"""<<<<<<< HEAD\n- item2
=======\n- item0\n>>>>>>> [0-9a-f]{7}\n""",
)
page = Page.objects.get(slug="test2") # refresh
self.assertEqual(page.title, "another title")
# can edit in conflict
response = self.client.get(url)
data1 = response.context[0]["form"].initial
data1["raw"] = "- item0\n- item2\n"
data1["message"] = "fixing :)"
response = self.client.post(url, data1)
self.assertRedirects(response, reverse("waliki_detail", args=(page.slug,)))
self.assertEqual(page.raw, "- item0\n- item2\n")
def emails(self, format_arg=None):
if format_arg is None:
format_string = "'%aN' <%cE>"
elif format_arg == 'dict':
format_string = "%aN\t%cE"
result = sort(git.log(
"--all", "--format=" + format_string,
_tty_in=True, _tty_out=False, _piped=True), "-u")
if format_arg is None:
return result
elif format_arg == "dict":
list = result.replace("\n", "\t").split("\t")[:-1]
it = iter(list)
authors = dict(zip(it, it))
for name in authors:
authors[name] = authors[name]
return authors
def history(self, page):
GIT_COMMIT_FIELDS = ['commit', 'author', 'date', 'date_relative', 'message']
GIT_LOG_FORMAT = '%x1f'.join(['%h', '%an', '%ad', '%ar', '%s']) + '%x1e'
output = git.log('--format=%s' % GIT_LOG_FORMAT, '--follow', '-z', '--shortstat', page.abspath)
output = output.split('\n')
history = []
for line in output:
if '\x1f' in line:
log = line.strip('\x1e\x00').split('\x1f')
history.append(dict(zip(GIT_COMMIT_FIELDS, log)))
else:
insertion = re.match(r'.* (\d+) insertion', line)
deletion = re.match(r'.* (\d+) deletion', line)
history[-1]['insertion'] = int(insertion.group(1)) if insertion else 0
history[-1]['deletion'] = int(deletion.group(1)) if deletion else 0
max_changes = float(max([(v['insertion'] + v['deletion']) for v in history])) or 1.0
for v in history:
v.update({'insertion_relative': str((v['insertion'] / max_changes) * 100),
'deletion_relative': str((v['deletion'] / max_changes) * 100)})
return history
def stat(self, email):
sums = [0, 0, 0]
for line in git.log("--all", "--stat", '--author={0}'.format(email), _tty_in=True, _tty_out=False, _iter=True):
line = line[:-1]
if " files changed" in line:
line = line.replace(" insertions(+)", "")
line = line.replace(" insertion(+)", "")
line = line.replace(" deletion(-)", "")
line = line.replace(" deletions(-)", "")
line = line.replace(" files changed", "")
line = line.split(",")
data = [int(i) for i in line]
for index in range(0, len(data)):
sums[index] += data[index]
return {"email": email,
"fileschanged": sums[0],
"inserted": sums[1],
"deleted": sums[2],
"lineschanged": sums[1] + sums[2],
}
def history(self, page):
data = [("commit", "%h"),
("author", "%an"),
("date", "%ad"),
("date_relative", "%ar"),
("message", "%s")]
format = "{%s}" % ','.join([""" \"%s\": \"%s\" """ % item for item in data])
output = git.log('--format=%s' % format, '-z', '--shortstat', page.abspath)
output = output.replace('\x00', '').split('\n')[:-1]
history = []
for line in output:
if line.startswith('{'):
history.append(json.loads(line))
else:
insertion = re.match(r'.* (\d+) insertion', line)
deletion = re.match(r'.* (\d+) deletion', line)
history[-1]['insertion'] = int(insertion.group(1)) if insertion else 0
history[-1]['deletion'] = int(deletion.group(1)) if deletion else 0
max_changes = float(max([(v['insertion'] + v['deletion']) for v in history])) or 1.0
for v in history:
v.update({'insertion_relative': (v['insertion'] / max_changes) * 100,
'deletion_relative': (v['deletion'] / max_changes) * 100})
return history
def __latest_commit(self):
return git.log(-1, pretty="format:%H")
def __latest_commit(self):
return git.log(-1, pretty="format:%H")
def last_version(self, page):
try:
return six.text_type(git.log("--pretty=format:%h", "-n 1", page.path))
except ErrorReturnCode:
return None
def get_current_commit(repo_url):
source = source_path(repo_url)
os.chdir('{}'.format(source))
git.pull('origin', 'master')
commit_sha = git.log("-n1", "--pretty='%H'")
return get_raw_sha(commit_sha)
state, path = line.split()
if state == "UU":
print("ours", path)
git.checkout("--ours", path)
git.add(path)
else:
print(state, path)
line = out_buf.readline()
# Check to see if any files changed only in formatting
out_buf = StringIO()
git.status("--porcelain=1", ".", _out=out_buf)
out_buf.seek(0)
line = out_buf.readline()
while line:
state = line.split()[0]
if state in ("D", "R", "DD"):
line = out_buf.readline()
continue
state, path = line.split()
log_buf = StringIO()
git.log("--pretty=tformat:%H", "25ae98f..HEAD", path, _out=log_buf, _tty_out=False)
log_buf.seek(0)
commits = []
for line in log_buf.readlines():
commits.append(line.strip())
if state in ["UU", "M"] and commits == ["a52eb88031620a81521b937f2a0651dbac2bb350"]:
git.checkout("--theirs", path)
git.add(path)
line = out_buf.readline()
def last_version(self, page):
try:
return six.text_type(
git.log("--pretty=format:%h", "-n 1", page.path))
except ErrorReturnCode:
return None
