def diff_files_slow(self, left, right):
DATA_ROOT = current_app.config.get('DATA_ROOT')
_in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self)
names = prun("""git diff --name-only %(left)s %(right)s""" % locals(), cwd=_in)
data = prun("""git diff %(left)s %(right)s""" % locals(), cwd=_in)
file_diff = {}
# list of known xml file extensions
xml_ext = ['.svg', '.ttx', '.plist', '.glif', '.xml']
for f in names.splitlines():
param = {'left': left, 'right': right, 'name': f}
#file_diff[f] = prun("""git diff %(left)s %(right)s -- "%(name)s" """ % param, cwd=_in)
if os.path.splitext(f)[1] in xml_ext:
left_content = prun("""git show %(left)s:"%(name)s" """ % param, cwd=_in)
right_content = prun("""git show %(right)s:"%(name)s" """ % param, cwd=_in)
if left_content.startswith('fatal:'):
left_content = ''
if right_content.startswith('fatal:'):
right_content = ''
# left
try:
left_xml = etree.tostring(etree.fromstring(left_content), pretty_print=True)
except etree.XMLSyntaxError:
# if returned data is not real xml
left_xml = left_content
# right
try:
right_xml = etree.tostring(etree.fromstring(right_content), pretty_print=True)
except etree.XMLSyntaxError:
# if returned data is not real xml
right_xml = right_content
file_diff[f] = "".join([x for x in difflib.unified_diff(left_xml, right_xml, fromfile="a/" + f, tofile="b/" + f, lineterm='')])
else:
file_diff[f] = prun("""git diff %(left)s %(right)s -- "%(name)s" """ % param, cwd=_in)
for f in file_diff.keys():
text = file_diff[f]
text = text.replace("&", "&").replace(">", ">").replace("<", "<")
t = []
for l in text.splitlines():
if l[0] == '+':
l = "<ins>%s</ins>" % l
elif l[0] == "-":
l = "<del>%s</del>" % l
elif l[0] == "^":
l = "<ins>%s</ins>" % l
t.append(l.rstrip(" \t\n\r"))
file_diff[f] = "\n".join(t)
return file_diff
def revision_file(self, revision, fn):
""" Read specific file from revision """
# XXX: [xen] need review here, not sure that it is 100% safe
DATA_ROOT = current_app.config.get('DATA_ROOT')
_in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self)
fn = fn.replace('"', '')
# XXX: result can be tree
data = prun('git show "%(revision)s:%(fn)s"' % locals(), cwd=_in)
mime = magic.from_buffer(data, mime=True)
if mime.startswith("text"):
data = unicode(data, "utf8")
return mime, data
def execute(self, pipedata, prefix=""):
ttxfiles = []
ufo = []
sfd = []
bin = []
for p in pipedata['process_files']:
if p.endswith('.ttx'):
ttxfiles.append(p)
elif p.endswith('.sfd'):
sfd.append(p)
elif p.endswith('.ufo'):
ufo.append(p)
elif p.endswith('.ttf'):
bin.append(p)
elif p.endswith('.otf'):
bin.append(p)
self.stdout_pipe.write('Convert sources to TTF\n', prefix="### %s " % prefix)
if ttxfiles:
self.execute_ttx(self.project_root, self.builddir, ttxfiles)
if ufo:
self.execute_ufo_sfd(self.project_root, self.builddir, ufo)
if sfd:
self.execute_ufo_sfd(self.project_root, self.builddir, sfd)
if bin:
self.execute_bin(self.project_root, self.builddir, bin)
binfiles = self.movebin_to_builddir(self.builddir, ufo + ttxfiles + sfd + bin)
SCRIPTPATH = op.join('scripts', 'fix-ttf-vmet.py')
command = ' '.join(map(lambda x: op.join(self.builddir, x), binfiles))
prun('python %s %s' % (SCRIPTPATH, command),
cwd=op.abspath(op.join(op.dirname(__file__), '..', '..')),
log=self.stdout_pipe)
pipedata['bin_files'] = binfiles
return pipedata
def revision_tree(self, revision):
""" Get specific revision files as tree in format supported
by tree macros """
DATA_ROOT = current_app.config.get('DATA_ROOT')
_in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self)
d = {}
for file in prun("git ls-tree --name-only -r %(revision)s" % locals(), cwd=_in).splitlines():
level = d
for part in file.split("/"):
if part not in level:
level[part] = {}
level = level[part]
return d
def diff_files(self, left, right):
DATA_ROOT = current_app.config.get('DATA_ROOT')
_in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self)
data = prun("""git diff %(left)s %(right)s""" % locals(), cwd=_in)
data = data.decode('utf-8').encode('ascii', 'xmlcharrefreplace')
file_diff = {}
t = []
current_file = None
source_start = target_start = 0
RE_HUNK_HEADER = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)")
def unpack(source_start, source_len, target_start, target_len, section_header):
return int(source_start), int(target_start)
data_lines = data.splitlines()
data_lines.append(' ')
for l in data_lines:
if l.startswith('diff --git'):
if current_file:
file_diff[current_file] = "\n".join(t)
current_file = l[14 + (len(l[11:]) - 1) / 2:]
t = []
continue
elif l.startswith('index'):
continue
elif l.startswith('---'):
continue
elif l.startswith('+++'):
continue
elif l.startswith('new file'):
continue
l = l.replace("&", "&").replace(">", ">").replace("<", "<")
first = l[0]
last = l[1:]
l = l.rstrip(" \t\n\r")
if l.startswith('Binary files'):
l = '<tr><td></td><td></td><td><span class="text-center">Binary file not shown</span></td></tr>'
elif first == '@':
re_hunk_header = RE_HUNK_HEADER.match(l)
if re_hunk_header:
hunk_info = re_hunk_header.groups()
source_start, target_start = unpack(*hunk_info)
l = "<tr class='hunk'><td></td><td></td><td><pre>%s<pre></td></tr>" % (l)
elif first == "+":
target_start = target_start + 1
l = "<tr class='ins'><td></td><td class='num'>%s</td><td><pre>%s</pre></td></tr>" % (target_start, last)
elif first == "-":
source_start = source_start + 1
l = "<tr class='del'><td class='num'>%s</td><td></td><td class='del'><pre>%s</pre></td></tr>" % (source_start, last)
else:
target_start = target_start + 1
source_start = source_start + 1
l = "<tr><td class='num'>%s</td><td class='num'>%s</td><td><pre>%s<pre></td></tr>" % (source_start, target_start, l)
t.append(l)
if current_file:
file_diff[current_file] = "\n".join(t)
return file_diff
def current_revision(self):
DATA_ROOT = current_app.config.get('DATA_ROOT')
_in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self)
return prun("git rev-parse --short HEAD", cwd=_in).strip()
def revision_info(self, revision):
""" Return revision info for selected git commit """
DATA_ROOT = current_app.config.get('DATA_ROOT')
_in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self)
return prun("git show --quiet --format=short %(revision)s" % locals(), cwd=_in).decode('utf-8')
def test_ots(self):
""" Is TTF file correctly sanitized for Firefox and Chrome """
stdout = prun('{0} {1}'.format(app.config['OTS_BINARY_PATH'],
self.path),
app.config['ROOT'])
self.assertEqual('', stdout.replace('\n', '. '))
def project_git_sync(project):
""" Sync git repo, or download it if it doesn't yet exist.
Args:
project: A :class:`~bakery.models.Project` instance
log: A :class:`~bakery.utils.RedisFd` instance
"""
from bakery.app import db, app
project.is_ready = False
db.session.add(project)
db.session.commit()
db.session.refresh(project)
client = redis.StrictRedis()
_in = joinroot('%(login)s/%(id)s.in/' % project)
_out = joinroot('%(login)s/%(id)s.out/' % project)
if not op.exists(_out):
os.makedirs(_out)
try:
os.remove(op.join(_out, 'fontaine.yml'))
except OSError:
pass
try:
os.remove(op.join(_out, 'upstream.log'))
except OSError:
pass
log = RedisFd(op.join(_out, 'upstream.log'))
# Create the incoming repo directory (_in) if it doesn't exist
if not op.exists(_in):
os.makedirs(op.join(app.config['DATA_ROOT'], _in), log=log)
# Update _in if it already exists with a .git directory
from git import Repo, InvalidGitRepositoryError
try:
repo = Repo(_in)
log.write('$ git reset --hard\n')
log.write(repo.git.reset(hard=True) + '\n')
log.write('$ git clean --force\n')
repo.git.clean(force=True)
log.write('$ git pull origin master\n')
repo.remotes.origin.pull()
except InvalidGitRepositoryError:
# clone the repository
# log.write('Copying Git Repository\n', prefix='### ')
try:
# TODO in the future, to validate the URL string use
# http://schacon.github.io/git/git-ls-remote.html
# http://stackoverflow.com/questions/9610131/how-to-check-the-validity-of-a-remote-git-repository-url
prun(('git clone --progress --depth=100'
' --branch=master %(clone)s .') % project, cwd=_in, log=log)
except:
# if the clone action didn't work, just copy it
# if this is a file URL, copy the files, and set up
# the _in directory as a git repo
if project.clone[:7] == "file://":
# cp recursively, keeping all attributes, not following
# symlinks, not deleting existing files, verbosely
prun('cp -a %(clone)s .' % project, cwd=_in, log=log)
#
prun('git init .', cwd=_in, log=log)
prun('git add *', cwd=_in, log=log)
msg = "Initial commit made automatically by Font Bakery"
prun('git commit -a -m "%s"' % msg, cwd=_in, log=log)
else:
raise
# Now we have it, create an initial project state
finally:
config = project.config
generate_subsets_coverage_list(project, log=log)
revision = prun("git rev-parse --short HEAD", cwd=_in).strip()
upstream_revision_tests(project, revision, log=log)
log.write('End: Repository is ready. Please Setup\n', prefix='### ')
# set project state as ready after sync is done
project.is_ready = True
db.session.add(project)
db.session.commit()
import json
client.publish('global:%s' % project.login,
json.dumps({'type': 'UPSTREAMFINISHED',
'project_id': project.id}))
