def diff_files_slow(self, left, right): DATA_ROOT = current_app.config.get('DATA_ROOT') _in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self) names = prun("""git diff --name-only %(left)s %(right)s""" % locals(), cwd=_in) data = prun("""git diff %(left)s %(right)s""" % locals(), cwd=_in) file_diff = {} # list of known xml file extensions xml_ext = ['.svg', '.ttx', '.plist', '.glif', '.xml'] for f in names.splitlines(): param = {'left': left, 'right': right, 'name': f} #file_diff[f] = prun("""git diff %(left)s %(right)s -- "%(name)s" """ % param, cwd=_in) if os.path.splitext(f)[1] in xml_ext: left_content = prun("""git show %(left)s:"%(name)s" """ % param, cwd=_in) right_content = prun("""git show %(right)s:"%(name)s" """ % param, cwd=_in) if left_content.startswith('fatal:'): left_content = '' if right_content.startswith('fatal:'): right_content = '' # left try: left_xml = etree.tostring(etree.fromstring(left_content), pretty_print=True) except etree.XMLSyntaxError: # if returned data is not real xml left_xml = left_content # right try: right_xml = etree.tostring(etree.fromstring(right_content), pretty_print=True) except etree.XMLSyntaxError: # if returned data is not real xml right_xml = right_content file_diff[f] = "".join([x for x in difflib.unified_diff(left_xml, right_xml, fromfile="a/" + f, tofile="b/" + f, lineterm='')]) else: file_diff[f] = prun("""git diff %(left)s %(right)s -- "%(name)s" """ % param, cwd=_in) for f in file_diff.keys(): text = file_diff[f] text = text.replace("&", "&").replace(">", ">").replace("<", "<") t = [] for l in text.splitlines(): if l[0] == '+': l = "<ins>%s</ins>" % l elif l[0] == "-": l = "<del>%s</del>" % l elif l[0] == "^": l = "<ins>%s</ins>" % l t.append(l.rstrip(" \t\n\r")) file_diff[f] = "\n".join(t) return file_diff
def revision_file(self, revision, fn): """ Read specific file from revision """ # XXX: [xen] need review here, not sure that it is 100% safe DATA_ROOT = current_app.config.get('DATA_ROOT') _in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self) fn = fn.replace('"', '') # XXX: result can be tree data = prun('git show "%(revision)s:%(fn)s"' % locals(), cwd=_in) mime = magic.from_buffer(data, mime=True) if mime.startswith("text"): data = unicode(data, "utf8") return mime, data
def execute(self, pipedata, prefix=""): ttxfiles = [] ufo = [] sfd = [] bin = [] for p in pipedata['process_files']: if p.endswith('.ttx'): ttxfiles.append(p) elif p.endswith('.sfd'): sfd.append(p) elif p.endswith('.ufo'): ufo.append(p) elif p.endswith('.ttf'): bin.append(p) elif p.endswith('.otf'): bin.append(p) self.stdout_pipe.write('Convert sources to TTF\n', prefix="### %s " % prefix) if ttxfiles: self.execute_ttx(self.project_root, self.builddir, ttxfiles) if ufo: self.execute_ufo_sfd(self.project_root, self.builddir, ufo) if sfd: self.execute_ufo_sfd(self.project_root, self.builddir, sfd) if bin: self.execute_bin(self.project_root, self.builddir, bin) binfiles = self.movebin_to_builddir(self.builddir, ufo + ttxfiles + sfd + bin) SCRIPTPATH = op.join('scripts', 'fix-ttf-vmet.py') command = ' '.join(map(lambda x: op.join(self.builddir, x), binfiles)) prun('python %s %s' % (SCRIPTPATH, command), cwd=op.abspath(op.join(op.dirname(__file__), '..', '..')), log=self.stdout_pipe) pipedata['bin_files'] = binfiles return pipedata
def revision_tree(self, revision): """ Get specific revision files as tree in format supported by tree macros """ DATA_ROOT = current_app.config.get('DATA_ROOT') _in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self) d = {} for file in prun("git ls-tree --name-only -r %(revision)s" % locals(), cwd=_in).splitlines(): level = d for part in file.split("/"): if part not in level: level[part] = {} level = level[part] return d
def diff_files(self, left, right): DATA_ROOT = current_app.config.get('DATA_ROOT') _in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self) data = prun("""git diff %(left)s %(right)s""" % locals(), cwd=_in) data = data.decode('utf-8').encode('ascii', 'xmlcharrefreplace') file_diff = {} t = [] current_file = None source_start = target_start = 0 RE_HUNK_HEADER = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)") def unpack(source_start, source_len, target_start, target_len, section_header): return int(source_start), int(target_start) data_lines = data.splitlines() data_lines.append(' ') for l in data_lines: if l.startswith('diff --git'): if current_file: file_diff[current_file] = "\n".join(t) current_file = l[14 + (len(l[11:]) - 1) / 2:] t = [] continue elif l.startswith('index'): continue elif l.startswith('---'): continue elif l.startswith('+++'): continue elif l.startswith('new file'): continue l = l.replace("&", "&").replace(">", ">").replace("<", "<") first = l[0] last = l[1:] l = l.rstrip(" \t\n\r") if l.startswith('Binary files'): l = '<tr><td></td><td></td><td><span class="text-center">Binary file not shown</span></td></tr>' elif first == '@': re_hunk_header = RE_HUNK_HEADER.match(l) if re_hunk_header: hunk_info = re_hunk_header.groups() source_start, target_start = unpack(*hunk_info) l = "<tr class='hunk'><td></td><td></td><td><pre>%s<pre></td></tr>" % (l) elif first == "+": target_start = target_start + 1 l = "<tr class='ins'><td></td><td class='num'>%s</td><td><pre>%s</pre></td></tr>" % (target_start, last) elif first == "-": source_start = source_start + 1 l = "<tr class='del'><td class='num'>%s</td><td></td><td class='del'><pre>%s</pre></td></tr>" % (source_start, last) else: target_start = target_start + 1 source_start = source_start + 1 l = "<tr><td class='num'>%s</td><td class='num'>%s</td><td><pre>%s<pre></td></tr>" % (source_start, target_start, l) t.append(l) if current_file: file_diff[current_file] = "\n".join(t) return file_diff
def current_revision(self): DATA_ROOT = current_app.config.get('DATA_ROOT') _in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self) return prun("git rev-parse --short HEAD", cwd=_in).strip()
def revision_info(self, revision): """ Return revision info for selected git commit """ DATA_ROOT = current_app.config.get('DATA_ROOT') _in = os.path.join(DATA_ROOT, '%(login)s/%(id)s.in/' % self) return prun("git show --quiet --format=short %(revision)s" % locals(), cwd=_in).decode('utf-8')
def test_ots(self): """ Is TTF file correctly sanitized for Firefox and Chrome """ stdout = prun('{0} {1}'.format(app.config['OTS_BINARY_PATH'], self.path), app.config['ROOT']) self.assertEqual('', stdout.replace('\n', '. '))
def project_git_sync(project): """ Sync git repo, or download it if it doesn't yet exist. Args: project: A :class:`~bakery.models.Project` instance log: A :class:`~bakery.utils.RedisFd` instance """ from bakery.app import db, app project.is_ready = False db.session.add(project) db.session.commit() db.session.refresh(project) client = redis.StrictRedis() _in = joinroot('%(login)s/%(id)s.in/' % project) _out = joinroot('%(login)s/%(id)s.out/' % project) if not op.exists(_out): os.makedirs(_out) try: os.remove(op.join(_out, 'fontaine.yml')) except OSError: pass try: os.remove(op.join(_out, 'upstream.log')) except OSError: pass log = RedisFd(op.join(_out, 'upstream.log')) # Create the incoming repo directory (_in) if it doesn't exist if not op.exists(_in): os.makedirs(op.join(app.config['DATA_ROOT'], _in), log=log) # Update _in if it already exists with a .git directory from git import Repo, InvalidGitRepositoryError try: repo = Repo(_in) log.write('$ git reset --hard\n') log.write(repo.git.reset(hard=True) + '\n') log.write('$ git clean --force\n') repo.git.clean(force=True) log.write('$ git pull origin master\n') repo.remotes.origin.pull() except InvalidGitRepositoryError: # clone the repository # log.write('Copying Git Repository\n', prefix='### ') try: # TODO in the future, to validate the URL string use # http://schacon.github.io/git/git-ls-remote.html # http://stackoverflow.com/questions/9610131/how-to-check-the-validity-of-a-remote-git-repository-url prun(('git clone --progress --depth=100' ' --branch=master %(clone)s .') % project, cwd=_in, log=log) except: # if the clone action didn't work, just copy it # if this is a file URL, copy the files, and set up # the _in directory as a git repo if project.clone[:7] == "file://": # cp recursively, keeping all attributes, not following # symlinks, not deleting existing files, verbosely prun('cp -a %(clone)s .' % project, cwd=_in, log=log) # prun('git init .', cwd=_in, log=log) prun('git add *', cwd=_in, log=log) msg = "Initial commit made automatically by Font Bakery" prun('git commit -a -m "%s"' % msg, cwd=_in, log=log) else: raise # Now we have it, create an initial project state finally: config = project.config generate_subsets_coverage_list(project, log=log) revision = prun("git rev-parse --short HEAD", cwd=_in).strip() upstream_revision_tests(project, revision, log=log) log.write('End: Repository is ready. Please Setup\n', prefix='### ') # set project state as ready after sync is done project.is_ready = True db.session.add(project) db.session.commit() import json client.publish('global:%s' % project.login, json.dumps({'type': 'UPSTREAMFINISHED', 'project_id': project.id}))