def upstream_revision_tests(project, revision, log=None):
""" This function run upstream tests on all sources fonts in project.
This mean that success (aka getting any result) should be occasional
particular case. Because data and set of folders are changing during
font development process.
Args:
project: A :class:`~bakery.models.Project` instance
revision: Git revision
Returns:
A dict with serialized tests results formatted by `repr_testcase`.
"""
param = {'login': project.login, 'id': project.id, 'revision': revision}
_in = joinroot('%(login)s/%(id)s.in/' % project)
_out_folder = joinroot('%(login)s/%(id)s.out/utests/' % param)
_out_yaml = op.join(_out_folder, '%(revision)s.yaml' % param)
# if op.exists(_out_yaml):
# return yaml.safe_load(open(_out_yaml, 'r'))
if not op.exists(_out_folder):
os.makedirs(_out_folder)
result = {}
os.chdir(_in)
try:
git_checkout(_in, revision)
result[project.clone] = run_set(_in, 'upstream-repo', log=log)
ufo_dirs, ttx_files, metadata_files = get_sources_lists(_in)
for font in ufo_dirs:
if op.exists(op.join(_in, font)):
result[font] = run_set(op.join(_in, font), 'upstream', log=log)
for metadata_path in metadata_files:
result[metadata_path] = run_set(metadata_path, 'metadata', log=log)
for font in ttx_files:
if op.exists(op.join(_in, font)):
result[font] = run_set(op.join(_in, font), 'upstream-ttx', log=log)
l = open(_out_yaml, mode='w')
l.write(yaml.safe_dump(result))
l.close()
return yaml.safe_load(open(_out_yaml, 'r'))
except Exception, ex:
if log:
log.write('UPSTREAM: FAILED')
log.write('Details: %s' % ex)
return {}
def generate_subsets_coverage_list(project, log=None):
""" Generates available subsets from project sources.
Method writes result to yaml file to avoid calling pyfontaine
api each time.
Args:
project: A :class:`~bakery.project.models.Project` instance
log: A :class:`~bakery.utils.RedisFd` instance
Returns:
Sorted subsets from prepared yaml file in tuple
[(common_name, coverage),]
"""
from .app import app
if log:
log.write('PyFontaine subsets with coverage values\n')
_in = joinroot('%(login)s/%(id)s.in/' % project)
ufo_dirs, ttx_files, _ = get_sources_lists(_in)
_out_yaml = op.join(app.config['DATA_ROOT'],
'%(login)s/%(id)s.out/fontaine.yml' % project)
if op.exists(_out_yaml):
return sorted(yaml.safe_load(open(_out_yaml, 'r')).items())
if not op.exists(op.dirname(_out_yaml)):
os.makedirs(op.dirname(_out_yaml), log=log)
source_fonts_paths = []
# `get_sources_list` returns list of paths relative to root.
# To complete to absolute paths use python os.path.join method
# on root and path
for path in ufo_dirs + ttx_files:
source_fonts_paths.append(op.join(_in, path))
subsets = get_subsets_coverage_data(source_fonts_paths, log)
contents = yaml.safe_dump(subsets)
yamlf = codecs.open(_out_yaml, mode='w', encoding="utf-8")
yamlf.write(contents)
yamlf.close()
return sorted(yaml.safe_load(open(_out_yaml, 'r')).items())
def project_git_sync(project):
""" Sync git repo, or download it if it doesn't yet exist.
Args:
project: A :class:`~bakery.models.Project` instance
log: A :class:`~bakery.utils.RedisFd` instance
"""
from bakery.app import db, app
project.is_ready = False
db.session.add(project)
db.session.commit()
db.session.refresh(project)
client = redis.StrictRedis()
_in = joinroot('%(login)s/%(id)s.in/' % project)
_out = joinroot('%(login)s/%(id)s.out/' % project)
if not op.exists(_out):
os.makedirs(_out)
try:
os.remove(op.join(_out, 'fontaine.yml'))
except OSError:
pass
try:
os.remove(op.join(_out, 'upstream.log'))
except OSError:
pass
log = RedisFd(op.join(_out, 'upstream.log'))
# Create the incoming repo directory (_in) if it doesn't exist
if not op.exists(_in):
os.makedirs(op.join(app.config['DATA_ROOT'], _in), log=log)
# Update _in if it already exists with a .git directory
from git import Repo, InvalidGitRepositoryError
try:
repo = Repo(_in)
log.write('$ git reset --hard\n')
log.write(repo.git.reset(hard=True) + '\n')
log.write('$ git clean --force\n')
repo.git.clean(force=True)
log.write('$ git pull origin master\n')
repo.remotes.origin.pull()
except InvalidGitRepositoryError:
# clone the repository
# log.write('Copying Git Repository\n', prefix='### ')
try:
# TODO in the future, to validate the URL string use
# http://schacon.github.io/git/git-ls-remote.html
# http://stackoverflow.com/questions/9610131/how-to-check-the-validity-of-a-remote-git-repository-url
prun(('git clone --progress --depth=100'
' --branch=master %(clone)s .') % project, cwd=_in, log=log)
except:
# if the clone action didn't work, just copy it
# if this is a file URL, copy the files, and set up
# the _in directory as a git repo
if project.clone[:7] == "file://":
# cp recursively, keeping all attributes, not following
# symlinks, not deleting existing files, verbosely
prun('cp -a %(clone)s .' % project, cwd=_in, log=log)
#
prun('git init .', cwd=_in, log=log)
prun('git add *', cwd=_in, log=log)
msg = "Initial commit made automatically by Font Bakery"
prun('git commit -a -m "%s"' % msg, cwd=_in, log=log)
else:
raise
# Now we have it, create an initial project state
finally:
config = project.config
generate_subsets_coverage_list(project, log=log)
revision = prun("git rev-parse --short HEAD", cwd=_in).strip()
upstream_revision_tests(project, revision, log=log)
log.write('End: Repository is ready. Please Setup\n', prefix='### ')
# set project state as ready after sync is done
project.is_ready = True
db.session.add(project)
db.session.commit()
import json
client.publish('global:%s' % project.login,
json.dumps({'type': 'UPSTREAMFINISHED',
'project_id': project.id}))