def page(photon, content, sub=None):
from os import path
from photon.util.locations import search_location, change_location
from photon.util.system import get_timestamp, get_hostname
settings = photon.settings.get
pwd = path.dirname(__file__)
out = path.join(settings['web']['output'], sub if sub else '', 'index.html')
search_location(out, create_in=path.dirname(out))
if sub: prfx, sub = '../', '— %s' %(sub)
else:
prfx, sub = './', ''
change_location(path.join(pwd, 'static'), path.join(settings['web']['output'], 'static'))
template = photon.template_handler(
path.join(pwd, 'main.tpl'),
fields=dict(
hostname=get_hostname(),
prfx=prfx,
sub=sub,
content=content,
timestamp=get_timestamp()
)
)
template.write(out, append=False)
def page(photon, content, sub=None):
'''
Helps creating webpages by placing the **content** into the ``main.tpl``
and write the output to ``web/output/`` **sub/** ``index.html``.
'''
settings = photon.settings.get
pwd = path.dirname(__file__)
out = path.join(settings['web']['output'], sub if sub else '',
'index.html')
search_location(out, create_in=path.dirname(out))
if sub:
prfx, sub = '../', '— %s' % (sub)
else:
prfx, sub = './', ''
change_location(path.join(pwd, 'static'),
path.join(settings['web']['output'], 'static'))
template = photon.template_handler(path.join(pwd, 'main.tpl'),
fields=dict(hostname=get_hostname(),
prfx=prfx,
sub=sub,
content=content,
timestamp=get_timestamp()))
template.write(out, append=False)
def pinit(mname, clean=False, verbose=True):
'''
Creates a new Photon instance and stages into common stage directory
defined in :ref:`defaults`.
It is some subfolder of the httpd-dir, so you can see what's going on
while building.
:param mname: |mname|
:param clean: Starts a new meta file replacing the old one.
Used in :func:`prepare`
:param verbose: |verbose|
:returns: a new :py:class:`photon.Photon` instance with it's
:py:class:`settings.Settings` as tuple
'''
photon = _pinit(mname, verbose)
settings = photon.settings.get
if clean:
change_location(settings['prepare']['stage_dir'], False, move=True)
photon.meta.stage(
search_location(
'builder_meta.json',
create_in=settings['prepare']['stage_dir']
),
clean=clean
)
return photon, settings
def uni_manifest(branch, manifest):
'''
After building the images, a manifest file gets created.
:param branch: The branch currently building
:param manifest: The path to the manifest file
Before the manifest gets signed, this function is called to enable
cross releasing of branches by clever symlinking.
.. seealso:: :func:`common.uni_args` for command line syntax
'''
photon, settings = pinit('uni_manifest', verbose=True)
manifest = path.abspath(manifest)
manifest_content = read_file(manifest)
if manifest_content and manifest_content.count('BRANCH=') == 1:
uni_branch = '\n'.join(
'BRANCH=%s' % (avail) for avail in sorted(
settings['common']['branches']['avail'].keys()
)
)
new_manifest = photon.template_handler(
manifest_content.replace('BRANCH=%s' % (branch), '${uni_branch}'),
fields=dict(uni_branch=uni_branch)
)
change_location(manifest, False, move=True)
new_manifest.write(
manifest.replace('%s.manifest' % (branch), 'manifest'),
append=False
)
for avail in settings['common']['branches']['avail'].keys():
manifest_link = manifest.replace(
'%s.manifest' % (branch), '%s.manifest' % (avail)
)
change_location(manifest_link, False, move=True)
photon.m(
'linking manifests %s' % (manifest_link),
cmdd=dict(
cmd='ln -s manifest %s' % (path.basename(manifest_link)),
cwd=path.dirname(manifest_link)
)
)
photon.m(
'uni_manifest written',
more=dict(branch=branch),
verbose=True
)
else:
photon.m(
'Refusing to write uni_manifest',
more=dict(branch=branch, manifest=manifest),
state=False
)
def gen_documentation():
'''
Pulls updates from our repositories containing (Sphinx) documentation,
builds them, and on success copies over the generated files.
You may or may not see the results on
`rtfm.freifunk-mwu.de <http://rtfm.freifunk-mwu.de>`_,
the mirrored repos are currently:
:backend_scripts: https://github.com/freifunk-mwu/backend-scripts
:gluon_builder: https://github.com/freifunk-mwu/gluon-builder-ffmwu
:gluon_gateway: https://github.com/freifunk-mwu/technik-meta
:photon: https://github.com/spookey/photon
'''
photon, settings = pinit('gen_documentation', verbose=True)
result = {
'build': {},
'result': {},
'update': {}
}
lroot = search_location(
settings['documentation']['local'],
create_in=settings['documentation']['local']
)
for name, data in settings['documentation']['repositories'].items():
local = path.join(lroot, name)
builddir = path.join(settings['documentation']['builddir'], name)
outdir = path.join(settings['documentation']['output'], name)
result['update'][name] = photon.git_handler(
local,
remote_url=data['remote']
)._pull()
build = photon.m(
'building documentation for %s' % (name),
cmdd=dict(
cmd='sphinx-build -a -b html %s %s' % (
path.join(local, data['docpath']),
builddir
)
),
critical=False
)
result['build'][name] = build
if build.get('returncode') == 0:
change_location(outdir, False, move=True)
change_location(builddir, outdir, move=True)
result['result'] = photon.m('build for %s successful')
photon.m('all done', more=result)
def gen_documentation():
'''
Pulls updates from our repositories containing (Sphinx) documentation,
builds them, and on success copies over the generated files.
You may or may not see the results on
`rtfm.freifunk-mwu.de <http://rtfm.freifunk-mwu.de>`_,
the mirrored repos are currently:
:backend_scripts: https://github.com/freifunk-mwu/backend-scripts
:gluon_builder: https://github.com/freifunk-mwu/gluon-builder-ffmwu
:gluon_gateway: https://github.com/freifunk-mwu/technik-meta
:photon: https://github.com/spookey/photon
'''
photon, settings = pinit('gen_documentation', verbose=True)
result = {'build': {}, 'result': {}, 'update': {}}
lroot = search_location(settings['documentation']['local'],
create_in=settings['documentation']['local'])
for name, data in settings['documentation']['repositories'].items():
local = path.join(lroot, name)
builddir = path.join(settings['documentation']['builddir'], name)
outdir = path.join(settings['documentation']['output'], name)
result['update'][name] = photon.git_handler(
local, remote_url=data['remote'])._pull()
build = photon.m(
'building documentation for %s' % (name),
cmdd=dict(cmd='sphinx-build -a -b html %s %s' %
(path.join(local, data['docpath']), builddir)),
critical=False)
result['build'][name] = build
if build.get('returncode') == 0:
change_location(outdir, False, move=True)
change_location(builddir, outdir, move=True)
result['result'] = photon.m('build for %s successful')
photon.m('all done', more=result)
def publish(folder, branch):
'''
Publishes freshly built images.
This is accomplished by setting symbolic-links in the form
of ``communitiy``/``branch`` pointing into the library (in the
form of _library/``build``-``branch``/``short-community``)
.. seealso:: :func:`common.uni_args` for command line syntax
'''
photon = _pinit('publish')
settings = photon.settings.get
folder = path.realpath(folder)
if not settings['publish']['library_dir'] == path.dirname(folder):
photon.m(
'wrong folder selected!',
more=dict(
folder=folder,
should_be_subfolder_of=settings['publish']['library_dir']
),
state=True
)
for short_name, long_name in settings['common']['communities'].items():
full_target = path.join(
settings['publish']['http_fw_dir'],
long_name,
branch
)
change_location(full_target, False, move=True)
target = path.dirname(full_target)
link = path.relpath(path.join(folder, short_name), target)
photon.m(
'linking release',
cmdd=dict(
cmd='ln -s %s %s' % (link, branch),
cwd=target
),
more=dict(full_target=full_target)
)
def snapshot_configs():
from os import path
from photon.util.locations import change_location
from photon.util.files import write_file
from common import pinit
photon, settings = pinit('snapshot_configs', verbose=True)
git = photon.git_handler(
settings['configs']['local'],
remote_url=settings['configs']['remote']
)
git.cleanup
if not photon.settings.load('queue', settings['configs']['queue']):
photon.m(
'could not load snapshot queue from git',
more=dict(
queue=settings['configs']['queue']
),
state=True
)
photon.s2m
change_location(settings['configs']['target'], False, move=True)
for loc in settings['queue'].get('locations') + settings['configs']['qadd']:
change_location(loc, path.join(settings['configs']['target'], loc.lstrip('/')))
for b_cmd, b_file in [
('crontab -l', 'crontab'), ('dpkg -l', 'package_list')
]:
result = photon.m(
'retrieving %s contents' %(b_cmd),
cmdd=dict(cmd=b_cmd),
critical=False
)
if result.get('returncode') == 0:
write_file(path.join(settings['configs']['target'], b_file), result.get('out'))
git.publish
def page(photon, content, sub=None):
'''
Helps creating webpages by placing the **content** into the ``main.tpl``
and write the output to ``web/output/`` **sub/** ``index.html``.
'''
settings = photon.settings.get
pwd = path.dirname(__file__)
out = path.join(
settings['web']['output'],
sub if sub else '',
'index.html'
)
search_location(out, create_in=path.dirname(out))
if sub:
prfx, sub = '../', '— %s' % (sub)
else:
prfx, sub = './', ''
change_location(
path.join(pwd, 'static'),
path.join(settings['web']['output'], 'static')
)
template = photon.template_handler(
path.join(pwd, 'main.tpl'),
fields=dict(
hostname=get_hostname(),
prfx=prfx,
sub=sub,
content=content,
timestamp=get_timestamp()
)
)
template.write(out, append=False)
def prepare(
branch,
gluon_tag=None, site_tag=None,
nomodules=False, onlyone=False, priority=None
):
'''
Checks out Gluon sources and site-conf repositories at proper commit-ids
or tags according to the branch to build.
Generates a site-conf afterwards.
Automatically invokes :func:`_gen_bconf` afterwards
.. seealso:: :func:`common.prepare_args` for command line syntax
'''
photon, settings = pinit('prepare', clean=True)
for community in [
onlyone
] if onlyone else settings['common']['communities'].keys():
change_location(
settings['gluon']['local'][community],
False,
move=True
)
tags = settings['common']['branches']['avail'][branch]
gluon, site = ginit(photon, community=community)
if gluon_tag:
if gluon.tag and gluon_tag in gluon.tag:
gluon.tag = gluon_tag
elif gluon.commit and (
gluon_tag in gluon.commit or gluon_tag in gluon.short_commit
):
gluon.commit = gluon_tag
else:
photon.m(
'Invalid git commit-id or tag specified for gluon',
state=True
)
else:
if tags[0]:
gluon.tag = None
else:
gluon.branch = None
if site_tag:
if site.tag and site_tag in site.tag:
site.tag = site_tag
elif site.commit and (
site_tag in site.commit or site_tag in site.short_commit
):
site.commit = site_tag
else:
photon.m(
'Invalid git commit-id or tag specified for site',
state=True
)
else:
if tags[1]:
site.tag = None
else:
site.branch = None
photon.m(
'generating site for %s' % (community),
cmdd=dict(
cmd='%s generate.py %s %s %s' % (
settings['common']['pycmd'],
community,
'--nomodules' if nomodules else '',
'--priority %s' % (priority) if priority else ''
),
cwd=settings['site']['local'][community]
),
verbose=True
)
def gen_expansion_map():
'''
This script generates several expansion-maps.
It pulls updates first, generates a map, patches assets, and then
copies over the generated files.
A common ``app.chache`` file is used for all builds, for speedup reasons.
When finished, it leaves the clean repo behind.
'''
photon, settings = pinit('gen_expansion_map')
# fetch updates from remote repository
git = photon.git_handler(
settings['expansion']['local'],
remote_url=settings['expansion']['remote']
)
git._pull()
for name, sub in settings['expansion']['maps'].items():
# generate map
build = photon.m(
'generate %s expansion map' % (name),
cmdd=dict(
cmd='./mkpoly -f nodelist %s' % (sub['url']),
cwd=settings['expansion']['local'],
timeout=600, # for initial run, only (to create app.cache)
)
)
if build.get('returncode') == 0:
for patch in settings['expansion']['patch']:
content = read_file(patch)
if not content:
# skip empty files
continue
# set title
content = re_sub(
r'<title>(.*?)</title>',
'<title>%s</title>' % (sub['title']),
content
)
# set description
content = re_sub(
r'this._div.innerHTML\ =\ \'<h4>(.*?)</h4>\'',
'this._div.innerHTML = \'<h4>%s</h4>\'' % (sub['descr']),
content
)
# set initial position
content = re_sub(
r'L\.map\(\'map\'\)\.setView\((.*?),\ 10\);',
'L.map(\'map\').setView([%s, %s], 10);' % (
sub['ipos'][0], sub['ipos'][1]
),
content
)
# save result
photon.m('written %d bytes into %s' % (
write_file(patch, content),
patch
))
# copy generated files & folders
search_location(sub['output'], create_in=sub['output'])
for folder in ['js', 'css']:
change_location(
path.join(settings['expansion']['local'], folder),
path.join(sub['output'], folder),
move=False
)
for fdoc in ['nodes.geojson', 'index.html', 'LICENSE']:
change_location(
path.join(settings['expansion']['local'], fdoc),
sub['output'],
move=False
)
# reset changed files
for reset in settings['expansion']['patch'] + [
path.join(settings['expansion']['local'], 'nodes.geojson')
]:
photon.m('clean up %s' % (reset))
git._checkout('-- %s' % (reset))
def gen_expansion_map():
'''
This script generates several expansion-maps.
It pulls updates first, generates a map, patches assets, and then
copies over the generated files.
A common ``app.chache`` file is used for all builds, for speedup reasons.
When finished, it leaves the clean repo behind.
'''
photon, settings = pinit('gen_expansion_map')
# fetch updates from remote repository
git = photon.git_handler(settings['expansion']['local'],
remote_url=settings['expansion']['remote'])
git._pull()
for name, sub in settings['expansion']['maps'].items():
# generate map
build = photon.m(
'generate %s expansion map' % (name),
cmdd=dict(
cmd='./mkpoly -f nodelist %s' % (sub['url']),
cwd=settings['expansion']['local'],
timeout=600, # for initial run, only (to create app.cache)
))
if build.get('returncode') == 0:
for patch in settings['expansion']['patch']:
content = read_file(patch)
if not content:
# skip empty files
continue
# set title
content = re_sub(r'<title>(.*?)</title>',
'<title>%s</title>' % (sub['title']), content)
# set description
content = re_sub(
r'this._div.innerHTML\ =\ \'<h4>(.*?)</h4>\'',
'this._div.innerHTML = \'<h4>%s</h4>\'' % (sub['descr']),
content)
# set initial position
content = re_sub(
r'L\.map\(\'map\'\)\.setView\((.*?),\ 10\);',
'L.map(\'map\').setView([%s, %s], 10);' %
(sub['ipos'][0], sub['ipos'][1]), content)
# save result
photon.m('written %d bytes into %s' %
(write_file(patch, content), patch))
# copy generated files & folders
search_location(sub['output'], create_in=sub['output'])
for folder in ['js', 'css']:
change_location(path.join(settings['expansion']['local'],
folder),
path.join(sub['output'], folder),
move=False)
for fdoc in ['nodes.geojson', 'index.html', 'LICENSE']:
change_location(path.join(settings['expansion']['local'],
fdoc),
sub['output'],
move=False)
# reset changed files
for reset in settings['expansion']['patch'] + [
path.join(settings['expansion']['local'], 'nodes.geojson')
]:
photon.m('clean up %s' % (reset))
git._checkout('-- %s' % (reset))
