def page(photon, content, sub=None):
from os import path
from photon.util.locations import search_location, change_location
from photon.util.system import get_timestamp, get_hostname
settings = photon.settings.get
pwd = path.dirname(__file__)
out = path.join(settings['web']['output'], sub if sub else '', 'index.html')
search_location(out, create_in=path.dirname(out))
if sub: prfx, sub = '../', '— %s' %(sub)
else:
prfx, sub = './', ''
change_location(path.join(pwd, 'static'), path.join(settings['web']['output'], 'static'))
template = photon.template_handler(
path.join(pwd, 'main.tpl'),
fields=dict(
hostname=get_hostname(),
prfx=prfx,
sub=sub,
content=content,
timestamp=get_timestamp()
)
)
template.write(out, append=False)
def page(photon, content, sub=None):
'''
Helps creating webpages by placing the **content** into the ``main.tpl``
and write the output to ``web/output/`` **sub/** ``index.html``.
'''
settings = photon.settings.get
pwd = path.dirname(__file__)
out = path.join(settings['web']['output'], sub if sub else '',
'index.html')
search_location(out, create_in=path.dirname(out))
if sub:
prfx, sub = '../', '— %s' % (sub)
else:
prfx, sub = './', ''
change_location(path.join(pwd, 'static'),
path.join(settings['web']['output'], 'static'))
template = photon.template_handler(path.join(pwd, 'main.tpl'),
fields=dict(hostname=get_hostname(),
prfx=prfx,
sub=sub,
content=content,
timestamp=get_timestamp()))
template.write(out, append=False)
def __init__(self, defaults, config='config.yaml', verbose=True):
super().__init__()
self.__verbose = verbose
self.__settings = {'locations': get_locations(), 'files': dict()}
loaders = [(
'!str_join',
yaml_str_join,
), (
'!loc_join',
yaml_loc_join,
)]
defaults, sdict = ('startup import', defaults) if isinstance(
defaults, dict) else (search_location(defaults), None)
if not self.load(
'defaults', defaults, sdict=sdict, loaders=loaders,
merge=True):
shell_notify('could not load defaults',
state=True,
more=dict(defaults=defaults, sdict=sdict))
if config:
config = search_location(config, create_in='conf_dir')
if self.__settings != self.load(
'config', config, loaders=loaders, merge=True,
writeback=True):
shell_notify('settings config written',
more=config,
verbose=verbose)
def __init__(self, defaults, config='config.yaml', verbose=True):
super().__init__()
self.__verbose = verbose
self.__settings = {
'locations': get_locations(),
'files': dict()
}
loaders = [
('!str_join', yaml_str_join,),
('!loc_join', yaml_loc_join,)
]
defaults, sdict = (
'startup import',
defaults
) if isinstance(defaults, dict) else (
search_location(defaults),
None
)
if not self.load(
'defaults',
defaults,
sdict=sdict,
loaders=loaders,
merge=True
):
shell_notify(
'could not load defaults',
state=True,
more=dict(defaults=defaults, sdict=sdict)
)
if config:
config = search_location(config, create_in='conf_dir')
if self.__settings != self.load(
'config',
config,
loaders=loaders,
merge=True,
writeback=True
):
shell_notify(
'settings config written',
more=config,
verbose=verbose
)
def __init__(self, m, local, remote_url=None, mbranch=None):
super().__init__()
self.m = check_m(m)
self.__local = search_location(local, create_in=local)
self.__remote_url = remote_url
if not mbranch:
mbranch = 'master'
self.__mbranch = mbranch
if self.m(
'checking for git repo',
cmdd=dict(cmd='git rev-parse --show-toplevel', cwd=self.local),
critical=False,
verbose=False
).get('out') != self.local:
if not self.remote_url:
self.m(
'a new git clone without remote url is not possible.',
state=True,
more=dict(local=self.local)
)
self.m(
'cloning into repo',
cmdd=dict(
cmd='git clone %s %s' % (self.remote_url, self.local)
)
)
self.m(
'git tool startup done',
more=dict(remote_url=self.remote_url, local=self.local),
verbose=False
)
def __init__(self, m, pid, sudo=True, cmdd_if_no_pid=None):
super().__init__()
self.m = check_m(m)
pidfile = read_file(search_location(str(pid)))
if pidfile and pidfile.strip().isdigit():
pid = int(pidfile)
if not isinstance(pid, int):
if cmdd_if_no_pid:
self.m('running post command',
cmdd=cmdd_if_no_pid,
critical=True)
self.m('could not determine pid%s' %
(' from file' if pidfile else '!'),
more=dict(pid=pid, pidfile=pidfile),
state=True)
self.__pid = pid
self.__sudo = 'sudo' if sudo else ''
self.m('signal tool startup done',
more=dict(pid=self.__pid,
pidfile=pidfile if pidfile else 'passed directly'),
verbose=False)
def stage(self, name, clean=False):
'''
Switch stage
:param name:
Filename of new meta file. |filelocate|
* File must not already exist, will be created in 'data_dir' \
from :func:`util.locations.get_locations`
* Can also be a full path to place it anywhere desired
:param clean: What to do with preexisting meta files?
* ``False``: Merge current meta with preexisting one
* ``True``: Replace preexisting meta with current one
'''
name = search_location(name, create_in='data_dir')
if not clean:
self.load('stage', name, merge=True)
self.__meta['header'].update({'stage': name})
self.log = shell_notify('%s stage' %
('new clean' if clean else 'loaded'),
more=dict(meta=name, clean=clean),
verbose=self.__verbose)
def stage(self, name, clean=False):
'''
Switch stage
:param name:
Filename of new meta file. |filelocate|
* File must not already exist, will be created in 'data_dir' \
from :func:`util.locations.get_locations`
* Can also be a full path to place it anywhere desired
:param clean: What to do with preexisting meta files?
* ``False``: Merge current meta with preexisting one
* ``True``: Replace preexisting meta with current one
'''
name = search_location(name, create_in='data_dir')
if not clean:
self.load('stage', name, merge=True)
self.__meta['header'].update({'stage': name})
self.log = shell_notify(
'%s stage' % ('new clean' if clean else 'loaded'),
more=dict(meta=name, clean=clean),
verbose=self.__verbose
)
def pinit(mname, clean=False, verbose=True):
'''
Creates a new Photon instance and stages into common stage directory
defined in :ref:`defaults`.
It is some subfolder of the httpd-dir, so you can see what's going on
while building.
:param mname: |mname|
:param clean: Starts a new meta file replacing the old one.
Used in :func:`prepare`
:param verbose: |verbose|
:returns: a new :py:class:`photon.Photon` instance with it's
:py:class:`settings.Settings` as tuple
'''
photon = _pinit(mname, verbose)
settings = photon.settings.get
if clean:
change_location(settings['prepare']['stage_dir'], False, move=True)
photon.meta.stage(
search_location(
'builder_meta.json',
create_in=settings['prepare']['stage_dir']
),
clean=clean
)
return photon, settings
def __init__(self, m, pid, sudo=True, cmdd_if_no_pid=None):
super().__init__()
self.m = check_m(m)
pidfile = read_file(search_location(str(pid)))
if pidfile and pidfile.strip().isdigit():
pid = int(pidfile)
if not isinstance(pid, int):
if cmdd_if_no_pid:
self.m(
'running post command',
cmdd=cmdd_if_no_pid,
critical=True
)
self.m(
'could not determine pid%s' % (
' from file' if pidfile else '!'
),
more=dict(pid=pid, pidfile=pidfile),
state=True
)
self.__pid = pid
self.__sudo = 'sudo' if sudo else ''
self.m(
'signal tool startup done',
more=dict(
pid=self.__pid,
pidfile=pidfile if pidfile else 'passed directly'
),
verbose=False
)
def __init__(self, m, local, remote_url=None, mbranch=None):
super().__init__()
self.m = check_m(m)
self.__local = search_location(local, create_in=local)
self.__remote_url = remote_url
if not mbranch:
mbranch = 'master'
self.__mbranch = mbranch
if self.m('checking for git repo',
cmdd=dict(cmd='git rev-parse --show-toplevel',
cwd=self.local),
critical=False,
verbose=False).get('out') != self.local:
if not self.remote_url:
self.m('a new git clone without remote url is not possible.',
state=True,
more=dict(local=self.local))
self.m('cloning into repo',
cmdd=dict(cmd='git clone %s %s' %
(self.remote_url, self.local)))
self.m('git tool startup done',
more=dict(remote_url=self.remote_url, local=self.local),
verbose=False)
def draw_traffic():
from os import path
from photon.util.locations import search_location
from common import pinit
from common.html import page
photon, settings = pinit('draw_traffic', verbose=True)
traffic = '<small>click to show or hide</small><br />'
avail_if = photon.m(
'checking for available interfaces',
cmdd=dict(
cmd='sudo vnstat --iflist'
)
).get('out', '')
interfaces = settings['web']['traffic']['interfaces'] + [settings['fastd'][community]['interface'] for community in settings['fastd'].keys()]
for interface in interfaces:
if interface in avail_if:
if not search_location(path.join(settings['web']['traffic']['dbdir'], interface)):
photon.m(
'creating vnstat db for %s' %(interface),
cmdd=dict(
cmd='sudo vnstat -u -i %s' %(interface)
),
verbose=True
)
images = ''
for flag, itype in settings['web']['traffic']['types']:
image = '%s-%s.png' %(interface, itype)
photon.m(
'drawing %s graph for %s' %(itype, interface),
cmdd=dict(
cmd='vnstati -i %s -%s -o %s' %(interface, flag, path.join(settings['web']['output'], 'traffic', image))
),
critical=False
)
images += photon.template_handler(
IMAGE,
fields=dict(
interface=interface,
itype=itype,
image=image
)
).sub
traffic += photon.template_handler(
IFBLOCK,
fields=dict(
interface=interface,
images=images
)
).sub
page(photon, traffic, sub='traffic')
def gen_documentation():
'''
Pulls updates from our repositories containing (Sphinx) documentation,
builds them, and on success copies over the generated files.
You may or may not see the results on
`rtfm.freifunk-mwu.de <http://rtfm.freifunk-mwu.de>`_,
the mirrored repos are currently:
:backend_scripts: https://github.com/freifunk-mwu/backend-scripts
:gluon_builder: https://github.com/freifunk-mwu/gluon-builder-ffmwu
:gluon_gateway: https://github.com/freifunk-mwu/technik-meta
:photon: https://github.com/spookey/photon
'''
photon, settings = pinit('gen_documentation', verbose=True)
result = {
'build': {},
'result': {},
'update': {}
}
lroot = search_location(
settings['documentation']['local'],
create_in=settings['documentation']['local']
)
for name, data in settings['documentation']['repositories'].items():
local = path.join(lroot, name)
builddir = path.join(settings['documentation']['builddir'], name)
outdir = path.join(settings['documentation']['output'], name)
result['update'][name] = photon.git_handler(
local,
remote_url=data['remote']
)._pull()
build = photon.m(
'building documentation for %s' % (name),
cmdd=dict(
cmd='sphinx-build -a -b html %s %s' % (
path.join(local, data['docpath']),
builddir
)
),
critical=False
)
result['build'][name] = build
if build.get('returncode') == 0:
change_location(outdir, False, move=True)
change_location(builddir, outdir, move=True)
result['result'] = photon.m('build for %s successful')
photon.m('all done', more=result)
def __init__(self, m, template, fields=None):
super().__init__()
self.m = check_m(m)
tfile = search_location(template)
self.__template = read_file(tfile) if tfile else template
self.__fields = fields
self.m('template tool startup done',
more=dict(fields=self.__fields, file=tfile),
verbose=False)
def draw_traffic():
'''
Draws `vnstat <http://humdi.net/vnstat/>`_ Graphs, and glues them
into a Webseite.
Run only on machines which are connected to the mesh (Gateways,
Service-Machines).
* If a specified interface does not exist on this machine, it's skipped.
* If no vnstat database was found for one interface, it will be created.
'''
photon, settings = pinit('draw_traffic', verbose=True)
traffic = '<small>click to show or hide</small><br />'
avail_if = photon.m('checking for available interfaces',
cmdd=dict(cmd='sudo vnstat --iflist')).get('out', '')
interfaces = settings['web']['traffic']['interfaces'] + [
settings['fastd'][com]['interface']
for com in settings['common']['communities']
]
for interface in interfaces:
if interface in avail_if:
if not search_location(
path.join(settings['web']['traffic']['dbdir'], interface)):
photon.m('creating vnstat db for %s' % (interface),
cmdd=dict(cmd='sudo vnstat -u -i %s' % (interface)),
verbose=True)
images = ''
for flag, itype in settings['web']['traffic']['types']:
image = '%s-%s.png' % (interface, itype)
photon.m('drawing %s graph for %s' % (itype, interface),
cmdd=dict(cmd='vnstati -i %s -%s -o %s' %
(interface, flag,
path.join(settings['web']['output'],
'traffic', image))),
critical=False)
images += photon.template_handler(IMAGE,
fields=dict(
interface=interface,
itype=itype,
image=image)).sub
traffic += photon.template_handler(IFBLOCK,
fields=dict(interface=interface,
images=images)).sub
page(photon, traffic, sub='traffic')
def __init__(self, m, template, fields=None):
super().__init__()
self.m = check_m(m)
tfile = search_location(template)
self.__template = read_file(tfile) if tfile else template
self.__fields = fields
self.m(
'template tool startup done',
more=dict(fields=self.__fields, file=tfile),
verbose=False
)
def page(photon, content, sub=None):
'''
Helps creating webpages by placing the **content** into the ``main.tpl``
and write the output to ``web/output/`` **sub/** ``index.html``.
'''
settings = photon.settings.get
pwd = path.dirname(__file__)
out = path.join(
settings['web']['output'],
sub if sub else '',
'index.html'
)
search_location(out, create_in=path.dirname(out))
if sub:
prfx, sub = '../', '— %s' % (sub)
else:
prfx, sub = './', ''
change_location(
path.join(pwd, 'static'),
path.join(settings['web']['output'], 'static')
)
template = photon.template_handler(
path.join(pwd, 'main.tpl'),
fields=dict(
hostname=get_hostname(),
prfx=prfx,
sub=sub,
content=content,
timestamp=get_timestamp()
)
)
template.write(out, append=False)
def getpub_ssh():
if not search_location(settings['crypt']['ssh']['prv']):
mkprv_ssh()
pub = photon.m(
'generating new public ssh key from private',
cmdd=dict(
cmd='ssh-keygen -f %s -y' % (settings['crypt']['ssh']['prv']),
cwd=settings['crypt']['ssh']['folder']
)
)
if not pub.get('returncode') == 0:
photon.m('Error creating public ssh key', state=False)
if read_file(settings['crypt']['ssh']['pub']) != pub.get('out'):
write_file(settings['crypt']['ssh']['pub'], pub.get('out'))
photon.m('wrote public ssh key', more=pub)
return pub.get('out')
def gen_documentation():
'''
Pulls updates from our repositories containing (Sphinx) documentation,
builds them, and on success copies over the generated files.
You may or may not see the results on
`rtfm.freifunk-mwu.de <http://rtfm.freifunk-mwu.de>`_,
the mirrored repos are currently:
:backend_scripts: https://github.com/freifunk-mwu/backend-scripts
:gluon_builder: https://github.com/freifunk-mwu/gluon-builder-ffmwu
:gluon_gateway: https://github.com/freifunk-mwu/technik-meta
:photon: https://github.com/spookey/photon
'''
photon, settings = pinit('gen_documentation', verbose=True)
result = {'build': {}, 'result': {}, 'update': {}}
lroot = search_location(settings['documentation']['local'],
create_in=settings['documentation']['local'])
for name, data in settings['documentation']['repositories'].items():
local = path.join(lroot, name)
builddir = path.join(settings['documentation']['builddir'], name)
outdir = path.join(settings['documentation']['output'], name)
result['update'][name] = photon.git_handler(
local, remote_url=data['remote'])._pull()
build = photon.m(
'building documentation for %s' % (name),
cmdd=dict(cmd='sphinx-build -a -b html %s %s' %
(path.join(local, data['docpath']), builddir)),
critical=False)
result['build'][name] = build
if build.get('returncode') == 0:
change_location(outdir, False, move=True)
change_location(builddir, outdir, move=True)
result['result'] = photon.m('build for %s successful')
photon.m('all done', more=result)
def gen_expansion_map():
'''
This script generates several expansion-maps.
It pulls updates first, generates a map, patches assets, and then
copies over the generated files.
A common ``app.chache`` file is used for all builds, for speedup reasons.
When finished, it leaves the clean repo behind.
'''
photon, settings = pinit('gen_expansion_map')
# fetch updates from remote repository
git = photon.git_handler(settings['expansion']['local'],
remote_url=settings['expansion']['remote'])
git._pull()
for name, sub in settings['expansion']['maps'].items():
# generate map
build = photon.m(
'generate %s expansion map' % (name),
cmdd=dict(
cmd='./mkpoly -f nodelist %s' % (sub['url']),
cwd=settings['expansion']['local'],
timeout=600, # for initial run, only (to create app.cache)
))
if build.get('returncode') == 0:
for patch in settings['expansion']['patch']:
content = read_file(patch)
if not content:
# skip empty files
continue
# set title
content = re_sub(r'<title>(.*?)</title>',
'<title>%s</title>' % (sub['title']), content)
# set description
content = re_sub(
r'this._div.innerHTML\ =\ \'<h4>(.*?)</h4>\'',
'this._div.innerHTML = \'<h4>%s</h4>\'' % (sub['descr']),
content)
# set initial position
content = re_sub(
r'L\.map\(\'map\'\)\.setView\((.*?),\ 10\);',
'L.map(\'map\').setView([%s, %s], 10);' %
(sub['ipos'][0], sub['ipos'][1]), content)
# save result
photon.m('written %d bytes into %s' %
(write_file(patch, content), patch))
# copy generated files & folders
search_location(sub['output'], create_in=sub['output'])
for folder in ['js', 'css']:
change_location(path.join(settings['expansion']['local'],
folder),
path.join(sub['output'], folder),
move=False)
for fdoc in ['nodes.geojson', 'index.html', 'LICENSE']:
change_location(path.join(settings['expansion']['local'],
fdoc),
sub['output'],
move=False)
# reset changed files
for reset in settings['expansion']['patch'] + [
path.join(settings['expansion']['local'], 'nodes.geojson')
]:
photon.m('clean up %s' % (reset))
git._checkout('-- %s' % (reset))
def gen_expansion_map():
'''
This script generates several expansion-maps.
It pulls updates first, generates a map, patches assets, and then
copies over the generated files.
A common ``app.chache`` file is used for all builds, for speedup reasons.
When finished, it leaves the clean repo behind.
'''
photon, settings = pinit('gen_expansion_map')
# fetch updates from remote repository
git = photon.git_handler(
settings['expansion']['local'],
remote_url=settings['expansion']['remote']
)
git._pull()
for name, sub in settings['expansion']['maps'].items():
# generate map
build = photon.m(
'generate %s expansion map' % (name),
cmdd=dict(
cmd='./mkpoly -f nodelist %s' % (sub['url']),
cwd=settings['expansion']['local'],
timeout=600, # for initial run, only (to create app.cache)
)
)
if build.get('returncode') == 0:
for patch in settings['expansion']['patch']:
content = read_file(patch)
if not content:
# skip empty files
continue
# set title
content = re_sub(
r'<title>(.*?)</title>',
'<title>%s</title>' % (sub['title']),
content
)
# set description
content = re_sub(
r'this._div.innerHTML\ =\ \'<h4>(.*?)</h4>\'',
'this._div.innerHTML = \'<h4>%s</h4>\'' % (sub['descr']),
content
)
# set initial position
content = re_sub(
r'L\.map\(\'map\'\)\.setView\((.*?),\ 10\);',
'L.map(\'map\').setView([%s, %s], 10);' % (
sub['ipos'][0], sub['ipos'][1]
),
content
)
# save result
photon.m('written %d bytes into %s' % (
write_file(patch, content),
patch
))
# copy generated files & folders
search_location(sub['output'], create_in=sub['output'])
for folder in ['js', 'css']:
change_location(
path.join(settings['expansion']['local'], folder),
path.join(sub['output'], folder),
move=False
)
for fdoc in ['nodes.geojson', 'index.html', 'LICENSE']:
change_location(
path.join(settings['expansion']['local'], fdoc),
sub['output'],
move=False
)
# reset changed files
for reset in settings['expansion']['patch'] + [
path.join(settings['expansion']['local'], 'nodes.geojson')
]:
photon.m('clean up %s' % (reset))
git._checkout('-- %s' % (reset))