def build_logger(msg):
'''
This function starts a new photon instance to append new messages
into the ``builder_meta.json`` file.
It is used in :ref:`builder` to mark certain steps taken for validation
of all actions done while compiling (remember: the resulting images
need to get signed for the autoupdater to work)
:param msg: The message to write into the meta file
.. seealso:: :func:`common.log_args` for command line syntax
'''
pinit('build_logger', verbose=False)[0].m(msg, verbose=True)
def check_radvd():
'''
Checks if ``radvd`` is running.
Will try to restart it if previous check failed.
'''
photon, settings = pinit('check_radvd', verbose=True)
status = photon.m(
'testing for radvd',
cmdd=dict(
cmd='sudo service radvd status',
),
critical=False
)
if status.get('returncode') != 0:
photon.m(
'(re) starting radvd',
cmdd=dict(
cmd='sudo service radvd start'
),
critical=False
)
else:
photon.m('radvd is running')
def gen_website():
'''
Generates a basic landing page, linking to firmware and/or traffic
statistics (if any).
Adds some basic system statistics by running shell commands, putting
the output into the status page.
'''
photon, settings = pinit('gen_website', verbose=True)
main = ''
if path.exists(path.join(settings['web']['output'], 'firmware')):
main += BLOCK.format(href='firmware', text='Firmware')
if path.exists(path.join(settings['web']['output'], '_archive')):
main += BLOCK.format(href='_archive', text='Firmware Archive')
if path.exists(path.join(settings['web']['output'], 'traffic')):
main += BLOCK.format(href='traffic', text='Traffic')
main += BLOCK.format(href='system', text='System Statistics')
page(photon, main)
sys = '<small>click to show or hide</small><br />'
for command in settings['web']['system']:
cmd_output = photon.m('retrieving system info',
cmdd=dict(cmd=command),
critical=False).get('out')
sys += SYSBLOCK.format(command=command,
command_tag=command.split()[0],
cmd_output=cmd_output)
page(photon, sys, sub='system')
def deploy_ssh():
'''
Adds keys into the ``~/.ssh/authorized_keys`` which are not already there.
The Keys are read from the ``ssh.yaml`` inside the
`gateway-configs <https://github.com/freifunk-mwu/gateway-configs>`_
repository.
'''
parser = ArgumentParser(prog='deploy_ssh')
photon, settings = pinit('deploy_ssh', verbose=True)
# initialize the gateway-configs repo ...
photon.git_handler(settings['configs']['local'],
remote_url=settings['configs']['remote'])._pull()
# .. to load contents from the ssh.yaml into the settings
if not photon.settings.load('ssh_deploy',
settings['configs']['ssh_deploy']):
photon.m('could not load ssh_deploy',
more=dict(ssh_deploy=settings['configs']['ssh_deploy']),
state=True)
photon.s2m
parser.add_argument('mtype',
action='store',
choices=settings['ssh_deploy'].keys())
args = parser.parse_args()
for key in settings['ssh_deploy'][args.mtype]:
conf_t = photon.template_handler(
'%s\n' % (settings['ssh_deploy'][args.mtype][key]))
conf_t.write(settings['crypt']['ssh']['authorized'])
def check_bind():
'''
Checks if ``bind9`` is still running.
Will try to restart it if previous check failed.
'''
photon, settings = pinit('check_bind', verbose=True)
status = photon.m(
'testing for bind9',
cmdd=dict(
cmd='sudo rndc status',
),
critical=False
)
if status.get('returncode') != 0:
photon.m(
'(re) starting bind9',
cmdd=dict(
cmd='sudo service bind9 start'
),
critical=False
)
else:
photon.m('bind9 is running')
def uni_manifest(branch, manifest):
'''
After building the images, a manifest file gets created.
:param branch: The branch currently building
:param manifest: The path to the manifest file
Before the manifest gets signed, this function is called to enable
cross releasing of branches by clever symlinking.
.. seealso:: :func:`common.uni_args` for command line syntax
'''
photon, settings = pinit('uni_manifest', verbose=True)
manifest = path.abspath(manifest)
manifest_content = read_file(manifest)
if manifest_content and manifest_content.count('BRANCH=') == 1:
uni_branch = '\n'.join(
'BRANCH=%s' % (avail) for avail in sorted(
settings['common']['branches']['avail'].keys()
)
)
new_manifest = photon.template_handler(
manifest_content.replace('BRANCH=%s' % (branch), '${uni_branch}'),
fields=dict(uni_branch=uni_branch)
)
change_location(manifest, False, move=True)
new_manifest.write(
manifest.replace('%s.manifest' % (branch), 'manifest'),
append=False
)
for avail in settings['common']['branches']['avail'].keys():
manifest_link = manifest.replace(
'%s.manifest' % (branch), '%s.manifest' % (avail)
)
change_location(manifest_link, False, move=True)
photon.m(
'linking manifests %s' % (manifest_link),
cmdd=dict(
cmd='ln -s manifest %s' % (path.basename(manifest_link)),
cwd=path.dirname(manifest_link)
)
)
photon.m(
'uni_manifest written',
more=dict(branch=branch),
verbose=True
)
else:
photon.m(
'Refusing to write uni_manifest',
more=dict(branch=branch, manifest=manifest),
state=False
)
def draw_traffic():
from os import path
from photon.util.locations import search_location
from common import pinit
from common.html import page
photon, settings = pinit('draw_traffic', verbose=True)
traffic = '<small>click to show or hide</small><br />'
avail_if = photon.m(
'checking for available interfaces',
cmdd=dict(
cmd='sudo vnstat --iflist'
)
).get('out', '')
interfaces = settings['web']['traffic']['interfaces'] + [settings['fastd'][community]['interface'] for community in settings['fastd'].keys()]
for interface in interfaces:
if interface in avail_if:
if not search_location(path.join(settings['web']['traffic']['dbdir'], interface)):
photon.m(
'creating vnstat db for %s' %(interface),
cmdd=dict(
cmd='sudo vnstat -u -i %s' %(interface)
),
verbose=True
)
images = ''
for flag, itype in settings['web']['traffic']['types']:
image = '%s-%s.png' %(interface, itype)
photon.m(
'drawing %s graph for %s' %(itype, interface),
cmdd=dict(
cmd='vnstati -i %s -%s -o %s' %(interface, flag, path.join(settings['web']['output'], 'traffic', image))
),
critical=False
)
images += photon.template_handler(
IMAGE,
fields=dict(
interface=interface,
itype=itype,
image=image
)
).sub
traffic += photon.template_handler(
IFBLOCK,
fields=dict(
interface=interface,
images=images
)
).sub
page(photon, traffic, sub='traffic')
def update_bird_conf():
from photon.util.files import read_file
from common import pinit
photon, settings = pinit('update_bird_conf', verbose=True)
for repo in ['scripts', 'meta']:
photon.git_handler(
settings['icvpn']['bird'][repo]['local'],
remote_url=settings['icvpn']['bird'][repo]['remote']
)._pull()
for ip_ver in settings['icvpn']['bird']['ip_ver']:
do_restart = False
bird_conf = photon.template_handler('${config_content}')
# peers
peers_config_content=photon.m(
'generating ip_ver%s bgp peers conf' %(ip_ver),
cmdd=dict(
cmd='./mkbgp -f bird -%s -s %s -x mainz -x wiesbaden -d ebgp_ic' %(ip_ver, settings['icvpn']['bird']['meta']['local']),
cwd=settings['icvpn']['bird']['scripts']['local']
)
).get('out')
bird_peers_conf.sub = dict(peers_config_content=peers_config_content)
peers_conf = settings['icvpn']['bird']['ip_ver'][ip_ver]['peers_conf']
if bird_peers_conf.sub != read_file(peers_conf):
bird_peers_conf.write(peers_conf, append=False)
do_restart = True
# roa
roa_config_content=photon.m(
'generating ip_ver%s bgp roa conf' %(ip_ver),
cmdd=dict(
cmd='./mkroa -f bird -%s -s %s -x mainz -x wiesbaden' %(ip_ver, settings['icvpn']['bird']['meta']['local']),
cwd=settings['icvpn']['bird']['scripts']['local']
)
).get('out')
bird_roa_conf.sub = dict(roa_config_content=roa_config_content)
roa_conf = settings['icvpn']['bird']['ip_ver'][ip_ver]['roa_conf']
if bird_roa_conf.sub != read_file(roa_conf):
bird_roa_conf.write(roa_conf, append=False)
do_restart = True
if do_restart:
photon.m(
'restarting bird daemon for v%s' %(ip_ver),
cmdd=dict(
cmd='sudo service %s restart' %(settings['icvpn']['bird']['ip_ver'][ip_ver]['exec'])
)
)
def gen_documentation():
'''
Pulls updates from our repositories containing (Sphinx) documentation,
builds them, and on success copies over the generated files.
You may or may not see the results on
`rtfm.freifunk-mwu.de <http://rtfm.freifunk-mwu.de>`_,
the mirrored repos are currently:
:backend_scripts: https://github.com/freifunk-mwu/backend-scripts
:gluon_builder: https://github.com/freifunk-mwu/gluon-builder-ffmwu
:gluon_gateway: https://github.com/freifunk-mwu/technik-meta
:photon: https://github.com/spookey/photon
'''
photon, settings = pinit('gen_documentation', verbose=True)
result = {
'build': {},
'result': {},
'update': {}
}
lroot = search_location(
settings['documentation']['local'],
create_in=settings['documentation']['local']
)
for name, data in settings['documentation']['repositories'].items():
local = path.join(lroot, name)
builddir = path.join(settings['documentation']['builddir'], name)
outdir = path.join(settings['documentation']['output'], name)
result['update'][name] = photon.git_handler(
local,
remote_url=data['remote']
)._pull()
build = photon.m(
'building documentation for %s' % (name),
cmdd=dict(
cmd='sphinx-build -a -b html %s %s' % (
path.join(local, data['docpath']),
builddir
)
),
critical=False
)
result['build'][name] = build
if build.get('returncode') == 0:
change_location(outdir, False, move=True)
change_location(builddir, outdir, move=True)
result['result'] = photon.m('build for %s successful')
photon.m('all done', more=result)
def check_exitvpn():
from common import pinit
photon, settings = pinit('check_exitvpn', verbose=True)
uping = photon.ping_handler(net_if=settings['exitping']['interface'])
aping = photon.ping_handler(net_if=settings['exitping']['interface'])
uping.probe = settings['exitping']['urls']
aping.probe = settings['exitping']['addresses']
photon.m(
'ping results',
more=dict(
ping_urls=uping.status,
ping_addresses=aping.status
)
)
for community in settings['common']['communities']:
cif = settings['batman'][community]['interface']
cbw = settings['batman'][community]['bandwidth']
if uping.status['ratio'] <= settings['exitping']['min_ratio'] or aping.status['ratio'] <= settings['exitping']['min_ratio']:
photon.m('exitvpn for %s - it seems you are not properly connected!' %(community))
photon.m(
'removing batman server flag for %s' %(cif),
cmdd=dict(
cmd='sudo batctl -m %s gw off' %(cif)
)
)
photon.m(
'stopping isc-dhcp-server',
cmdd=dict(
cmd='sudo initctl stop isc-dhcp-server'
)
)
else:
photon.m('exitvpn for %s - you are connected!!' %(community))
photon.m(
'setting batman server flag for %s (bw: %s)' %(cif, cbw),
cmdd=dict(
cmd='sudo batctl -m %s gw server %s' %(cif, cbw)
),
)
photon.m(
'starting isc-dhcp-server',
cmdd=dict(
cmd='sudo initctl start isc-dhcp-server'
),
critical=False
)
def check_exitvpn():
'''
Sets the ``BATMAN`` Server Flag and starts/stops ``isc-dhcp-server``,
if some given Hosts can be reached through the `ExitVPN`.
'''
photon, settings = pinit('check_exitvpn', verbose=True)
ping = photon.ping_handler(net_if=settings['exitping']['interface'])
photon.m('sending pings', more=settings['exitping']['targets'])
ping.probe = settings['exitping']['targets']
photon.m('ping results', more=ping.status)
for community in settings['common']['communities']:
interface = settings['batman'][community]['interface']
bandwidth = settings['batman'][community]['bandwidth']
if ping.status['ratio'] <= settings['exitping']['min_ratio']:
photon.m('exitvpn for %s - you are _not_ connected!' % (community))
photon.m(
'removing batman server flag for %s' % (interface),
cmdd=dict(cmd='sudo batctl -m %s gw off' % (interface))
)
if 'start/running' in photon.m(
'check if isc-dhcp-server is running to stop it',
cmdd=dict(cmd='initctl status isc-dhcp-server')
).get('out', ''):
photon.m(
'stopping isc-dhcp-server',
cmdd=dict(cmd='sudo initctl stop isc-dhcp-server')
)
else:
photon.m('exitvpn for %s - you are connected!!' % (community))
photon.m(
'setting batman server flag for %s (bw: %s)' % (
interface, bandwidth
),
cmdd=dict(cmd='sudo batctl -m %s gw server %s' % (
interface, bandwidth
)),
)
if 'stop/waiting' in photon.m(
'check if isc-dhcp-server is not running to start it',
cmdd=dict(cmd='initctl status isc-dhcp-server')
).get('out', ''):
photon.m(
'starting isc-dhcp-server',
cmdd=dict(cmd='sudo initctl start isc-dhcp-server')
)
def nagg_exitvpn_accouts():
from common import pinit
from datetime import datetime, timedelta
photon, settings = pinit('nagg_exitvpn_accouts', verbose=True)
# initialize the gateway-configs repo ...
photon.git_handler(
settings['configs']['local'],
remote_url=settings['configs']['remote']
)
# .. to load contents from the exitvpn.yaml into the settings
if not photon.settings.load('exitvpn', settings['configs']['exitvpn']):
photon.m(
'could not load exitvpn from git',
more=dict(
exitvpn=settings['configs']['exitvpn']
),
state=True
)
photon.s2m
res=dict(overdue=list(), warning=list(), good=list())
now = datetime.now()
for gateway in sorted(settings['exitvpn']['gateways'].keys()):
if settings['exitvpn']['gateways'][gateway].get('until'):
until = datetime.strptime(settings['exitvpn']['gateways'][gateway]['until'], settings['exitvpn']['conf']['date_format'])
delta = until - now
f = 'overdue' if delta <= timedelta(days=0) else 'warning' if delta <= timedelta(days=settings['exitvpn']['conf']['warndays']) else 'good'
res[f].append({
gateway: settings['exitvpn']['gateways'][gateway]
})
photon.m('results', more=res)
if now.weekday() == settings['exitvpn']['conf']['digestday'] or res['warning']:
punchline = 'Achtung! VPN Account läuft aus' if res['warning'] else 'VPN Wochenbericht'
mail = photon.mail_handler(
to=settings['common']['mailto']['admin'],
cc=settings['common']['mailto']['kontakt'],
sender=settings['common']['mailto']['local'],
subject='photon exitVPN notify',
punchline=punchline,
add_settings=False
)
mail.text = ''
mail.text = res
mail.text = 'Do not forget to update the exitvpn.yaml ( https://github.com/freifunk-mwu/gateway-configs.git )'
mail.text = ''
mail.send
def draw_traffic():
'''
Draws `vnstat <http://humdi.net/vnstat/>`_ Graphs, and glues them
into a Webseite.
Run only on machines which are connected to the mesh (Gateways,
Service-Machines).
* If a specified interface does not exist on this machine, it's skipped.
* If no vnstat database was found for one interface, it will be created.
'''
photon, settings = pinit('draw_traffic', verbose=True)
traffic = '<small>click to show or hide</small><br />'
avail_if = photon.m('checking for available interfaces',
cmdd=dict(cmd='sudo vnstat --iflist')).get('out', '')
interfaces = settings['web']['traffic']['interfaces'] + [
settings['fastd'][com]['interface']
for com in settings['common']['communities']
]
for interface in interfaces:
if interface in avail_if:
if not search_location(
path.join(settings['web']['traffic']['dbdir'], interface)):
photon.m('creating vnstat db for %s' % (interface),
cmdd=dict(cmd='sudo vnstat -u -i %s' % (interface)),
verbose=True)
images = ''
for flag, itype in settings['web']['traffic']['types']:
image = '%s-%s.png' % (interface, itype)
photon.m('drawing %s graph for %s' % (itype, interface),
cmdd=dict(cmd='vnstati -i %s -%s -o %s' %
(interface, flag,
path.join(settings['web']['output'],
'traffic', image))),
critical=False)
images += photon.template_handler(IMAGE,
fields=dict(
interface=interface,
itype=itype,
image=image)).sub
traffic += photon.template_handler(IFBLOCK,
fields=dict(interface=interface,
images=images)).sub
page(photon, traffic, sub='traffic')
def sync_meshkeys():
from common import pinit
photon, settings = pinit('sync_meshkeys', verbose=True)
for community in settings['common']['communities']:
git = photon.git_handler(
settings['fastd'][community]['local'],
remote_url=settings['fastd'][community]['remote']
)
git.cleanup
# send sighup to fastd to reload configuration (and keys)
photon.signal_handler(settings['fastd'][community]['pidfile'], cmdd_if_no_pid=dict(cmd='sudo service fastd start')).hup
git.publish
def mirror_openwrt_repo():
from os import path
from photon.util.files import read_file
from common import pinit
photon, settings = pinit('mirror_openwrt_repo', verbose=True)
lftp_conf = path.join(path.dirname(__file__), 'common', settings['openwrt']['lftp_conf'])
config_content=photon.m(
'mirror openwrt repo dir %s%s' %(settings['openwrt']['remote_repo_url'], settings['openwrt']['bb_stable_dir']),
cmdd=dict(
cmd='lftp -f %s' %(lftp_conf),
timeout=43200,
)
).get('out')
def sync_ffapi():
'''
Synchronizes the Freifunk-API Files with it's repositories
:Mainz: https://github.com/freifunk-mwu/ffapi-mainz
:Wiesbaden: https://github.com/freifunk-mwu/ffapi-wiesbaden
'''
photon, settings = pinit('sync_ffapi', verbose=True)
for community in settings['common']['communities']:
git = photon.git_handler(
settings['ffapi'][community]['local'],
remote_url=settings['ffapi'][community]['remote'])
git.cleanup
git.publish
def sync_ffapi():
'''
Synchronizes the Freifunk-API Files with it's repositories
:Mainz: https://github.com/freifunk-mwu/ffapi-mainz
:Wiesbaden: https://github.com/freifunk-mwu/ffapi-wiesbaden
'''
photon, settings = pinit('sync_ffapi', verbose=True)
for community in settings['common']['communities']:
git = photon.git_handler(
settings['ffapi'][community]['local'],
remote_url=settings['ffapi'][community]['remote']
)
git.cleanup
git.publish
def check_exitvpn():
'''
Sets the ``BATMAN`` Server Flag and starts/stops ``isc-dhcp-server``,
if some given Hosts can be reached through the `ExitVPN`.
'''
photon, settings = pinit('check_exitvpn', verbose=True)
ping = photon.ping_handler(net_if=settings['exitping']['interface'])
photon.m('sending pings', more=settings['exitping']['targets'])
ping.probe = settings['exitping']['targets']
photon.m('ping results', more=ping.status)
for community in settings['common']['communities']:
interface = settings['batman'][community]['interface']
bandwidth = settings['batman'][community]['bandwidth']
if ping.status['ratio'] <= settings['exitping']['min_ratio']:
photon.m('exitvpn for %s - you are _not_ connected!' % (community))
photon.m('removing batman server flag for %s' % (interface),
cmdd=dict(cmd='sudo batctl -m %s gw off' % (interface)))
if 'start/running' in photon.m(
'check if isc-dhcp-server is running to stop it',
cmdd=dict(cmd='initctl status isc-dhcp-server')).get(
'out', ''):
photon.m('stopping isc-dhcp-server',
cmdd=dict(cmd='sudo initctl stop isc-dhcp-server'))
else:
photon.m('exitvpn for %s - you are connected!!' % (community))
photon.m(
'setting batman server flag for %s (bw: %s)' %
(interface, bandwidth),
cmdd=dict(cmd='sudo batctl -m %s gw server %s' %
(interface, bandwidth)),
)
if 'stop/waiting' in photon.m(
'check if isc-dhcp-server is not running to start it',
cmdd=dict(cmd='initctl status isc-dhcp-server')).get(
'out', ''):
photon.m('starting isc-dhcp-server',
cmdd=dict(cmd='sudo initctl start isc-dhcp-server'))
def check_radvd():
'''
Checks if ``radvd`` is running.
Will try to restart it if previous check failed.
'''
photon, settings = pinit('check_radvd', verbose=True)
status = photon.m('testing for radvd',
cmdd=dict(cmd='sudo service radvd status', ),
critical=False)
if status.get('returncode') != 0:
photon.m('(re) starting radvd',
cmdd=dict(cmd='sudo service radvd start'),
critical=False)
else:
photon.m('radvd is running')
def check_bind():
'''
Checks if ``bind9`` is still running.
Will try to restart it if previous check failed.
'''
photon, settings = pinit('check_bind', verbose=True)
status = photon.m('testing for bind9',
cmdd=dict(cmd='sudo rndc status', ),
critical=False)
if status.get('returncode') != 0:
photon.m('(re) starting bind9',
cmdd=dict(cmd='sudo service bind9 start'),
critical=False)
else:
photon.m('bind9 is running')
def gen_documentation():
'''
Pulls updates from our repositories containing (Sphinx) documentation,
builds them, and on success copies over the generated files.
You may or may not see the results on
`rtfm.freifunk-mwu.de <http://rtfm.freifunk-mwu.de>`_,
the mirrored repos are currently:
:backend_scripts: https://github.com/freifunk-mwu/backend-scripts
:gluon_builder: https://github.com/freifunk-mwu/gluon-builder-ffmwu
:gluon_gateway: https://github.com/freifunk-mwu/technik-meta
:photon: https://github.com/spookey/photon
'''
photon, settings = pinit('gen_documentation', verbose=True)
result = {'build': {}, 'result': {}, 'update': {}}
lroot = search_location(settings['documentation']['local'],
create_in=settings['documentation']['local'])
for name, data in settings['documentation']['repositories'].items():
local = path.join(lroot, name)
builddir = path.join(settings['documentation']['builddir'], name)
outdir = path.join(settings['documentation']['output'], name)
result['update'][name] = photon.git_handler(
local, remote_url=data['remote'])._pull()
build = photon.m(
'building documentation for %s' % (name),
cmdd=dict(cmd='sphinx-build -a -b html %s %s' %
(path.join(local, data['docpath']), builddir)),
critical=False)
result['build'][name] = build
if build.get('returncode') == 0:
change_location(outdir, False, move=True)
change_location(builddir, outdir, move=True)
result['result'] = photon.m('build for %s successful')
photon.m('all done', more=result)
def snapshot_configs():
from os import path
from photon.util.locations import change_location
from photon.util.files import write_file
from common import pinit
photon, settings = pinit('snapshot_configs', verbose=True)
git = photon.git_handler(
settings['configs']['local'],
remote_url=settings['configs']['remote']
)
git.cleanup
if not photon.settings.load('queue', settings['configs']['queue']):
photon.m(
'could not load snapshot queue from git',
more=dict(
queue=settings['configs']['queue']
),
state=True
)
photon.s2m
change_location(settings['configs']['target'], False, move=True)
for loc in settings['queue'].get('locations') + settings['configs']['qadd']:
change_location(loc, path.join(settings['configs']['target'], loc.lstrip('/')))
for b_cmd, b_file in [
('crontab -l', 'crontab'), ('dpkg -l', 'package_list')
]:
result = photon.m(
'retrieving %s contents' %(b_cmd),
cmdd=dict(cmd=b_cmd),
critical=False
)
if result.get('returncode') == 0:
write_file(path.join(settings['configs']['target'], b_file), result.get('out'))
git.publish
def deploy_ssh():
'''
Adds keys into the ``~/.ssh/authorized_keys`` which are not already there.
The Keys are read from the ``ssh.yaml`` inside the
`gateway-configs <https://github.com/freifunk-mwu/gateway-configs>`_
repository.
'''
parser = ArgumentParser(prog='deploy_ssh')
photon, settings = pinit('deploy_ssh', verbose=True)
# initialize the gateway-configs repo ...
photon.git_handler(
settings['configs']['local'],
remote_url=settings['configs']['remote']
)._pull()
# .. to load contents from the ssh.yaml into the settings
if not photon.settings.load(
'ssh_deploy', settings['configs']['ssh_deploy']
):
photon.m(
'could not load ssh_deploy',
more=dict(ssh_deploy=settings['configs']['ssh_deploy']),
state=True
)
photon.s2m
parser.add_argument(
'mtype',
action='store',
choices=settings['ssh_deploy'].keys()
)
args = parser.parse_args()
for key in settings['ssh_deploy'][args.mtype]:
conf_t = photon.template_handler(
'%s\n' % (settings['ssh_deploy'][args.mtype][key])
)
conf_t.write(settings['crypt']['ssh']['authorized'])
def gen_website():
'''
Generates a basic landing page, linking to firmware and/or traffic
statistics (if any).
Adds some basic system statistics by running shell commands, putting
the output into the status page.
'''
photon, settings = pinit('gen_website', verbose=True)
main = ''
if path.exists(path.join(settings['web']['output'], 'firmware')):
main += BLOCK.format(href='firmware', text='Firmware')
if path.exists(path.join(settings['web']['output'], '_archive')):
main += BLOCK.format(href='_archive', text='Firmware Archive')
if path.exists(path.join(settings['web']['output'], 'traffic')):
main += BLOCK.format(href='traffic', text='Traffic')
main += BLOCK.format(href='system', text='System Statistics')
page(photon, main)
sys = '<small>click to show or hide</small><br />'
for command in settings['web']['system']:
cmd_output = photon.m(
'retrieving system info',
cmdd=dict(cmd=command),
critical=False
).get('out')
sys += SYSBLOCK.format(
command=command,
command_tag=command.split()[0],
cmd_output=cmd_output
)
page(photon, sys, sub='system')
def check_bind9():
from common import pinit
photon, settings = pinit('check_bind9', verbose=True)
status = photon.m(
'testing for bind9',
cmdd=dict(
cmd='sudo rndc status',
),
critical=False
)
if status.get('returncode') != 0:
photon.m(
'(re) starting bind9',
cmdd=dict(
cmd='sudo service bind9 start'
),
critical=False
)
else:
photon.m('bind9 is running')
def gen_info(images, ccmd, start=None, finish=None):
'''
Before the build gets started, :ref:`bconf` starts the ``info.json``
with some general information.
It's purpose is to provide a single file to easily include links to the
latest firmware in foreign websites.
This file gets extended here, to hold a dictionary mapping each
router-model (keys) to appropriate `factory` and `sysupgrade` image
files with checksums.
To produce checksums, gluon's ``scripts/sha512sum.sh`` is used.
It is possible to change this using the ``-c`` command line flag.
.. seealso:: :func:`common.info_args` for command line syntax
'''
photon, settings = pinit('gen_info', verbose=True)
info = read_json(path.join(
settings['prepare']['stage_dir'],
settings['prepare']['info']
))
images = path.abspath(images)
if start and finish:
try:
seconds = abs(int(finish) - int(start))
except (TypeError, ValueError):
seconds = None
if seconds:
info['_info']['build_seconds'] = seconds
build_time = []
for name, val in [
('d', 60*60*24),
('h', 60*60),
('m', 60),
('s', 1)
]:
if seconds > val:
pval, seconds = divmod(seconds, val)
build_time.append('%d%s' % (pval, name))
info['_info']['build_time'] = ' '.join(build_time)
for variety in ['factory', 'sysupgrade']:
image_dir = path.join(images, variety)
if info and path.exists(image_dir):
for image_name in [
i for i in listdir(image_dir) if not i.endswith('manifest')
]:
model = image_name.split(
'%s-' % (info['_info']['release'])
)[-1].split(
'-%s.bin' % (variety)
)[0].split(
'.bin'
)[0]
checksum = photon.m(
'checksumming %s' % (model),
cmdd=dict(
cmd='%s %s' % (
path.abspath(ccmd),
path.join(image_dir, image_name)
)
)
).get('out')
info[model] = info.get(model, dict())
info[model][variety] = dict(
image=image_name,
checksum=checksum
)
write_json(path.join(images, settings['prepare']['info']), info)
photon.m('info generated', more=dict(images=images, info=info))
def gen_bconf(
branch, targets, signkey,
gluon_tag=None, site_tag=None,
broken=False, onlyone=False, priority=None
):
'''
Provides all information needed by the builder in placing a ``bconf``-file.
Since we are already collecting information here, the ``info.json``
is created as well.
The same arguments as in :func:`prepare` are used here.
.. seealso:: :func:`common.prepare_args` for command line syntax
'''
photon, settings = pinit('gen_bconf')
if photon.settings.load(
'siteconf',
path.join(
settings['site']['local']['wi'],
settings['site']['generator_settings']
)
):
photon.s2m
gluon, site = ginit(photon)
gluon_tag = gluon_tag if gluon_tag else gluon.short_commit[0]
site_tag = site_tag if site_tag else site.short_commit[0]
broken = '1' if broken else ''
version = settings['siteconf']['site']['gluon_release_num']
priority = (
priority if priority
else settings['siteconf']['site']['gluon_priority']
)
description = '-%s%s' % (
branch, '-%s' % (get_timestamp(time=False)) if
not all(settings['common']['branches']['avail'][branch])
else ''
)
# these fields appear in the info.json
fields = dict(
broken_flag=broken,
call_branch=branch,
communities=onlyone if onlyone else ' '.join(
settings['common']['communities'].keys()
),
gluon_tag=gluon_tag,
site_tag=site_tag,
priority=priority,
release='%s%s' % (version, description),
targets=' '.join(targets),
version=version
)
write_json(
path.join(
settings['prepare']['stage_dir'],
settings['prepare']['info']
),
dict(_info=fields)
)
# these fields only appear in the bconf
fields.update(dict(
build_branch=settings['common']['branches']['build'],
build_dir=settings['gluon']['local']['dir'],
info_file=settings['prepare']['info'],
library_dir=path.join(
settings['publish']['library_dir'],
'%s%s' % (version, description)
),
mkcmd=settings['common']['mkcmd'],
pycmd=settings['common']['pycmd'],
signkey=signkey,
stage_dir=settings['prepare']['stage_dir']
))
bconf = photon.template_handler(
settings['prepare']['bconf']['tpl'],
fields=fields
)
bconf.write(
settings['prepare']['bconf']['out'],
append=False
)
#!/usr/bin/env python3
from common import pinit
photon, settings = pinit('bootstrap_git', verbose=True)
DESC = 'Make sure this key is associated with the "freifunkmwu" user on github'
SSH_TPL = '''\n
# freifunk-mwu github access
## ${desc}
Host ${gh_ident}
\tUser git
\tHostname github.com
\tPreferredAuthentications publickey
\tIdentityFile ~/.ssh/${prv_s}
\n'''
def mkprv_ssh():
return photon.m(
'generating new private ssh keypair',
cmdd=dict(
cmd='ssh-keygen -t rsa -b 4096 -N "" -f %s' %(settings['crypt']['ssh']['prv']),
cwd=settings['crypt']['ssh']['folder'], verbose=True
)
)
def getpub_ssh():
from photon.util.locations import search_location
from photon.util.files import write_file, read_file
if not search_location(settings['crypt']['ssh']['prv']):
def nagg_exitvpn_accouts():
'''
Runs through a list of VPN-accounts inside the
`gateway configs <https://github.com/freifunk-mwu/gateway-configs.git>`_,
and sends mails.
Either as weekly digest, or as daily reminder, a week before a VPN-account
is running out.
'''
photon, settings = pinit('nagg_exitvpn_accouts', verbose=True)
# initialize the gateway-configs repo ...
photon.git_handler(
settings['configs']['local'],
remote_url=settings['configs']['remote']
)._pull()
# .. to load contents from the exitvpn.yaml into the settings
if not photon.settings.load('exitvpn', settings['configs']['exitvpn']):
photon.m(
'could not load exitvpn from git',
more=dict(
exitvpn=settings['configs']['exitvpn']
),
state=True
)
photon.s2m
res = dict(overdue=list(), warning=list(), good=list())
now = datetime.now()
warndays = settings['exitvpn']['conf']['warndays']
digestday = settings['exitvpn']['conf']['digestday']
for gateway in sorted(settings['exitvpn']['gateways'].keys()):
if settings['exitvpn']['gateways'][gateway].get('until'):
until = datetime.strptime(
settings['exitvpn']['gateways'][gateway]['until'],
settings['exitvpn']['conf']['date_format']
)
delta = until - now
flag = 'good'
if delta <= timedelta(days=0):
flag = 'overdue'
elif delta <= timedelta(days=warndays):
flag = 'warning'
res[flag].append({
gateway: settings['exitvpn']['gateways'][gateway]
})
photon.m('results', more=res)
if now.weekday() == digestday or res['warning']:
punchline = 'VPN Wochenbericht'
if res['warning']:
punchline = 'Achtung! VPN Account läuft aus'
mail = photon.mail_handler(
to=settings['common']['mailto']['admin'],
cc=[
settings['common']['mailto']['kontakt_mz'],
settings['common']['mailto']['kontakt_wi']
],
sender=settings['common']['mailto']['local'],
subject='photon exitVPN notify',
punchline=punchline,
add_settings=False
)
mail.text = ''
mail.text = res
mail.text = 'Do not forget to update the exitvpn.yaml ' \
'( https://github.com/freifunk-mwu/gateway-configs.git )'
mail.text = ''
mail.send
def prepare(
branch,
gluon_tag=None, site_tag=None,
nomodules=False, onlyone=False, priority=None
):
'''
Checks out Gluon sources and site-conf repositories at proper commit-ids
or tags according to the branch to build.
Generates a site-conf afterwards.
Automatically invokes :func:`_gen_bconf` afterwards
.. seealso:: :func:`common.prepare_args` for command line syntax
'''
photon, settings = pinit('prepare', clean=True)
for community in [
onlyone
] if onlyone else settings['common']['communities'].keys():
change_location(
settings['gluon']['local'][community],
False,
move=True
)
tags = settings['common']['branches']['avail'][branch]
gluon, site = ginit(photon, community=community)
if gluon_tag:
if gluon.tag and gluon_tag in gluon.tag:
gluon.tag = gluon_tag
elif gluon.commit and (
gluon_tag in gluon.commit or gluon_tag in gluon.short_commit
):
gluon.commit = gluon_tag
else:
photon.m(
'Invalid git commit-id or tag specified for gluon',
state=True
)
else:
if tags[0]:
gluon.tag = None
else:
gluon.branch = None
if site_tag:
if site.tag and site_tag in site.tag:
site.tag = site_tag
elif site.commit and (
site_tag in site.commit or site_tag in site.short_commit
):
site.commit = site_tag
else:
photon.m(
'Invalid git commit-id or tag specified for site',
state=True
)
else:
if tags[1]:
site.tag = None
else:
site.branch = None
photon.m(
'generating site for %s' % (community),
cmdd=dict(
cmd='%s generate.py %s %s %s' % (
settings['common']['pycmd'],
community,
'--nomodules' if nomodules else '',
'--priority %s' % (priority) if priority else ''
),
cwd=settings['site']['local'][community]
),
verbose=True
)
def nagg_exitvpn_accouts():
'''
Runs through a list of VPN-accounts inside the
`gateway configs <https://github.com/freifunk-mwu/gateway-configs.git>`_,
and sends mails.
Either as weekly digest, or as daily reminder, a week before a VPN-account
is running out.
'''
photon, settings = pinit('nagg_exitvpn_accouts', verbose=True)
# initialize the gateway-configs repo ...
photon.git_handler(settings['configs']['local'],
remote_url=settings['configs']['remote'])._pull()
# .. to load contents from the exitvpn.yaml into the settings
if not photon.settings.load('exitvpn', settings['configs']['exitvpn']):
photon.m('could not load exitvpn from git',
more=dict(exitvpn=settings['configs']['exitvpn']),
state=True)
photon.s2m
res = dict(overdue=list(), warning=list(), good=list())
now = datetime.now()
warndays = settings['exitvpn']['conf']['warndays']
digestday = settings['exitvpn']['conf']['digestday']
for gateway in sorted(settings['exitvpn']['gateways'].keys()):
if settings['exitvpn']['gateways'][gateway].get('until'):
until = datetime.strptime(
settings['exitvpn']['gateways'][gateway]['until'],
settings['exitvpn']['conf']['date_format'])
delta = until - now
flag = 'good'
if delta <= timedelta(days=0):
flag = 'overdue'
elif delta <= timedelta(days=warndays):
flag = 'warning'
res[flag].append(
{gateway: settings['exitvpn']['gateways'][gateway]})
photon.m('results', more=res)
if now.weekday() == digestday or res['warning']:
punchline = 'VPN Wochenbericht'
if res['warning']:
punchline = 'Achtung! VPN Account läuft aus'
mail = photon.mail_handler(
to=settings['common']['mailto']['admin'],
cc=[
settings['common']['mailto']['kontakt_mz'],
settings['common']['mailto']['kontakt_wi']
],
sender=settings['common']['mailto']['local'],
subject='photon exitVPN notify',
punchline=punchline,
add_settings=False)
mail.text = ''
mail.text = res
mail.text = 'Do not forget to update the exitvpn.yaml ' \
'( https://github.com/freifunk-mwu/gateway-configs.git )'
mail.text = ''
mail.send
def bootstrap_git():
'''
Creates an ssh-keypair (*hostname_rsa*, *hostname_rsa.pub*) and
adds them for unified access to GitHub into the ``~/.ssh/config``.
Do not forget to add the generated key to the GitHub-User
`ffmwu <https://github.com/freifunkmwu>`_.
It is member of the group
`machines <https://github.com/orgs/freifunk-mwu/teams/machines>`_,
and thus has access to all needed reposotories.
'''
photon, settings = pinit('bootstrap_git', verbose=True)
def mkprv_ssh():
return photon.m(
'generating new private ssh keypair',
cmdd=dict(
cmd='ssh-keygen -t rsa -b 4096 -N "" -f %s' % (
settings['crypt']['ssh']['prv']
),
cwd=settings['crypt']['ssh']['folder'], verbose=True
)
)
def getpub_ssh():
if not search_location(settings['crypt']['ssh']['prv']):
mkprv_ssh()
pub = photon.m(
'generating new public ssh key from private',
cmdd=dict(
cmd='ssh-keygen -f %s -y' % (settings['crypt']['ssh']['prv']),
cwd=settings['crypt']['ssh']['folder']
)
)
if not pub.get('returncode') == 0:
photon.m('Error creating public ssh key', state=False)
if read_file(settings['crypt']['ssh']['pub']) != pub.get('out'):
write_file(settings['crypt']['ssh']['pub'], pub.get('out'))
photon.m('wrote public ssh key', more=pub)
return pub.get('out')
pub = getpub_ssh()
if not pub:
photon.m('Can not bootstrap git configuration', state=True)
# add entry to config
conf_t = photon.template_handler(SSH_TPL)
conf_t.sub = dict(
desc=DESC,
gh_ident=settings['common']['gh_ident'],
prv_s=settings['crypt']['ssh']['prv_s']
)
conf_t.write(settings['crypt']['ssh']['conf'])
photon.m(
'setting git user.name',
cmdd=dict(
cmd='git config --global --replace-all user.name "%s"' % (
settings['common']['hostname']
)
)
)
photon.m(
'setting git user.email',
cmdd=dict(
cmd='git config --global --replace-all user.email "%s@%s"' % (
settings['common']['hostname'],
settings['common']['domain']
)
)
)
# display public-key on shell
shell_t = photon.template_handler(
'${br}${desc}${br}\n${pub} ${local}\n${br}',
fields=dict(
desc=DESC,
pub=pub,
local=settings['common']['mailto']['local'],
br='\n%s\n' % ('~'*8)
)
)
photon.m(shell_t.sub, verbose=True)
def gen_expansion_map():
'''
This script generates several expansion-maps.
It pulls updates first, generates a map, patches assets, and then
copies over the generated files.
A common ``app.chache`` file is used for all builds, for speedup reasons.
When finished, it leaves the clean repo behind.
'''
photon, settings = pinit('gen_expansion_map')
# fetch updates from remote repository
git = photon.git_handler(
settings['expansion']['local'],
remote_url=settings['expansion']['remote']
)
git._pull()
for name, sub in settings['expansion']['maps'].items():
# generate map
build = photon.m(
'generate %s expansion map' % (name),
cmdd=dict(
cmd='./mkpoly -f nodelist %s' % (sub['url']),
cwd=settings['expansion']['local'],
timeout=600, # for initial run, only (to create app.cache)
)
)
if build.get('returncode') == 0:
for patch in settings['expansion']['patch']:
content = read_file(patch)
if not content:
# skip empty files
continue
# set title
content = re_sub(
r'<title>(.*?)</title>',
'<title>%s</title>' % (sub['title']),
content
)
# set description
content = re_sub(
r'this._div.innerHTML\ =\ \'<h4>(.*?)</h4>\'',
'this._div.innerHTML = \'<h4>%s</h4>\'' % (sub['descr']),
content
)
# set initial position
content = re_sub(
r'L\.map\(\'map\'\)\.setView\((.*?),\ 10\);',
'L.map(\'map\').setView([%s, %s], 10);' % (
sub['ipos'][0], sub['ipos'][1]
),
content
)
# save result
photon.m('written %d bytes into %s' % (
write_file(patch, content),
patch
))
# copy generated files & folders
search_location(sub['output'], create_in=sub['output'])
for folder in ['js', 'css']:
change_location(
path.join(settings['expansion']['local'], folder),
path.join(sub['output'], folder),
move=False
)
for fdoc in ['nodes.geojson', 'index.html', 'LICENSE']:
change_location(
path.join(settings['expansion']['local'], fdoc),
sub['output'],
move=False
)
# reset changed files
for reset in settings['expansion']['patch'] + [
path.join(settings['expansion']['local'], 'nodes.geojson')
]:
photon.m('clean up %s' % (reset))
git._checkout('-- %s' % (reset))
def gen_expansion_map():
'''
This script generates several expansion-maps.
It pulls updates first, generates a map, patches assets, and then
copies over the generated files.
A common ``app.chache`` file is used for all builds, for speedup reasons.
When finished, it leaves the clean repo behind.
'''
photon, settings = pinit('gen_expansion_map')
# fetch updates from remote repository
git = photon.git_handler(settings['expansion']['local'],
remote_url=settings['expansion']['remote'])
git._pull()
for name, sub in settings['expansion']['maps'].items():
# generate map
build = photon.m(
'generate %s expansion map' % (name),
cmdd=dict(
cmd='./mkpoly -f nodelist %s' % (sub['url']),
cwd=settings['expansion']['local'],
timeout=600, # for initial run, only (to create app.cache)
))
if build.get('returncode') == 0:
for patch in settings['expansion']['patch']:
content = read_file(patch)
if not content:
# skip empty files
continue
# set title
content = re_sub(r'<title>(.*?)</title>',
'<title>%s</title>' % (sub['title']), content)
# set description
content = re_sub(
r'this._div.innerHTML\ =\ \'<h4>(.*?)</h4>\'',
'this._div.innerHTML = \'<h4>%s</h4>\'' % (sub['descr']),
content)
# set initial position
content = re_sub(
r'L\.map\(\'map\'\)\.setView\((.*?),\ 10\);',
'L.map(\'map\').setView([%s, %s], 10);' %
(sub['ipos'][0], sub['ipos'][1]), content)
# save result
photon.m('written %d bytes into %s' %
(write_file(patch, content), patch))
# copy generated files & folders
search_location(sub['output'], create_in=sub['output'])
for folder in ['js', 'css']:
change_location(path.join(settings['expansion']['local'],
folder),
path.join(sub['output'], folder),
move=False)
for fdoc in ['nodes.geojson', 'index.html', 'LICENSE']:
change_location(path.join(settings['expansion']['local'],
fdoc),
sub['output'],
move=False)
# reset changed files
for reset in settings['expansion']['patch'] + [
path.join(settings['expansion']['local'], 'nodes.geojson')
]:
photon.m('clean up %s' % (reset))
git._checkout('-- %s' % (reset))
#!/usr/bin/env python3
if __name__ == '__main__':
from common import pinit
from argparse import ArgumentParser
argp = ArgumentParser(prog='deploy_ssh')
photon, settings = pinit('deploy_ssh', verbose=True)
# initialize the gateway-configs repo ...
photon.git_handler(
settings['configs']['local'],
remote_url=settings['configs']['remote']
)
# .. to load contents from the ssh.yaml into the settings
if not photon.settings.load('ssh_deploy', settings['configs']['ssh_deploy']):
photon.m(
'could not load ssh_deploy',
more=dict(
ssh_deploy=settings['configs']['ssh_deploy']
),
state=True
)
photon.s2m
argp.add_argument(
'mtype',
action='store',
choices=settings['ssh_deploy'].keys()
)
