def build_logger(msg):
    '''
    This function starts a new photon instance to append new messages
    into the ``builder_meta.json`` file.
    It is used in :ref:`builder` to mark certain steps taken for validation
    of all actions done while compiling (remember: the resulting images
    need to get signed for the autoupdater to work)

    :param msg: The message to write into the meta file

    .. seealso:: :func:`common.log_args` for command line syntax
    '''
    pinit('build_logger', verbose=False)[0].m(msg, verbose=True)
Ejemplo n.º 2
0
def check_radvd():
    '''
    Checks if ``radvd`` is running.

    Will try to restart it if previous check failed.
    '''
    photon, settings = pinit('check_radvd', verbose=True)

    status = photon.m(
        'testing for radvd',
        cmdd=dict(
            cmd='sudo service radvd status',
        ),
        critical=False
    )

    if status.get('returncode') != 0:
        photon.m(
            '(re) starting radvd',
            cmdd=dict(
                cmd='sudo service radvd start'
            ),
            critical=False
        )
    else:
        photon.m('radvd is running')
def gen_website():
    '''
    Generates a basic landing page, linking to firmware and/or traffic
    statistics (if any).

    Adds some basic system statistics by running shell commands, putting
    the output into the status page.
    '''
    photon, settings = pinit('gen_website', verbose=True)

    main = ''
    if path.exists(path.join(settings['web']['output'], 'firmware')):
        main += BLOCK.format(href='firmware', text='Firmware')

    if path.exists(path.join(settings['web']['output'], '_archive')):
        main += BLOCK.format(href='_archive', text='Firmware Archive')

    if path.exists(path.join(settings['web']['output'], 'traffic')):
        main += BLOCK.format(href='traffic', text='Traffic')

    main += BLOCK.format(href='system', text='System Statistics')

    page(photon, main)

    sys = '<small>click to show or hide</small><br />'
    for command in settings['web']['system']:
        cmd_output = photon.m('retrieving system info',
                              cmdd=dict(cmd=command),
                              critical=False).get('out')
        sys += SYSBLOCK.format(command=command,
                               command_tag=command.split()[0],
                               cmd_output=cmd_output)

    page(photon, sys, sub='system')
Ejemplo n.º 4
0
def deploy_ssh():
    '''
    Adds keys into the ``~/.ssh/authorized_keys`` which are not already there.

    The Keys are read from the ``ssh.yaml`` inside the
    `gateway-configs <https://github.com/freifunk-mwu/gateway-configs>`_
    repository.
    '''
    parser = ArgumentParser(prog='deploy_ssh')
    photon, settings = pinit('deploy_ssh', verbose=True)

    # initialize the gateway-configs repo ...
    photon.git_handler(settings['configs']['local'],
                       remote_url=settings['configs']['remote'])._pull()

    # .. to load contents from the ssh.yaml into the settings
    if not photon.settings.load('ssh_deploy',
                                settings['configs']['ssh_deploy']):
        photon.m('could not load ssh_deploy',
                 more=dict(ssh_deploy=settings['configs']['ssh_deploy']),
                 state=True)
    photon.s2m

    parser.add_argument('mtype',
                        action='store',
                        choices=settings['ssh_deploy'].keys())
    args = parser.parse_args()

    for key in settings['ssh_deploy'][args.mtype]:
        conf_t = photon.template_handler(
            '%s\n' % (settings['ssh_deploy'][args.mtype][key]))
        conf_t.write(settings['crypt']['ssh']['authorized'])
Ejemplo n.º 5
0
def check_bind():
    '''
    Checks if ``bind9`` is still running.

    Will try to restart it if previous check failed.
    '''
    photon, settings = pinit('check_bind', verbose=True)

    status = photon.m(
        'testing for bind9',
        cmdd=dict(
            cmd='sudo rndc status',
        ),
        critical=False
    )

    if status.get('returncode') != 0:
        photon.m(
            '(re) starting bind9',
            cmdd=dict(
                cmd='sudo service bind9 start'
            ),
            critical=False
        )
    else:
        photon.m('bind9 is running')
def uni_manifest(branch, manifest):
    '''
    After building the images, a manifest file gets created.

    :param branch: The branch currently building
    :param manifest: The path to the manifest file

    Before the manifest gets signed, this function is called to enable
    cross releasing of branches by clever symlinking.

    .. seealso:: :func:`common.uni_args` for command line syntax
    '''
    photon, settings = pinit('uni_manifest', verbose=True)

    manifest = path.abspath(manifest)
    manifest_content = read_file(manifest)
    if manifest_content and manifest_content.count('BRANCH=') == 1:

        uni_branch = '\n'.join(
            'BRANCH=%s' % (avail) for avail in sorted(
                settings['common']['branches']['avail'].keys()
            )
        )
        new_manifest = photon.template_handler(
            manifest_content.replace('BRANCH=%s' % (branch), '${uni_branch}'),
            fields=dict(uni_branch=uni_branch)
        )
        change_location(manifest, False, move=True)
        new_manifest.write(
            manifest.replace('%s.manifest' % (branch), 'manifest'),
            append=False
        )

        for avail in settings['common']['branches']['avail'].keys():
            manifest_link = manifest.replace(
                '%s.manifest' % (branch), '%s.manifest' % (avail)
            )

            change_location(manifest_link, False, move=True)
            photon.m(
                'linking manifests %s' % (manifest_link),
                cmdd=dict(
                    cmd='ln -s manifest %s' % (path.basename(manifest_link)),
                    cwd=path.dirname(manifest_link)
                )
            )

        photon.m(
            'uni_manifest written',
            more=dict(branch=branch),
            verbose=True
        )

    else:
        photon.m(
            'Refusing to write uni_manifest',
            more=dict(branch=branch, manifest=manifest),
            state=False
        )
Ejemplo n.º 7
0
def draw_traffic():
    from os import path
    from photon.util.locations import search_location
    from common import pinit
    from common.html import page

    photon, settings = pinit('draw_traffic', verbose=True)

    traffic = '<small>click to show or hide</small><br />'
    avail_if = photon.m(
        'checking for available interfaces',
        cmdd=dict(
            cmd='sudo vnstat --iflist'
        )
    ).get('out', '')
    interfaces = settings['web']['traffic']['interfaces'] + [settings['fastd'][community]['interface'] for community in settings['fastd'].keys()]
    for interface in interfaces:
        if interface in avail_if:
            if not search_location(path.join(settings['web']['traffic']['dbdir'], interface)):
                photon.m(
                    'creating vnstat db for %s' %(interface),
                    cmdd=dict(
                        cmd='sudo vnstat -u -i %s' %(interface)
                    ),
                    verbose=True
                )

            images = ''
            for flag, itype in settings['web']['traffic']['types']:
                image = '%s-%s.png' %(interface, itype)
                photon.m(
                    'drawing %s graph for %s' %(itype, interface),
                    cmdd=dict(
                        cmd='vnstati -i %s -%s -o %s' %(interface, flag, path.join(settings['web']['output'], 'traffic', image))
                    ),
                    critical=False
                )

                images += photon.template_handler(
                    IMAGE,
                    fields=dict(
                        interface=interface,
                        itype=itype,
                        image=image
                    )
                ).sub

            traffic += photon.template_handler(
                IFBLOCK,
                fields=dict(
                    interface=interface,
                    images=images
                )
            ).sub

    page(photon, traffic, sub='traffic')
Ejemplo n.º 8
0
def update_bird_conf():
    from photon.util.files import read_file
    from common import pinit

    photon, settings = pinit('update_bird_conf', verbose=True)

    for repo in ['scripts', 'meta']:
        photon.git_handler(
            settings['icvpn']['bird'][repo]['local'],
            remote_url=settings['icvpn']['bird'][repo]['remote']
        )._pull()

    for ip_ver in settings['icvpn']['bird']['ip_ver']:
        do_restart = False
        bird_conf = photon.template_handler('${config_content}')

        # peers
        peers_config_content=photon.m(
            'generating ip_ver%s bgp peers conf' %(ip_ver),
            cmdd=dict(
                cmd='./mkbgp -f bird -%s -s %s -x mainz -x wiesbaden -d ebgp_ic' %(ip_ver, settings['icvpn']['bird']['meta']['local']),
                cwd=settings['icvpn']['bird']['scripts']['local']
            )
        ).get('out')
        bird_peers_conf.sub = dict(peers_config_content=peers_config_content)

        peers_conf = settings['icvpn']['bird']['ip_ver'][ip_ver]['peers_conf']

        if bird_peers_conf.sub != read_file(peers_conf):
            bird_peers_conf.write(peers_conf, append=False)
            do_restart = True

        # roa
        roa_config_content=photon.m(
            'generating ip_ver%s bgp roa conf' %(ip_ver),
            cmdd=dict(
                cmd='./mkroa -f bird -%s -s %s -x mainz -x wiesbaden' %(ip_ver, settings['icvpn']['bird']['meta']['local']),
                cwd=settings['icvpn']['bird']['scripts']['local']
            )
        ).get('out')
        bird_roa_conf.sub = dict(roa_config_content=roa_config_content)

        roa_conf = settings['icvpn']['bird']['ip_ver'][ip_ver]['roa_conf']

        if bird_roa_conf.sub != read_file(roa_conf):
            bird_roa_conf.write(roa_conf, append=False)
            do_restart = True

        if do_restart:
            photon.m(
                'restarting bird daemon for v%s' %(ip_ver),
                cmdd=dict(
                    cmd='sudo service %s restart' %(settings['icvpn']['bird']['ip_ver'][ip_ver]['exec'])
                )
            )
def gen_documentation():
    '''
    Pulls updates from our repositories containing (Sphinx) documentation,
    builds them, and on success copies over the generated files.

    You may or may not see the results on
    `rtfm.freifunk-mwu.de <http://rtfm.freifunk-mwu.de>`_,
    the mirrored repos are currently:

        :backend_scripts: https://github.com/freifunk-mwu/backend-scripts
        :gluon_builder: https://github.com/freifunk-mwu/gluon-builder-ffmwu
        :gluon_gateway: https://github.com/freifunk-mwu/technik-meta
        :photon: https://github.com/spookey/photon

    '''
    photon, settings = pinit('gen_documentation', verbose=True)
    result = {
        'build': {},
        'result': {},
        'update': {}
    }

    lroot = search_location(
        settings['documentation']['local'],
        create_in=settings['documentation']['local']
    )

    for name, data in settings['documentation']['repositories'].items():
        local = path.join(lroot, name)
        builddir = path.join(settings['documentation']['builddir'], name)
        outdir = path.join(settings['documentation']['output'], name)

        result['update'][name] = photon.git_handler(
            local,
            remote_url=data['remote']
        )._pull()

        build = photon.m(
            'building documentation for %s' % (name),
            cmdd=dict(
                cmd='sphinx-build -a -b html %s %s' % (
                    path.join(local, data['docpath']),
                    builddir
                )
            ),
            critical=False
        )
        result['build'][name] = build
        if build.get('returncode') == 0:
            change_location(outdir, False, move=True)
            change_location(builddir, outdir, move=True)
            result['result'] = photon.m('build for %s successful')

    photon.m('all done', more=result)
Ejemplo n.º 10
0
def check_exitvpn():
    from common import pinit

    photon, settings = pinit('check_exitvpn', verbose=True)

    uping = photon.ping_handler(net_if=settings['exitping']['interface'])
    aping = photon.ping_handler(net_if=settings['exitping']['interface'])

    uping.probe = settings['exitping']['urls']
    aping.probe = settings['exitping']['addresses']

    photon.m(
        'ping results',
        more=dict(
            ping_urls=uping.status,
            ping_addresses=aping.status
        )
    )

    for community in settings['common']['communities']:
        cif = settings['batman'][community]['interface']
        cbw = settings['batman'][community]['bandwidth']

        if uping.status['ratio'] <= settings['exitping']['min_ratio'] or aping.status['ratio'] <= settings['exitping']['min_ratio']:

            photon.m('exitvpn for %s - it seems you are not properly connected!' %(community))
            photon.m(
                'removing batman server flag for %s' %(cif),
                cmdd=dict(
                    cmd='sudo batctl -m %s gw off' %(cif)
                )
            )
            photon.m(
                'stopping isc-dhcp-server',
                cmdd=dict(
                    cmd='sudo initctl stop isc-dhcp-server'
                )
            )
        else:

            photon.m('exitvpn for %s - you are connected!!' %(community))
            photon.m(
                'setting batman server flag for %s (bw: %s)' %(cif, cbw),
                cmdd=dict(
                    cmd='sudo batctl -m %s gw server %s' %(cif, cbw)
                ),
            )
            photon.m(
                'starting isc-dhcp-server',
                cmdd=dict(
                    cmd='sudo initctl start isc-dhcp-server'
                ),
                critical=False
            )
def check_exitvpn():
    '''
    Sets the ``BATMAN`` Server Flag and starts/stops ``isc-dhcp-server``,
    if some given Hosts can be reached through the `ExitVPN`.
    '''
    photon, settings = pinit('check_exitvpn', verbose=True)

    ping = photon.ping_handler(net_if=settings['exitping']['interface'])

    photon.m('sending pings', more=settings['exitping']['targets'])
    ping.probe = settings['exitping']['targets']
    photon.m('ping results', more=ping.status)

    for community in settings['common']['communities']:
        interface = settings['batman'][community]['interface']
        bandwidth = settings['batman'][community]['bandwidth']

        if ping.status['ratio'] <= settings['exitping']['min_ratio']:
            photon.m('exitvpn for %s - you are _not_ connected!' % (community))

            photon.m(
                'removing batman server flag for %s' % (interface),
                cmdd=dict(cmd='sudo batctl -m %s gw off' % (interface))
            )

            if 'start/running' in photon.m(
                'check if isc-dhcp-server is running to stop it',
                cmdd=dict(cmd='initctl status isc-dhcp-server')
            ).get('out', ''):
                photon.m(
                    'stopping isc-dhcp-server',
                    cmdd=dict(cmd='sudo initctl stop isc-dhcp-server')
                )

        else:
            photon.m('exitvpn for %s - you are connected!!' % (community))

            photon.m(
                'setting batman server flag for %s (bw: %s)' % (
                    interface, bandwidth
                ),
                cmdd=dict(cmd='sudo batctl -m %s gw server %s' % (
                    interface, bandwidth
                )),
            )

            if 'stop/waiting' in photon.m(
                'check if isc-dhcp-server is not running to start it',
                cmdd=dict(cmd='initctl status isc-dhcp-server')
            ).get('out', ''):
                photon.m(
                    'starting isc-dhcp-server',
                    cmdd=dict(cmd='sudo initctl start isc-dhcp-server')
                )
def nagg_exitvpn_accouts():
    from common import pinit
    from datetime import datetime, timedelta

    photon, settings = pinit('nagg_exitvpn_accouts', verbose=True)

    # initialize the gateway-configs repo ...
    photon.git_handler(
        settings['configs']['local'],
        remote_url=settings['configs']['remote']
    )

    # .. to load contents from the exitvpn.yaml into the settings
    if not photon.settings.load('exitvpn', settings['configs']['exitvpn']):
        photon.m(
            'could not load exitvpn from git',
            more=dict(
                exitvpn=settings['configs']['exitvpn']
            ),
            state=True
        )
    photon.s2m

    res=dict(overdue=list(), warning=list(), good=list())
    now = datetime.now()

    for gateway in sorted(settings['exitvpn']['gateways'].keys()):
        if settings['exitvpn']['gateways'][gateway].get('until'):
            until = datetime.strptime(settings['exitvpn']['gateways'][gateway]['until'], settings['exitvpn']['conf']['date_format'])
            delta = until - now
            f = 'overdue' if delta <= timedelta(days=0) else 'warning' if delta <= timedelta(days=settings['exitvpn']['conf']['warndays']) else 'good'
            res[f].append({
                gateway: settings['exitvpn']['gateways'][gateway]
            })

    photon.m('results', more=res)

    if now.weekday() == settings['exitvpn']['conf']['digestday'] or res['warning']:
        punchline = 'Achtung! VPN Account läuft aus' if res['warning'] else 'VPN Wochenbericht'
        mail = photon.mail_handler(
            to=settings['common']['mailto']['admin'],
            cc=settings['common']['mailto']['kontakt'],
            sender=settings['common']['mailto']['local'],
            subject='photon exitVPN notify',
            punchline=punchline,
            add_settings=False
        )
        mail.text = ''
        mail.text = res
        mail.text = 'Do not forget to update the exitvpn.yaml ( https://github.com/freifunk-mwu/gateway-configs.git )'
        mail.text = ''
        mail.send
Ejemplo n.º 13
0
def draw_traffic():
    '''
    Draws `vnstat <http://humdi.net/vnstat/>`_  Graphs, and glues them
    into a Webseite.

    Run only on machines which are connected to the mesh (Gateways,
    Service-Machines).

    * If a specified interface does not exist on this machine, it's skipped.
    * If no vnstat database was found for one interface, it will be created.
    '''
    photon, settings = pinit('draw_traffic', verbose=True)

    traffic = '<small>click to show or hide</small><br />'
    avail_if = photon.m('checking for available interfaces',
                        cmdd=dict(cmd='sudo vnstat --iflist')).get('out', '')

    interfaces = settings['web']['traffic']['interfaces'] + [
        settings['fastd'][com]['interface']
        for com in settings['common']['communities']
    ]
    for interface in interfaces:
        if interface in avail_if:
            if not search_location(
                    path.join(settings['web']['traffic']['dbdir'], interface)):
                photon.m('creating vnstat db for %s' % (interface),
                         cmdd=dict(cmd='sudo vnstat -u -i %s' % (interface)),
                         verbose=True)

            images = ''
            for flag, itype in settings['web']['traffic']['types']:
                image = '%s-%s.png' % (interface, itype)
                photon.m('drawing %s graph for %s' % (itype, interface),
                         cmdd=dict(cmd='vnstati -i %s -%s -o %s' %
                                   (interface, flag,
                                    path.join(settings['web']['output'],
                                              'traffic', image))),
                         critical=False)

                images += photon.template_handler(IMAGE,
                                                  fields=dict(
                                                      interface=interface,
                                                      itype=itype,
                                                      image=image)).sub

            traffic += photon.template_handler(IFBLOCK,
                                               fields=dict(interface=interface,
                                                           images=images)).sub

    page(photon, traffic, sub='traffic')
Ejemplo n.º 14
0
def sync_meshkeys():
    from common import pinit

    photon, settings = pinit('sync_meshkeys', verbose=True)

    for community in settings['common']['communities']:
        git = photon.git_handler(
            settings['fastd'][community]['local'],
            remote_url=settings['fastd'][community]['remote']
        )
        git.cleanup

        # send sighup to fastd to reload configuration (and keys)
        photon.signal_handler(settings['fastd'][community]['pidfile'], cmdd_if_no_pid=dict(cmd='sudo service fastd start')).hup

        git.publish
def mirror_openwrt_repo():
    from os import path
    from photon.util.files import read_file
    from common import pinit

    photon, settings = pinit('mirror_openwrt_repo', verbose=True)

    lftp_conf = path.join(path.dirname(__file__), 'common', settings['openwrt']['lftp_conf'])

    config_content=photon.m(
        'mirror openwrt repo dir %s%s' %(settings['openwrt']['remote_repo_url'], settings['openwrt']['bb_stable_dir']),
        cmdd=dict(
            cmd='lftp -f %s' %(lftp_conf),
            timeout=43200,
        )
    ).get('out')
Ejemplo n.º 16
0
def sync_ffapi():
    '''
    Synchronizes the Freifunk-API Files with it's repositories

    :Mainz: https://github.com/freifunk-mwu/ffapi-mainz
    :Wiesbaden: https://github.com/freifunk-mwu/ffapi-wiesbaden
    '''
    photon, settings = pinit('sync_ffapi', verbose=True)

    for community in settings['common']['communities']:
        git = photon.git_handler(
            settings['ffapi'][community]['local'],
            remote_url=settings['ffapi'][community]['remote'])
        git.cleanup

        git.publish
Ejemplo n.º 17
0
def sync_ffapi():
    '''
    Synchronizes the Freifunk-API Files with it's repositories

    :Mainz: https://github.com/freifunk-mwu/ffapi-mainz
    :Wiesbaden: https://github.com/freifunk-mwu/ffapi-wiesbaden
    '''
    photon, settings = pinit('sync_ffapi', verbose=True)

    for community in settings['common']['communities']:
        git = photon.git_handler(
            settings['ffapi'][community]['local'],
            remote_url=settings['ffapi'][community]['remote']
        )
        git.cleanup

        git.publish
Ejemplo n.º 18
0
def check_exitvpn():
    '''
    Sets the ``BATMAN`` Server Flag and starts/stops ``isc-dhcp-server``,
    if some given Hosts can be reached through the `ExitVPN`.
    '''
    photon, settings = pinit('check_exitvpn', verbose=True)

    ping = photon.ping_handler(net_if=settings['exitping']['interface'])

    photon.m('sending pings', more=settings['exitping']['targets'])
    ping.probe = settings['exitping']['targets']
    photon.m('ping results', more=ping.status)

    for community in settings['common']['communities']:
        interface = settings['batman'][community]['interface']
        bandwidth = settings['batman'][community]['bandwidth']

        if ping.status['ratio'] <= settings['exitping']['min_ratio']:
            photon.m('exitvpn for %s - you are _not_ connected!' % (community))

            photon.m('removing batman server flag for %s' % (interface),
                     cmdd=dict(cmd='sudo batctl -m %s gw off' % (interface)))

            if 'start/running' in photon.m(
                    'check if isc-dhcp-server is running to stop it',
                    cmdd=dict(cmd='initctl status isc-dhcp-server')).get(
                        'out', ''):
                photon.m('stopping isc-dhcp-server',
                         cmdd=dict(cmd='sudo initctl stop isc-dhcp-server'))

        else:
            photon.m('exitvpn for %s - you are connected!!' % (community))

            photon.m(
                'setting batman server flag for %s (bw: %s)' %
                (interface, bandwidth),
                cmdd=dict(cmd='sudo batctl -m %s gw server %s' %
                          (interface, bandwidth)),
            )

            if 'stop/waiting' in photon.m(
                    'check if isc-dhcp-server is not running to start it',
                    cmdd=dict(cmd='initctl status isc-dhcp-server')).get(
                        'out', ''):
                photon.m('starting isc-dhcp-server',
                         cmdd=dict(cmd='sudo initctl start isc-dhcp-server'))
Ejemplo n.º 19
0
def check_radvd():
    '''
    Checks if ``radvd`` is running.

    Will try to restart it if previous check failed.
    '''
    photon, settings = pinit('check_radvd', verbose=True)

    status = photon.m('testing for radvd',
                      cmdd=dict(cmd='sudo service radvd status', ),
                      critical=False)

    if status.get('returncode') != 0:
        photon.m('(re) starting radvd',
                 cmdd=dict(cmd='sudo service radvd start'),
                 critical=False)
    else:
        photon.m('radvd is running')
Ejemplo n.º 20
0
def check_bind():
    '''
    Checks if ``bind9`` is still running.

    Will try to restart it if previous check failed.
    '''
    photon, settings = pinit('check_bind', verbose=True)

    status = photon.m('testing for bind9',
                      cmdd=dict(cmd='sudo rndc status', ),
                      critical=False)

    if status.get('returncode') != 0:
        photon.m('(re) starting bind9',
                 cmdd=dict(cmd='sudo service bind9 start'),
                 critical=False)
    else:
        photon.m('bind9 is running')
def gen_documentation():
    '''
    Pulls updates from our repositories containing (Sphinx) documentation,
    builds them, and on success copies over the generated files.

    You may or may not see the results on
    `rtfm.freifunk-mwu.de <http://rtfm.freifunk-mwu.de>`_,
    the mirrored repos are currently:

        :backend_scripts: https://github.com/freifunk-mwu/backend-scripts
        :gluon_builder: https://github.com/freifunk-mwu/gluon-builder-ffmwu
        :gluon_gateway: https://github.com/freifunk-mwu/technik-meta
        :photon: https://github.com/spookey/photon

    '''
    photon, settings = pinit('gen_documentation', verbose=True)
    result = {'build': {}, 'result': {}, 'update': {}}

    lroot = search_location(settings['documentation']['local'],
                            create_in=settings['documentation']['local'])

    for name, data in settings['documentation']['repositories'].items():
        local = path.join(lroot, name)
        builddir = path.join(settings['documentation']['builddir'], name)
        outdir = path.join(settings['documentation']['output'], name)

        result['update'][name] = photon.git_handler(
            local, remote_url=data['remote'])._pull()

        build = photon.m(
            'building documentation for %s' % (name),
            cmdd=dict(cmd='sphinx-build -a -b html %s %s' %
                      (path.join(local, data['docpath']), builddir)),
            critical=False)
        result['build'][name] = build
        if build.get('returncode') == 0:
            change_location(outdir, False, move=True)
            change_location(builddir, outdir, move=True)
            result['result'] = photon.m('build for %s successful')

    photon.m('all done', more=result)
Ejemplo n.º 22
0
def snapshot_configs():
    from os import path
    from photon.util.locations import change_location
    from photon.util.files import write_file
    from common import pinit

    photon, settings = pinit('snapshot_configs', verbose=True)

    git = photon.git_handler(
        settings['configs']['local'],
        remote_url=settings['configs']['remote']
    )
    git.cleanup

    if not photon.settings.load('queue', settings['configs']['queue']):
        photon.m(
            'could not load snapshot queue from git',
            more=dict(
                queue=settings['configs']['queue']
            ),
            state=True
        )
    photon.s2m

    change_location(settings['configs']['target'], False, move=True)

    for loc in settings['queue'].get('locations') + settings['configs']['qadd']:
        change_location(loc, path.join(settings['configs']['target'], loc.lstrip('/')))

    for b_cmd, b_file in [
        ('crontab -l', 'crontab'), ('dpkg -l', 'package_list')
    ]:
        result = photon.m(
            'retrieving %s contents' %(b_cmd),
            cmdd=dict(cmd=b_cmd),
            critical=False
        )
        if result.get('returncode') == 0:
            write_file(path.join(settings['configs']['target'], b_file), result.get('out'))

    git.publish
Ejemplo n.º 23
0
def deploy_ssh():
    '''
    Adds keys into the ``~/.ssh/authorized_keys`` which are not already there.

    The Keys are read from the ``ssh.yaml`` inside the
    `gateway-configs <https://github.com/freifunk-mwu/gateway-configs>`_
    repository.
    '''
    parser = ArgumentParser(prog='deploy_ssh')
    photon, settings = pinit('deploy_ssh', verbose=True)

    # initialize the gateway-configs repo ...
    photon.git_handler(
        settings['configs']['local'],
        remote_url=settings['configs']['remote']
    )._pull()

    # .. to load contents from the ssh.yaml into the settings
    if not photon.settings.load(
        'ssh_deploy', settings['configs']['ssh_deploy']
    ):
        photon.m(
            'could not load ssh_deploy',
            more=dict(ssh_deploy=settings['configs']['ssh_deploy']),
            state=True
        )
    photon.s2m

    parser.add_argument(
        'mtype',
        action='store',
        choices=settings['ssh_deploy'].keys()
    )
    args = parser.parse_args()

    for key in settings['ssh_deploy'][args.mtype]:
        conf_t = photon.template_handler(
            '%s\n' % (settings['ssh_deploy'][args.mtype][key])
        )
        conf_t.write(settings['crypt']['ssh']['authorized'])
Ejemplo n.º 24
0
def gen_website():
    '''
    Generates a basic landing page, linking to firmware and/or traffic
    statistics (if any).

    Adds some basic system statistics by running shell commands, putting
    the output into the status page.
    '''
    photon, settings = pinit('gen_website', verbose=True)

    main = ''
    if path.exists(path.join(settings['web']['output'], 'firmware')):
        main += BLOCK.format(href='firmware', text='Firmware')

    if path.exists(path.join(settings['web']['output'], '_archive')):
        main += BLOCK.format(href='_archive', text='Firmware Archive')

    if path.exists(path.join(settings['web']['output'], 'traffic')):
        main += BLOCK.format(href='traffic', text='Traffic')

    main += BLOCK.format(href='system', text='System Statistics')

    page(photon, main)

    sys = '<small>click to show or hide</small><br />'
    for command in settings['web']['system']:
        cmd_output = photon.m(
            'retrieving system info',
            cmdd=dict(cmd=command),
            critical=False
        ).get('out')
        sys += SYSBLOCK.format(
            command=command,
            command_tag=command.split()[0],
            cmd_output=cmd_output
        )

    page(photon, sys, sub='system')
Ejemplo n.º 25
0
def check_bind9():
    from common import pinit

    photon, settings = pinit('check_bind9', verbose=True)

    status = photon.m(
        'testing for bind9',
        cmdd=dict(
            cmd='sudo rndc status',
        ),
        critical=False
    )

    if status.get('returncode') != 0:
        photon.m(
            '(re) starting bind9',
            cmdd=dict(
                cmd='sudo service bind9 start'
            ),
            critical=False
        )
    else:
        photon.m('bind9 is running')
Ejemplo n.º 26
0
def gen_info(images, ccmd, start=None, finish=None):
    '''
    Before the build gets started, :ref:`bconf` starts the ``info.json``
    with some general information.
    It's purpose is to provide a single file to easily include links to the
    latest firmware in foreign websites.

    This file gets extended here, to hold a dictionary mapping each
    router-model (keys) to appropriate `factory` and `sysupgrade` image
    files with checksums.

    To produce checksums, gluon's ``scripts/sha512sum.sh`` is used.
    It is possible to change this using the ``-c`` command line flag.

    .. seealso:: :func:`common.info_args` for command line syntax
    '''
    photon, settings = pinit('gen_info', verbose=True)

    info = read_json(path.join(
        settings['prepare']['stage_dir'],
        settings['prepare']['info']
    ))
    images = path.abspath(images)

    if start and finish:
        try:
            seconds = abs(int(finish) - int(start))
        except (TypeError, ValueError):
            seconds = None
        if seconds:
            info['_info']['build_seconds'] = seconds
            build_time = []
            for name, val in [
                ('d', 60*60*24),
                ('h', 60*60),
                ('m', 60),
                ('s', 1)
            ]:
                if seconds > val:
                    pval, seconds = divmod(seconds, val)
                    build_time.append('%d%s' % (pval, name))
            info['_info']['build_time'] = ' '.join(build_time)

    for variety in ['factory', 'sysupgrade']:
        image_dir = path.join(images, variety)
        if info and path.exists(image_dir):
            for image_name in [
                i for i in listdir(image_dir) if not i.endswith('manifest')
            ]:
                model = image_name.split(
                    '%s-' % (info['_info']['release'])
                )[-1].split(
                    '-%s.bin' % (variety)
                )[0].split(
                    '.bin'
                )[0]

                checksum = photon.m(
                    'checksumming %s' % (model),
                    cmdd=dict(
                        cmd='%s %s' % (
                            path.abspath(ccmd),
                            path.join(image_dir, image_name)
                        )
                    )
                ).get('out')

                info[model] = info.get(model, dict())
                info[model][variety] = dict(
                    image=image_name,
                    checksum=checksum
                )

            write_json(path.join(images, settings['prepare']['info']), info)

    photon.m('info generated', more=dict(images=images, info=info))
Ejemplo n.º 27
0
def gen_bconf(
    branch, targets, signkey,
    gluon_tag=None, site_tag=None,
    broken=False, onlyone=False, priority=None
):
    '''
    Provides all information needed by the builder in placing a ``bconf``-file.

    Since we are already collecting information here, the ``info.json``
    is created as well.

    The same arguments as in :func:`prepare` are used here.

    .. seealso:: :func:`common.prepare_args` for command line syntax
    '''
    photon, settings = pinit('gen_bconf')

    if photon.settings.load(
        'siteconf',
        path.join(
            settings['site']['local']['wi'],
            settings['site']['generator_settings']
        )
    ):
        photon.s2m

        gluon, site = ginit(photon)
        gluon_tag = gluon_tag if gluon_tag else gluon.short_commit[0]
        site_tag = site_tag if site_tag else site.short_commit[0]

        broken = '1' if broken else ''
        version = settings['siteconf']['site']['gluon_release_num']
        priority = (
            priority if priority
            else settings['siteconf']['site']['gluon_priority']
        )
        description = '-%s%s' % (
            branch, '-%s' % (get_timestamp(time=False)) if
            not all(settings['common']['branches']['avail'][branch])
            else ''
        )

        # these fields appear in the info.json
        fields = dict(
            broken_flag=broken,
            call_branch=branch,
            communities=onlyone if onlyone else ' '.join(
                settings['common']['communities'].keys()
            ),
            gluon_tag=gluon_tag,
            site_tag=site_tag,
            priority=priority,
            release='%s%s' % (version, description),
            targets=' '.join(targets),
            version=version
        )
        write_json(
            path.join(
                settings['prepare']['stage_dir'],
                settings['prepare']['info']
            ),
            dict(_info=fields)
        )

        # these fields only appear in the bconf
        fields.update(dict(
            build_branch=settings['common']['branches']['build'],
            build_dir=settings['gluon']['local']['dir'],
            info_file=settings['prepare']['info'],
            library_dir=path.join(
                settings['publish']['library_dir'],
                '%s%s' % (version, description)
            ),
            mkcmd=settings['common']['mkcmd'],
            pycmd=settings['common']['pycmd'],
            signkey=signkey,
            stage_dir=settings['prepare']['stage_dir']
        ))

        bconf = photon.template_handler(
            settings['prepare']['bconf']['tpl'],
            fields=fields
        )
        bconf.write(
            settings['prepare']['bconf']['out'],
            append=False
        )
Ejemplo n.º 28
0
#!/usr/bin/env python3

from common import pinit

photon, settings = pinit('bootstrap_git', verbose=True)

DESC = 'Make sure this key is associated with the "freifunkmwu" user on github'
SSH_TPL = '''\n
# freifunk-mwu github access
## ${desc}
Host ${gh_ident}
\tUser git
\tHostname github.com
\tPreferredAuthentications publickey
\tIdentityFile ~/.ssh/${prv_s}
\n'''

def mkprv_ssh():
    return photon.m(
        'generating new private ssh keypair',
        cmdd=dict(
            cmd='ssh-keygen -t rsa -b 4096 -N "" -f %s' %(settings['crypt']['ssh']['prv']),
            cwd=settings['crypt']['ssh']['folder'], verbose=True
        )
    )

def getpub_ssh():
    from photon.util.locations import search_location
    from photon.util.files import write_file, read_file

    if not search_location(settings['crypt']['ssh']['prv']):
def nagg_exitvpn_accouts():
    '''
    Runs through a list of VPN-accounts inside the
    `gateway configs <https://github.com/freifunk-mwu/gateway-configs.git>`_,
    and sends mails.

    Either as weekly digest, or as daily reminder, a week before a VPN-account
    is running out.
    '''
    photon, settings = pinit('nagg_exitvpn_accouts', verbose=True)

    # initialize the gateway-configs repo ...
    photon.git_handler(
        settings['configs']['local'],
        remote_url=settings['configs']['remote']
    )._pull()

    # .. to load contents from the exitvpn.yaml into the settings
    if not photon.settings.load('exitvpn', settings['configs']['exitvpn']):
        photon.m(
            'could not load exitvpn from git',
            more=dict(
                exitvpn=settings['configs']['exitvpn']
            ),
            state=True
        )
    photon.s2m

    res = dict(overdue=list(), warning=list(), good=list())
    now = datetime.now()
    warndays = settings['exitvpn']['conf']['warndays']
    digestday = settings['exitvpn']['conf']['digestday']

    for gateway in sorted(settings['exitvpn']['gateways'].keys()):
        if settings['exitvpn']['gateways'][gateway].get('until'):
            until = datetime.strptime(
                settings['exitvpn']['gateways'][gateway]['until'],
                settings['exitvpn']['conf']['date_format']
            )
            delta = until - now
            flag = 'good'
            if delta <= timedelta(days=0):
                flag = 'overdue'
            elif delta <= timedelta(days=warndays):
                flag = 'warning'
            res[flag].append({
                gateway: settings['exitvpn']['gateways'][gateway]
            })

    photon.m('results', more=res)

    if now.weekday() == digestday or res['warning']:

        punchline = 'VPN Wochenbericht'
        if res['warning']:
            punchline = 'Achtung! VPN Account läuft aus'

        mail = photon.mail_handler(
            to=settings['common']['mailto']['admin'],
            cc=[
                settings['common']['mailto']['kontakt_mz'],
                settings['common']['mailto']['kontakt_wi']
            ],
            sender=settings['common']['mailto']['local'],
            subject='photon exitVPN notify',
            punchline=punchline,
            add_settings=False
        )
        mail.text = ''
        mail.text = res
        mail.text = 'Do not forget to update the exitvpn.yaml ' \
                    '( https://github.com/freifunk-mwu/gateway-configs.git )'
        mail.text = ''
        mail.send
Ejemplo n.º 30
0
def prepare(
    branch,
    gluon_tag=None, site_tag=None,
    nomodules=False, onlyone=False, priority=None
):
    '''
    Checks out Gluon sources and site-conf repositories at proper commit-ids
    or tags according to the branch to build.
    Generates a site-conf afterwards.
    Automatically invokes :func:`_gen_bconf` afterwards

    .. seealso:: :func:`common.prepare_args` for command line syntax
    '''
    photon, settings = pinit('prepare', clean=True)

    for community in [
        onlyone
    ] if onlyone else settings['common']['communities'].keys():

        change_location(
            settings['gluon']['local'][community],
            False,
            move=True
        )
        tags = settings['common']['branches']['avail'][branch]
        gluon, site = ginit(photon, community=community)

        if gluon_tag:
            if gluon.tag and gluon_tag in gluon.tag:
                gluon.tag = gluon_tag
            elif gluon.commit and (
                gluon_tag in gluon.commit or gluon_tag in gluon.short_commit
            ):
                gluon.commit = gluon_tag
            else:
                photon.m(
                    'Invalid git commit-id or tag specified for gluon',
                    state=True
                )
        else:
            if tags[0]:
                gluon.tag = None
            else:
                gluon.branch = None

        if site_tag:
            if site.tag and site_tag in site.tag:
                site.tag = site_tag
            elif site.commit and (
                site_tag in site.commit or site_tag in site.short_commit
            ):
                site.commit = site_tag
            else:
                photon.m(
                    'Invalid git commit-id or tag specified for site',
                    state=True
                )
        else:
            if tags[1]:
                site.tag = None
            else:
                site.branch = None

        photon.m(
            'generating site for %s' % (community),
            cmdd=dict(
                cmd='%s generate.py %s %s %s' % (
                    settings['common']['pycmd'],
                    community,
                    '--nomodules' if nomodules else '',
                    '--priority %s' % (priority) if priority else ''
                ),
                cwd=settings['site']['local'][community]
            ),
            verbose=True
        )
def nagg_exitvpn_accouts():
    '''
    Runs through a list of VPN-accounts inside the
    `gateway configs <https://github.com/freifunk-mwu/gateway-configs.git>`_,
    and sends mails.

    Either as weekly digest, or as daily reminder, a week before a VPN-account
    is running out.
    '''
    photon, settings = pinit('nagg_exitvpn_accouts', verbose=True)

    # initialize the gateway-configs repo ...
    photon.git_handler(settings['configs']['local'],
                       remote_url=settings['configs']['remote'])._pull()

    # .. to load contents from the exitvpn.yaml into the settings
    if not photon.settings.load('exitvpn', settings['configs']['exitvpn']):
        photon.m('could not load exitvpn from git',
                 more=dict(exitvpn=settings['configs']['exitvpn']),
                 state=True)
    photon.s2m

    res = dict(overdue=list(), warning=list(), good=list())
    now = datetime.now()
    warndays = settings['exitvpn']['conf']['warndays']
    digestday = settings['exitvpn']['conf']['digestday']

    for gateway in sorted(settings['exitvpn']['gateways'].keys()):
        if settings['exitvpn']['gateways'][gateway].get('until'):
            until = datetime.strptime(
                settings['exitvpn']['gateways'][gateway]['until'],
                settings['exitvpn']['conf']['date_format'])
            delta = until - now
            flag = 'good'
            if delta <= timedelta(days=0):
                flag = 'overdue'
            elif delta <= timedelta(days=warndays):
                flag = 'warning'
            res[flag].append(
                {gateway: settings['exitvpn']['gateways'][gateway]})

    photon.m('results', more=res)

    if now.weekday() == digestday or res['warning']:

        punchline = 'VPN Wochenbericht'
        if res['warning']:
            punchline = 'Achtung! VPN Account läuft aus'

        mail = photon.mail_handler(
            to=settings['common']['mailto']['admin'],
            cc=[
                settings['common']['mailto']['kontakt_mz'],
                settings['common']['mailto']['kontakt_wi']
            ],
            sender=settings['common']['mailto']['local'],
            subject='photon exitVPN notify',
            punchline=punchline,
            add_settings=False)
        mail.text = ''
        mail.text = res
        mail.text = 'Do not forget to update the exitvpn.yaml ' \
                    '( https://github.com/freifunk-mwu/gateway-configs.git )'
        mail.text = ''
        mail.send
Ejemplo n.º 32
0
def bootstrap_git():
    '''
    Creates an ssh-keypair (*hostname_rsa*, *hostname_rsa.pub*) and
    adds them for unified access to GitHub into the ``~/.ssh/config``.

    Do not forget to add the generated key to the GitHub-User
    `ffmwu <https://github.com/freifunkmwu>`_.
    It is member of the group
    `machines <https://github.com/orgs/freifunk-mwu/teams/machines>`_,
    and thus has access to all needed reposotories.
    '''
    photon, settings = pinit('bootstrap_git', verbose=True)

    def mkprv_ssh():
        return photon.m(
            'generating new private ssh keypair',
            cmdd=dict(
                cmd='ssh-keygen -t rsa -b 4096 -N "" -f %s' % (
                    settings['crypt']['ssh']['prv']
                ),
                cwd=settings['crypt']['ssh']['folder'], verbose=True
            )
        )

    def getpub_ssh():
        if not search_location(settings['crypt']['ssh']['prv']):
            mkprv_ssh()

        pub = photon.m(
            'generating new public ssh key from private',
            cmdd=dict(
                cmd='ssh-keygen -f %s -y' % (settings['crypt']['ssh']['prv']),
                cwd=settings['crypt']['ssh']['folder']
            )
        )
        if not pub.get('returncode') == 0:
            photon.m('Error creating public ssh key', state=False)

        if read_file(settings['crypt']['ssh']['pub']) != pub.get('out'):
            write_file(settings['crypt']['ssh']['pub'], pub.get('out'))
            photon.m('wrote public ssh key', more=pub)

        return pub.get('out')

    pub = getpub_ssh()
    if not pub:
        photon.m('Can not bootstrap git configuration', state=True)

    # add entry to config
    conf_t = photon.template_handler(SSH_TPL)
    conf_t.sub = dict(
        desc=DESC,
        gh_ident=settings['common']['gh_ident'],
        prv_s=settings['crypt']['ssh']['prv_s']
    )
    conf_t.write(settings['crypt']['ssh']['conf'])

    photon.m(
        'setting git user.name',
        cmdd=dict(
            cmd='git config --global --replace-all user.name "%s"' % (
                settings['common']['hostname']
            )
        )
    )
    photon.m(
        'setting git user.email',
        cmdd=dict(
            cmd='git config --global --replace-all user.email "%s@%s"' % (
                settings['common']['hostname'],
                settings['common']['domain']
            )
        )
    )

    # display public-key on shell
    shell_t = photon.template_handler(
        '${br}${desc}${br}\n${pub} ${local}\n${br}',
        fields=dict(
            desc=DESC,
            pub=pub,
            local=settings['common']['mailto']['local'],
            br='\n%s\n' % ('~'*8)
        )
    )
    photon.m(shell_t.sub, verbose=True)
def gen_expansion_map():
    '''
    This script generates several expansion-maps.

    It pulls updates first, generates a map, patches assets, and then
    copies over the generated files.

    A common ``app.chache`` file is used for all builds, for speedup reasons.

    When finished, it leaves the clean repo behind.
    '''
    photon, settings = pinit('gen_expansion_map')

    # fetch updates from remote repository
    git = photon.git_handler(
        settings['expansion']['local'],
        remote_url=settings['expansion']['remote']
    )
    git._pull()

    for name, sub in settings['expansion']['maps'].items():
        # generate map
        build = photon.m(
            'generate %s expansion map' % (name),
            cmdd=dict(
                cmd='./mkpoly -f nodelist %s' % (sub['url']),
                cwd=settings['expansion']['local'],
                timeout=600,  # for initial run, only (to create app.cache)
            )
        )
        if build.get('returncode') == 0:
            for patch in settings['expansion']['patch']:
                content = read_file(patch)
                if not content:
                    # skip empty files
                    continue
                # set title
                content = re_sub(
                    r'<title>(.*?)</title>',
                    '<title>%s</title>' % (sub['title']),
                    content
                )
                # set description
                content = re_sub(
                    r'this._div.innerHTML\ =\ \'<h4>(.*?)</h4>\'',
                    'this._div.innerHTML = \'<h4>%s</h4>\'' % (sub['descr']),
                    content
                )
                # set initial position
                content = re_sub(
                    r'L\.map\(\'map\'\)\.setView\((.*?),\ 10\);',
                    'L.map(\'map\').setView([%s, %s], 10);' % (
                        sub['ipos'][0], sub['ipos'][1]
                    ),
                    content
                )
                # save result
                photon.m('written %d bytes into %s' % (
                    write_file(patch, content),
                    patch
                ))

            # copy generated files & folders
            search_location(sub['output'], create_in=sub['output'])
            for folder in ['js', 'css']:
                change_location(
                    path.join(settings['expansion']['local'], folder),
                    path.join(sub['output'], folder),
                    move=False
                )
            for fdoc in ['nodes.geojson', 'index.html', 'LICENSE']:
                change_location(
                    path.join(settings['expansion']['local'], fdoc),
                    sub['output'],
                    move=False
                )

        # reset changed files
        for reset in settings['expansion']['patch'] + [
            path.join(settings['expansion']['local'], 'nodes.geojson')
        ]:
            photon.m('clean up %s' % (reset))
            git._checkout('-- %s' % (reset))
Ejemplo n.º 34
0
def gen_expansion_map():
    '''
    This script generates several expansion-maps.

    It pulls updates first, generates a map, patches assets, and then
    copies over the generated files.

    A common ``app.chache`` file is used for all builds, for speedup reasons.

    When finished, it leaves the clean repo behind.
    '''
    photon, settings = pinit('gen_expansion_map')

    # fetch updates from remote repository
    git = photon.git_handler(settings['expansion']['local'],
                             remote_url=settings['expansion']['remote'])
    git._pull()

    for name, sub in settings['expansion']['maps'].items():
        # generate map
        build = photon.m(
            'generate %s expansion map' % (name),
            cmdd=dict(
                cmd='./mkpoly -f nodelist %s' % (sub['url']),
                cwd=settings['expansion']['local'],
                timeout=600,  # for initial run, only (to create app.cache)
            ))
        if build.get('returncode') == 0:
            for patch in settings['expansion']['patch']:
                content = read_file(patch)
                if not content:
                    # skip empty files
                    continue
                # set title
                content = re_sub(r'<title>(.*?)</title>',
                                 '<title>%s</title>' % (sub['title']), content)
                # set description
                content = re_sub(
                    r'this._div.innerHTML\ =\ \'<h4>(.*?)</h4>\'',
                    'this._div.innerHTML = \'<h4>%s</h4>\'' % (sub['descr']),
                    content)
                # set initial position
                content = re_sub(
                    r'L\.map\(\'map\'\)\.setView\((.*?),\ 10\);',
                    'L.map(\'map\').setView([%s, %s], 10);' %
                    (sub['ipos'][0], sub['ipos'][1]), content)
                # save result
                photon.m('written %d bytes into %s' %
                         (write_file(patch, content), patch))

            # copy generated files & folders
            search_location(sub['output'], create_in=sub['output'])
            for folder in ['js', 'css']:
                change_location(path.join(settings['expansion']['local'],
                                          folder),
                                path.join(sub['output'], folder),
                                move=False)
            for fdoc in ['nodes.geojson', 'index.html', 'LICENSE']:
                change_location(path.join(settings['expansion']['local'],
                                          fdoc),
                                sub['output'],
                                move=False)

        # reset changed files
        for reset in settings['expansion']['patch'] + [
                path.join(settings['expansion']['local'], 'nodes.geojson')
        ]:
            photon.m('clean up %s' % (reset))
            git._checkout('-- %s' % (reset))
Ejemplo n.º 35
0
#!/usr/bin/env python3

if __name__ == '__main__':
    from common import pinit
    from argparse import ArgumentParser

    argp = ArgumentParser(prog='deploy_ssh')
    photon, settings = pinit('deploy_ssh', verbose=True)

    # initialize the gateway-configs repo ...
    photon.git_handler(
        settings['configs']['local'],
        remote_url=settings['configs']['remote']
    )

    # .. to load contents from the ssh.yaml into the settings
    if not photon.settings.load('ssh_deploy', settings['configs']['ssh_deploy']):
        photon.m(
            'could not load ssh_deploy',
            more=dict(
                ssh_deploy=settings['configs']['ssh_deploy']
            ),
            state=True
        )
    photon.s2m

    argp.add_argument(
        'mtype',
        action='store',
        choices=settings['ssh_deploy'].keys()
    )