def run_command(self, command_c, slice_name=None, rspec_path=None):
""" runs given command for slice name and returns output.
command_c should be an array
"""
# Required args
command = [self.java_path, '-jar', self.jfed_cli_path]
command += command_c
# Optional args
# All optional args need to be initialized as array
if rspec_path:
command += ['--rspec', rspec_path]
if slice_name:
slice_name = 'urn:publicid:IDN+wall2.ilabt.iminds.be:' + self.project_name + '+slice+' + slice_name
command += ['-s', slice_name]
if self.call_log:
command += ['--call-log', '/var/log/jfed_call_log']
if self.key_path:
command += ['-p', self.key_path]
if self.password:
command += ['-P', self.password]
elif self.properties:
command += ['--context-file', self.properties]
else:
fail("either keyfile or context file must be specified")
if self.s4cert:
command += ['--speaks-for', self.s4cert]
debug("command is: `{}`".format(' '.join(command)))
iserror = False
try:
output = subprocess.check_output(command, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
output = ex.output
iserror = True
print "DEBUG: output = {}".format(output)
return parse_output(output, iserror)
def __init__(self, file, check_installed = False, sort = False, debug = False):
self._root_file = file
self._check_installed = check_installed
self._sort = sort
self._debug = debug
if self._debug: output.debug(__file__, "self._root_file -> %s", self._root_file)
self._cpvs = {}
def generate_next_state(self):
postRep = self.reader.read_post_as(PollPost.PostInFeed, url=self.url)
poll = PollPost(postRep).poll
finished = self.check_finished(poll.votes)
if finished and self.notify_on_complete:
notify("All votes are in.")
debug(*[
format_poll_option(option, voters)
for option, voters in poll.votes.items()
])
return poll, finished
def get_active_service():
service_name = app_globals.OPTIONS[OPTS_KEY]
if service_name == PAGEFEED:
debug("URL SAVE: pagefeed mode")
from pagefeed import PageFeed
return PageFeed()
elif service_name == INSTAPAPER:
debug("URL SAVE: instapaper mode")
from instapaper import Ipaper
return Ipaper()
else:
raise ValueError("%s is %s (expected %s)" % (
OPTS_KEY, service_name,
' or '.join((INSTAPAPER, PAGEFEED))))
def _read_directories(self):
if self._debug: output.debug(__file__, "os.walk(self._root_file) -> %s", os.walk(self._root_file))
if os.path.isdir(self._root_file):
for root, dirs, files in os.walk(self._root_file):
if self._debug:
output.debug(__file__, "root -> %s", root)
output.debug(__file__, "dirs -> %s", dirs)
output.debug(__file__, "files -> %s", files)
for file in files:
if self._debug: output.debug(__file__, "os.path.join(root, file) -> %s", os.path.join(root, file))
if os.path.isfile(os.path.join(root, file)):
self._read_file(os.path.join(root, file))
else:
self._read_file(self._root_file)
def _write_directories(self, file):
from gentoolkit.cpv import CPV
self._make_directory(file)
for key,value in self._cpvs.items():
cpv = CPV(key.split('=')[-1].split('>')[-1].split('<')[-1])
self._make_directory(os.path.join(file, cpv.category))
f = open(os.path.join(file, cpv.category, cpv.name), 'w')
if self._debug:
output.debug(__file__, "path: %s", os.path.join(file, cpv.category, cpv.name))
output.debug(__file__, "%s %s", key, ' '.join(value))
f.write("%s %s\n" % (key, ' '.join(value)))
f.close()
def get():
if (Browser.inst is None):
debug("creating singeton browser instance")
PROFILE = webdriver.FirefoxProfile()
# PROFILE.DEFAULT_PREFERENCES['frozen']['javascript.enabled'] = False
PROFILE.set_preference("dom.webnotifications.enabled", False)
PROFILE.set_preference("app.update.enabled", False)
PROFILE.update_preferences()
firefoxOptions = webdriver.FirefoxOptions()
firefoxOptions.add_argument("--width=1920`")
firefoxOptions.add_argument("--height=1080")
Browser.inst = webdriver.Firefox(executable_path=geckodriver_exe_path,
firefox_binary=FIREFOX_BINARY,
firefox_profile=PROFILE,
firefox_options=firefoxOptions)
return Browser.inst
def _remove_uninstalled_packages(self):
from gentoolkit.helpers import get_installed_cpvs
for cpv in self._cpvs.keys():
from gentoolkit.cpv import split_cpv
scpv = cpv.split('=')[-1].split('<')[-1].split('>')[-1]
try:
(category, pkg_name, version, revision) = split_cpv(scpv)
scpv = category + "/" + pkg_name
except: continue
if self._debug: output.debug(__file__, "scpv: %s", scpv)
pred = lambda x: x.startswith(scpv)
if self._debug:
output.debug(__file__, "len(set(get_installed_cpvs(pred))): %s", len(set(get_installed_cpvs(pred))))
if len(set(get_installed_cpvs(pred))) <= 0:
self._cpvs.pop(cpv)
def _read_file(self, file):
"""Read the file into a dictionary mapping the cpv to any flags.
We read each line in and the first item space delimited becomes
a key in the dictionary.
"""
f = open(file, 'r')
for line in f:
sline = line.split()
if sline[0] in self._cpvs:
self._cpvs[sline[0]].extend(sline[1:])
else:
self._cpvs[sline[0]] = sline[1:]
for flag in sline[1:]:
while self._cpvs[sline[0]].count(flag) > 1:
self._cpvs[sline[0]].remove(flag)
f.close()
if self._debug: output.debug("self._cpvs %s", self._cpvs)
def loadFile(self, sFile):
"""load host file and update configuraion
"""
logging.info("load config file {}".format(sFile))
if self.lastLoad > 0 and time.time(
) - self.lastLoad < self.config_cache:
logging.debug("cache too young ({:.0f}<{}), abort".format(
time.time() - self.lastLoad, self.config_cache))
return
try:
f = file(sFile, 'r')
except IOError:
logging.error("cannot access config file {}".format(sFile))
return False
c = f.read()
f.close()
conf = ''
try:
conf = json.loads(c)
except Exception as ex:
assert False, "configuration file load exception : {}".format(
", ".join(ex.args))
self.lastLoad = time.time()
self.confReadGlobal(conf)
self.confAddTemplates(conf)
# clean outputer array before inserting new configuration
while (len(outputer) > 0):
outputer.pop()
if conf.__contains__('output'):
for outputConf in conf['output']:
if outputConf['active'] == "True":
self.confOutputer.addDebug(outputConf)
self.confOutputer.addElastic(outputConf)
self.confOutputer.addLogstash(outputConf)
self.confOutputer.addUkn(outputConf)
else:
outputer.append(output.debug())
self.confAddProbes(conf)
self.fileName = sFile
logging.info("config file loaded in DB {}".format(sFile))
def run_command(self,
command_c,
slice_name=None,
rspec_path=None):
""" runs given command for slice name and returns output.
command_c should be an array
"""
# Required args
command = [self.java_path, '-jar', self.jfed_cli_path]
command += command_c
# Optional args
# All optional args need to be initialized as array
if rspec_path:
command += ['--rspec', rspec_path]
if slice_name:
slice_name = 'urn:publicid:IDN+wall2.ilabt.iminds.be:' + self.project_name + '+slice+' + slice_name
command += ['-s', slice_name]
if self.call_log:
command += ['--call-log', '/var/log/jfed_call_log']
if self.key_path:
command += ['-p', self.key_path]
if self.password:
command += ['-P', self.password]
elif self.properties:
command += ['--context-file', self.properties]
else:
fail("either keyfile or context file must be specified")
if self.s4cert:
command += ['--speaks-for', self.s4cert]
debug("command is: `{}`".format(' '.join(command)))
iserror = False
try:
output = subprocess.check_output(command, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
output = ex.output
iserror = True
print "DEBUG: output = {}".format(output)
return parse_output(output, iserror)
def loadFile(self, sFile):
"""load host file and update configuraion
"""
logging.info("load config file {}".format(sFile))
if self.lastLoad > 0 and time.time() - self.lastLoad < self.config_cache:
logging.debug("cache too young ({:.0f}<{}), abort".format(time.time() - self.lastLoad, self.config_cache))
return
try:
f = file(sFile, 'r')
except IOError:
logging.error("cannot access config file {}".format(sFile))
return False
c = f.read()
f.close()
conf = ''
try:
conf = json.loads(c)
except Exception as ex:
assert False, "configuration file load exception : {}".format(", ".join(ex.args))
self.lastLoad = time.time()
self.confReadGlobal(conf)
self.confAddTemplates(conf)
# clean outputer array before inserting new configuration
while (len(outputer) > 0):
outputer.pop()
if conf.__contains__('output'):
for outputConf in conf['output']:
if outputConf['active'] == "True":
self.confOutputer.addDebug(outputConf)
self.confOutputer.addElastic(outputConf)
self.confOutputer.addLogstash(outputConf)
self.confOutputer.addUkn(outputConf)
else:
outputer.append(output.debug())
self.confAddProbes(conf)
self.fileName = sFile
logging.info("config file loaded in DB {}".format(sFile))
def create_slice(self, slice_name, rspec_path, manifestpath, exp_hours=168):
"""Create new slice with given slice_name. Creates manifest on succes
and returns status. Status can be one of the following:
- SUCCESS
- FAIL_NO_BANDWITH
- FAIL_NO_NODES
- FAIL_NOT_YOUR_CREDENTIAL
- FAIL_UNKNOWN"""
debug("creating slice {} and waiting until all nodes are ready."
"This can take a few minutes.".format(slice_name))
create_c = ['create', '--create-slice',
# do not include userkeys because
# this doesn't work with "speaks-for"
'--ssh-keys', 'usercert,rspec',
'--manifest', manifestpath,
'--expiration-hours', str(exp_hours),
'--rewrite-rspec']
odict = self.run_command(create_c, slice_name=slice_name, rspec_path=rspec_path)
sliver_available = odict.get('sliver_available', False)
manifest_exists = os.path.isfile(manifestpath)
exit_err = odict['is_exit_code_error']
if exit_err or not manifest_exists or not sliver_available:
raise JfedError('Create failed', odict)
return odict
def create_slice(self, slice_name, rspec_path, manifestpath, exp_hours=168):
"""Create new slice with given slice_name. Creates manifest on succes
and returns status. Status can be one of the following:
- SUCCESS
- FAIL_NO_BANDWITH
- FAIL_NO_NODES
- FAIL_NOT_YOUR_CREDENTIAL
- FAIL_UNKNOWN"""
debug("creating slice {} and waiting until all nodes are ready."
"This can take a few minutes.".format(slice_name))
create_c = ['create', '--create-slice',
# do not include userkeys because
# this doesn't work with "speaks-for"
'--ssh-keys', 'usercert,rspec',
'--manifest', manifestpath,
'--expiration-hours', str(exp_hours),
'--rewrite-rspec']
odict = self.run_command(create_c, slice_name=slice_name, rspec_path=rspec_path)
sliver_available = odict.get('sliver_available', False)
manifest_exists = os.path.isfile(manifestpath)
exit_err = odict['is_exit_code_error']
if exit_err or not manifest_exists or not sliver_available:
raise JfedError('Create failed', odict)
return odict
def _remove_uninstalled_packages(self):
from gentoolkit.helpers import get_installed_cpvs
for cpv in self._cpvs.keys():
from gentoolkit.cpv import split_cpv
scpv = rcpv = cpv.split('=')[-1].split('<')[-1].split('>')[-1]
try:
(category, pkg_name, version, revision) = split_cpv(rcpv)
scpv = category + "/" + pkg_name
except: continue
if self._debug: output.debug(__file__, "scpv: %s", scpv)
pred = lambda x: x.startswith(scpv)
if self._debug:
output.debug(__file__, "list(get_installed_cpvs(pred)): %s", list(get_installed_cpvs(pred)))
output.debug(__file__, "rcpv: %s", rcpv)
output.debug(__file__, "''.join(list(get_installed_cpvs(pred))): %s", ''.join(list(get_installed_cpvs(pred))))
import re
if not re.search(rcpv, ''.join(list(get_installed_cpvs(pred)))):
if self._verbose: output.verbose("Could not find %s installed on the system!", rcpv)
self._cpvs.pop(cpv)
def run(conf):
email = conf.facebook_login_email
password = conf.facebook_login_password
if email == "" or password == "":
print(
"Your email or password is missing. These must both be in conf.py")
exit()
reader = PostReader(scrollDepth=conf.depth)
debug('logging in')
login(email, password)
posts = []
command = conf.command
if command == "watch-poll":
if conf.post is None and len(conf.groups) == 0:
parser.error(
"--post or --group option must be provided for watch-poll command"
)
return
url = conf.post if not (conf.post is None) else conf.groups[0]
while (True):
debug('reading post')
post_rep = reader.read_post_as(PollPost.PostInFeed, url=url)
if post_rep.is_poll():
debug('post contains poll')
if post_rep.is_open():
debug('post poll is open')
do_watch_poll(conf, reader, url)
elif not (conf.post is None):
posts.append(Post(reader.read_post(conf.post)))
else:
if conf.groups:
for groupId in conf.groups:
posts.extend(
[Post(rep) for rep in reader.read_group_posts(groupId)])
if conf.pages:
for pageName in conf.pages:
posts.extend(
[Post(rep) for rep in reader.read_page_posts(pageName)])
inform_user(encode(posts))
def generate_and_time_next_state(self):
start = time.time()
res,finished = self.generate_next_state()
debug("check completed ("+str(int(time.time() - start)) + "s)")
return res,finished
def ip_batch( C, file=False ):
"""Generate a batch file to perform load balancing
In order to share outgoing traffic between multiple ISP's, a batch file is
produced for the ip utility.
Examples:
>>> class X(object):
... def __init__(self,_d={},**kwargs):
... kwargs.update(_d)
... self.__dict__=kwargs
>>> N1 = X(net=lambda:'192.168.10.0/24')
>>> N2 = X(net=lambda:'192.168.20.0/24')
>>> N3 = X(net=lambda:'192.168.30.0/24')
>>> I1 = X(wan_interface=False,name='int-1',subnets=[N1,N2])
>>> I2 = X(wan_interface=False,name='int-2',subnets=[N3])
>>> I3 = X(wan_interface=False,name='int-3',subnets=[N1,N2,N3])
>>> I4 = X(wan_interface=False,name='int-4',subnets=[])
>>> E1 = X(wan_interface=True,name='ext-1',address='12.34.56.78',load_balance=1)
>>> E2 = X(wan_interface=True,name='ext-2',address='34.56.78.12',load_balance=2)
>>> E3 = X(wan_interface=True,name='ext-3',address='56.78.12.34',load_balance=3)
>>> C1 = X(Interfaces=[I3, E1])
>>> C2 = X(Interfaces=[I2, E1,E2])
>>> C3 = X(Interfaces=[I1,I4, E1,E2,E3])
>>> ip_batch(C1)
route flush table main
route flush table RTint-3
route flush table RText-1
<BLANKLINE>
route add 12.34.56.0/24 dev ext-1 src 12.34.56.78 table RText-1
route add 192.168.10.0/24 dev int-3 table RText-1
route add 192.168.20.0/24 dev int-3 table RText-1
route add 192.168.30.0/24 dev int-3 table RText-1
route add 127.0.0.0/8 dev lo table RText-1
route add default via 12.34.56.1 table RText-1
<BLANKLINE>
route add 12.34.56.0/24 dev ext-1 src 12.34.56.78
route add default via ext-1
<BLANKLINE>
route add default scope global nexthop via 12.34.56.1 dev ext-1 weight 1
<BLANKLINE>
rule add from 12.34.56.78 table RText-1
>>> ip_batch(C2)
route flush table main
route flush table RTint-2
route flush table RText-1
route flush table RText-2
<BLANKLINE>
route add 12.34.56.0/24 dev ext-1 src 12.34.56.78 table RText-1
route add 34.56.78.0/24 dev ext-2 table RText-1
route add 192.168.30.0/24 dev int-2 table RText-1
route add 127.0.0.0/8 dev lo table RText-1
route add default via 12.34.56.1 table RText-1
<BLANKLINE>
route add 34.56.78.0/24 dev ext-2 src 34.56.78.12 table RText-2
route add 12.34.56.0/24 dev ext-1 table RText-2
route add 192.168.30.0/24 dev int-2 table RText-2
route add 127.0.0.0/8 dev lo table RText-2
route add default via 34.56.78.1 table RText-2
<BLANKLINE>
route add 12.34.56.0/24 dev ext-1 src 12.34.56.78
route add 34.56.78.0/24 dev ext-2 src 34.56.78.12
route add default via ext-1
<BLANKLINE>
route add default scope global nexthop via 12.34.56.1 dev ext-1 weight 1 nexthop via 34.56.78.1 dev ext-2 weight 2
<BLANKLINE>
rule add from 12.34.56.78 table RText-1
rule add from 34.56.78.12 table RText-2
>>> ip_batch(C3)
route flush table main
route flush table RTint-1
route flush table RTint-4
route flush table RText-1
route flush table RText-2
route flush table RText-3
<BLANKLINE>
route add 12.34.56.0/24 dev ext-1 src 12.34.56.78 table RText-1
route add 34.56.78.0/24 dev ext-2 table RText-1
route add 56.78.12.0/24 dev ext-3 table RText-1
route add 192.168.10.0/24 dev int-1 table RText-1
route add 192.168.20.0/24 dev int-1 table RText-1
route add 127.0.0.0/8 dev lo table RText-1
route add default via 12.34.56.1 table RText-1
<BLANKLINE>
route add 34.56.78.0/24 dev ext-2 src 34.56.78.12 table RText-2
route add 12.34.56.0/24 dev ext-1 table RText-2
route add 56.78.12.0/24 dev ext-3 table RText-2
route add 192.168.10.0/24 dev int-1 table RText-2
route add 192.168.20.0/24 dev int-1 table RText-2
route add 127.0.0.0/8 dev lo table RText-2
route add default via 34.56.78.1 table RText-2
<BLANKLINE>
route add 56.78.12.0/24 dev ext-3 src 56.78.12.34 table RText-3
route add 12.34.56.0/24 dev ext-1 table RText-3
route add 34.56.78.0/24 dev ext-2 table RText-3
route add 192.168.10.0/24 dev int-1 table RText-3
route add 192.168.20.0/24 dev int-1 table RText-3
route add 127.0.0.0/8 dev lo table RText-3
route add default via 56.78.12.1 table RText-3
<BLANKLINE>
route add 12.34.56.0/24 dev ext-1 src 12.34.56.78
route add 34.56.78.0/24 dev ext-2 src 34.56.78.12
route add 56.78.12.0/24 dev ext-3 src 56.78.12.34
route add default via ext-1
<BLANKLINE>
route add default scope global nexthop via 12.34.56.1 dev ext-1 weight 1 nexthop via 34.56.78.1 dev ext-2 weight 2 nexthop via 56.78.12.1 dev ext-3 weight 3
<BLANKLINE>
rule add from 12.34.56.78 table RText-1
rule add from 34.56.78.12 table RText-2
rule add from 56.78.12.34 table RText-3
"""
# The network interfaces
Ext = [ I for I in C.Interfaces if I.wan_interface ]
Int = [ I for I in C.Interfaces if not I.wan_interface ]
routes = [['flush','table','main']]
lb_route = ['add','default','scope','global']
rules = []
main = []
# Clear all ip routes for now
routes += [ ['flush','table','RT'+I.name] for I in C.Interfaces ]
routes += [[]]
if len(Ext) == 0:
debug( 0, 'No interfaces to balance any load. Exiting.' )
return
for ifc in Ext:
T = 'RT'+ifc.name
routes += [['add',net(ifc),'dev',ifc.name,'src',ifc.address,'table',T]]
main += [['add',net(ifc),'dev',ifc.name,'src',ifc.address]]
routes += [ ['add',net(E), 'dev',E.name,'table',T] for E in Ext if E != ifc ]
routes += [ ['add',N.net(),'dev',I.name,'table',T] for I in Int for N in I.subnets ]
routes += [
['add','127.0.0.0/8','dev','lo','table',T],
['add','default','via',gw(ifc),'table',T],
[]
]
lb_route += ['nexthop','via',gw(ifc),'dev',ifc.name,'weight',str(ifc.load_balance)]
rules += [['add','from',ifc.address,'table',T]]
routes += main
routes += [['add','default','via',Ext[0].name],
[], lb_route, []]
output = [ 'route ' + ' '.join(R) if len(R) > 0 else '' for R in routes ] + \
[ 'rule ' + ' '.join(R) if len(R) > 0 else '' for R in rules ]
if file:
file.writelines([ R + '\n' for R in output ])
else:
for R in output:
print R
def addDebug(cls, conf):
"""add debug default outputer from the configuration
"""
if conf['engine'] == "debug":
outputer.append(output.debug())
def addDebug(cls, conf):
"""add debug default outputer from the configuration
"""
if conf['engine'] == "debug":
outputer.append(output.debug())
def loadStructure(data, expect, conf, namespace):
output.debug("Loading structure")
result = True
if type(expect) == dict and type(data) == dict:
output.debug("Recursive Loading")
for key in expect:
if key[0] != '$' and key in data:
return loadStructure(
data[key],
expect[key],
conf,
namespace + [key])
elif checkOperator(key) and key in data:
msg = str(data[key]) + key + '{gray} (from API){/gray}' + \
' ' + str(expect[key]) + ' (expect value)'
output.verifyNode(namespace, 'Comparing ' + key + ": " + msg)
res = compare(data[key], key, expect[key])
if res is False:
conf['errors'].append("Comparison failed: " + msg)
result = res
else:
output.invalidExpectedDataKey(key, conf)
return False
elif type(expect) == dict and type(data) != dict:
output.debug("Checking rules directly")
for key in expect:
if checkOperator(key):
msg = '({magenta}' + str(data) + '{/magenta} {green}' + \
key + '{/green} ' + '{magenta}' + \
str(expect[key]) + '{/magenta}) ->'
ops = []
if expect[key] == dict:
for item in expect[key].keys():
ops.append(getOperatorDesc(item) + ' ' +
str(expect[key][item]))
strops = ' {cyan}' + getOperatorDesc(key) + '{/cyan} be '
else:
ops.append(str(expect[key]))
strops = ' {cyan}' + getOperatorDesc(key) + '{/cyan} be '
msg = msg + ' {black}(From API){/black} ' + str(data) + \
' {cyan}' + 'value must be{/cyan} ' + \
strops.join(ops) + '{black} (expect value){/black} '
output.verifyNode(namespace, 'Comparing ' + msg)
res = compare(data, key, expect[key])
if res is False:
conf['errors'].append("Comparison failed: " + msg)
result = res
else:
output.invalidOperator(key, "the key was not found on " +
"server's response", conf)
return False
elif type(expect) != dict and type(data) != list:
output.debug("Straight comparison: " + str(expect) +
' == ' + str(data))
output.verifyNode(namespace, 'test')
result = (expect == data)
elif type(expect) != dict and type(data) != list and type(data) != dict:
output.debug("Straight comparison: " + str(expect) +
' == ' + str(data))
output.verifyNode(namespace, 'test')
result = (expect == data)
return result
def import_config( self, C ):
"""Create a NAT firewall using the settings from {C}.
Examples:
>>> V = Berlin()
>>> from network_config import Config
>>> C = Config()
>>> V.import_config(C)
"""
Ext = [ t for t in C.Interfaces if t.enabled and t.wan_interface ]
Int = [ t for t in C.Interfaces if t.enabled and not t.wan_interface ]
debug( 0, "Generating rules..." )
self.reset()
if len([ I for I in Ext if I.qos_bandwidth ]):
self.qos_chains(Ext)
self.append_chain('INPUT','#Loopback interface is valid' )
self.append_chain('INPUT','-i lo -j ACCEPT')
self.append_chain('INPUT','# All other internal traffic is limited to the interface' )
for I in Int:
for N in I.subnets:
self.append_chain('INPUT','-s {1} -i {0} -j ACCEPT'.format(I.name,N.net()))
self.append_chain('INPUT','# Remote interface, claiming to be local machines, IP spoofing, get lost!' )
for I in Int:
for N in I.subnets:
for E in Ext:
self.append_chain('INPUT','-s {1} -i {0} -j REJECT --reject-with icmp-port-unreachable'.format(
E.name, N.net() ))
self.append_chain('INPUT','# External interfaces, from any source, for ICMP traffic is valid.' )
for E in Ext:
self.append_chain('INPUT','-d {1}/32 -i {0} -p icmp -j ACCEPT'.format(
E.name, E.address ))
self.append_chain('INPUT','# Allow any related traffic coming back to the MASQ server in.' )
for E in Ext:
self.append_chain('INPUT','-d {1}/32 -i {0} -m state '
'--state RELATED,ESTABLISHED -j ACCEPT'.format(
E.name, E.address
))
self.append_chain('INPUT','# Allow internal DHCP/DNS traffic.' )
for I in Int:
self.append_chain('INPUT','-i {0} -p tcp -m tcp --sport 68 --dport 67 -j ACCEPT'.format(I.name))
self.append_chain('INPUT','-i {0} -p udp -m udp --sport 68 --dport 67 -j ACCEPT'.format(I.name))
self.append_chain('INPUT','-i {0} -p tcp -m tcp --dport 53 -j ACCEPT'.format(I.name))
self.append_chain('INPUT','-i {0} -p udp -m udp --dport 53 -j ACCEPT'.format(I.name))
self.append_chain('INPUT','# Internal appliances (not implemented)' )
for I in Int:
for N in I.subnets:
for P in N.services:
self.append_chain('INPUT','-i {0} -p tcp --dport {1} -s {2} -j ACCEPT'.format(
I.name, P, N.net() ))
self.append_chain('INPUT','# Remote services' )
for P in C.local_services:
for E in Ext:
self.append_chain('INPUT','-d {1}/32 -i {0} -p tcp -m state --state '
'NEW,RELATED,ESTABLISHED -m tcp --dport {2} -j ACCEPT'.format(
E.name, E.address, P
))
debug( -2, ' TCP port {0} is open for business.'.format(P) )
self.append_chain('INPUT','# Active FTP' )
self.append_chain('INPUT','-m helper --helper "ftp" -j ACCEPT')
self.append_chain('INPUT','# Reject everything else.' )
self.append_chain('INPUT','-j REJECT --reject-with icmp-port-unreachable')
self.append_chain('OUTPUT','# Workaround bug in netfilter' )
self.append_chain('OUTPUT','-p icmp -m state --state INVALID -j DROP')
self.append_chain('OUTPUT','# Loopback interface is valid' )
self.append_chain('OUTPUT','-o lo -j ACCEPT')
self.append_chain('OUTPUT','# Local interfaces, any source going to local net is valid' )
for E in Ext:
for I in Int:
for N in I.subnets:
self.append_chain('OUTPUT','-s {1}/32 -d {2} -o {0} -j ACCEPT'.format(
I.name, E.address, N.net() ))
self.append_chain('OUTPUT','# Local interface, MASQ server source going to a local net is valid' )
for I in Int:
for N in I.subnets:
self.append_chain('OUTPUT','-s {1}/32 -d {2} -o {0} -j ACCEPT'.format(
I.name, N.gw(), N.net() ))
self.append_chain('OUTPUT','# Outgoing to local net on remote interface, stuffed routing, deny' )
for E in Ext:
for I in Int:
for N in I.subnets:
self.append_chain('OUTPUT','-d {1} -o {0} -j REJECT --reject-with icmp-port-unreachable'.format(
E.name, N.net() ))
self.append_chain('OUTPUT','# Anything else outgoing on remote interface is valid.' )
for E in Ext:
self.append_chain('OUTPUT','-s {1}/32 -o {0} -j ACCEPT'.format(
E.name, E.address ))
self.append_chain('OUTPUT','# Internal interface, DHCP traffic accepted' )
for I in Int:
for N in I.subnets:
self.append_chain('OUTPUT','-s {1}/32 -d 255.255.255.255/32 '
'-o {0} -p tcp -m tcp '
'--sport 67 --dport 68 -j ACCEPT'.format(
I.name, N.gw() ))
self.append_chain('OUTPUT','-s {1}/32 -d 255.255.255.255/32 '
'-o {0} -p udp -m udp '
'--sport 67 --dport 68 -j ACCEPT'.format(
I.name, N.gw() ))
debug( -1, 'Catch all rule, all other outgoing is denied and logged.' )
self.append_chain('OUTPUT','-j REJECT --reject-with icmp-port-unreachable')
self.append_chain('FORWARD','# Reject all SMTP traffic save for a few trusted destinations' )
self.create_filter( 'smtp_filter', ["smtp-hosts"], table='filter',
action='-j ACCEPT')
self.append_chain( 'smtp_filter', '-j LOG --log-prefix " [SMTP] "' )
self.append_chain( 'smtp_filter', '-j REJECT --reject-with icmp-port-unreachable' )
for E in Ext:
for I in Int:
self.append_chain('FORWARD',' -p tcp -m tcp --dport 25 '
'-j LOG --log-prefix " [SMTP] "'.format(
E.name, I.name ))
self.append_chain('FORWARD',' -p tcp -m tcp --dport 25 '
'-j REJECT --reject-with icmp-port-unreachable'.format(
E.name, I.name ))
self.append_chain('FORWARD',' -p tcp -m tcp --dport 587 '
'-g smtp_filter'.format(
E.name, I.name ))
self.append_chain('FORWARD',' -p tcp -m tcp --dport 465 '
'-g smtp_filter'.format(
E.name, I.name ))
self.append_chain('FORWARD','-m state --state RELATED,ESTABLISHED -j ACCEPT')
self.append_chain('FORWARD','# Network services' )
for E in Ext:
for port,host in C.network_services:
self.append_chain(
'PREROUTING',
'-d {1}/32 -i {0} -p tcp -m tcp --dport {2} -j DNAT --to-destination {3}'.format(
E.name, E.address, port, host ),
table='nat')
self.append_chain('FORWARD','-i {0} -p tcp -m state --state NEW -m tcp --dport {1} -j ACCEPT'.format(
E.name, port ))
self.append_chain('PREROUTING','# Redirect traffic to wan address to this gateway',table='nat')
for E in Ext:
if E.wan_address == E.address: continue
for I in Int:
for N in I.subnets:
self.append_chain(
'PREROUTING',
'-p tcp -s {0} -d {1} -m state --state NEW,ESTABLISHED,RELATED '
'-j DNAT --to {2}'.format(
N.net(), E.wan_address, N.gw() ),
table='nat')
self.append_chain('FORWARD','# Accept solicited tcp packets' )
for E in Ext:
for I in Int:
self.append_chain('FORWARD','-i {0} -o {1} '
'-m state --state RELATED,ESTABLISHED -j ACCEPT'.format(
E.name, I.name ))
self.append_chain('FORWARD','# Allow packets within subnet' )
for I in Int:
self.append_chain('FORWARD','-i {0} -o {0} -j ACCEPT'.format(I.name))
self.append_chain('FORWARD','# Forward packets from the internal network to the Internet.' )
for I in Int:
for E in Ext:
self.append_chain('FORWARD','-i {0} -o {1} -j ACCEPT'.format(I.name,E.name))
self.append_chain('FORWARD','# Catch-all REJECT rule' )
self.append_chain('FORWARD','-j REJECT --reject-with icmp-port-unreachable')
self.append_chain('POSTROUTING','# IP-Masquerade',table='nat')
for E in Ext:
self.append_chain(
'POSTROUTING',
'-o {0} -j SNAT --to-source {1}'.format(E.name,E.address),
table='nat')
debug( 0, 'done.' )
debug( 0, '' )
debug( 0, 'Applying policies:' )
debug( -1, 'Creating "to Gateway" chain.' )
self.new_chain( 'to_gateway', table='nat', policy='-' )
self.append_chain('PREROUTING','# Local nat traffic is allowed',table='nat')
for I in Int:
for N in I.subnets:
self.append_chain(
'PREROUTING',
'-s {0} -d {0} -p tcp -m tcp --dport 80 -m state --state '
'NEW,RELATED,ESTABLISHED -j ACCEPT'.format(N.net()),
table='nat'
)
self.append_chain(
'to_gateway',
'-s {0} -j DNAT --to-destination {1}'.format( N.net(), N.gw() ),
table='nat'
)
self.append_chain('to_gateway', '-j LOG --log-prefix " [impossible]"', table='nat')
for I in Int:
for N in I.subnets:
if 'adblock' in N.policies:
self.adblock( N )
elif 'malware' in N.policies:
self.malware( N )
def create_qos_qdisc( C, file=False ):
"""Create a QoS qdisc
Create a qdisc (Queueing Discipline) capable of simple rule-based Quality of
Service.
"""
# The external interfaces
ifaces = [ I for I in C.Interfaces if I.wan_interface ]
if len(ifaces) == 0:
debug( 0, 'No external interfaces found' )
def k( x ):
"""Format a bandwidth of x kbit/s"""
return '{0}kbit'.format(int(x))
rules = []
# Remove tc rules for all interfaces
for I in C.Interfaces:
rules += [['qdisc','del','dev',I.name,'root']]
rules += [[]]
# Add a simple priority tree for each QoS-enabled external interface
for ifc in ifaces:
if not ifc.qos_bandwidth: continue
# The max available bandwidth
ceil = ifc.qos_bandwidth
# The smallest significant unit of bandwidth
unit = round(ceil/25)
ifn = ifc.name
rules += [
['qdisc','add','dev',ifn,'root', 'handle', '1:', 'htb','default','16'],
['class','add','dev',ifn,'parent','1:', 'classid','1:1', 'htb','rate',k( ceil),'ceil',k( ceil ) ],
['class','add','dev',ifn,'parent','1:1', 'classid','1:11','htb','rate',k(5*unit),'ceil',k(10*unit),'prio','0'],
['class','add','dev',ifn,'parent','1:1', 'classid','1:12','htb','rate',k(7*unit),'ceil',k( ceil ),'prio','1'],
['class','add','dev',ifn,'parent','1:1', 'classid','1:13','htb','rate',k(7*unit),'ceil',k( ceil ),'prio','2'],
['class','add','dev',ifn,'parent','1:1', 'classid','1:14','htb','rate',k(2*unit),'ceil',k( ceil ),'prio','3'],
['class','add','dev',ifn,'parent','1:1', 'classid','1:15','htb','rate',k(2*unit),'ceil',k( ceil ),'prio','3'],
['class','add','dev',ifn,'parent','1:1', 'classid','1:16','htb','rate',k(1*unit),'ceil',k( ceil ),'prio','4'],
['class','add','dev',ifn,'parent','1:1', 'classid','1:17','htb','rate',k(3*unit),'ceil',k( ceil ),'prio','4'],
['qdisc','add','dev',ifn,'parent','1:13','handle', '130:','sfq','perturb','10'],
['qdisc','add','dev',ifn,'parent','1:14','handle', '140:','sfq','perturb','10'],
['qdisc','add','dev',ifn,'parent','1:15','handle', '150:','sfq','perturb','10'],
['qdisc','add','dev',ifn,'parent','1:16','handle', '160:','sfq','perturb','10'],
['qdisc','add','dev',ifn,'parent','1:17','handle', '170:','sfq','perturb','10'],
['filter','add','dev',ifn,'parent','1:0','protocol','ip','prio','1','handle','1','fw','classid','1:11'],
['filter','add','dev',ifn,'parent','1:0','protocol','ip','prio','2','handle','2','fw','classid','1:12'],
['filter','add','dev',ifn,'parent','1:0','protocol','ip','prio','3','handle','3','fw','classid','1:13'],
['filter','add','dev',ifn,'parent','1:0','protocol','ip','prio','4','handle','4','fw','classid','1:14'],
['filter','add','dev',ifn,'parent','1:0','protocol','ip','prio','5','handle','5','fw','classid','1:15'],
['filter','add','dev',ifn,'parent','1:0','protocol','ip','prio','6','handle','6','fw','classid','1:16'],
['filter','add','dev',ifn,'parent','1:0','protocol','ip','prio','7','handle','7','fw','classid','1:17'],
['filter','add','dev',ifn,'parent','1:0','protocol','ip','prio','255','handle','255','fw','classid','1:2'],
[]
]
if len(rules) == 0:
debug( 0, 'No rules to implement; QoS disabled' )
return
if file:
file.writelines([ ' '.join(R) + '\n' for R in rules ])
else:
for R in rules:
print ' '.join(R)
