def __init__(self, nsp_name: str, ipdb: IPDB):
"""
Creats a namespace for a specific vlan_iface
:param nsp_name:
:param vlan_iface_name:
:param ipdb: IPDB is a transactional database, containing records, representing network stack objects.
Any change in the database is not reflected immidiately in OS, but waits until commit() is called.
"""
Logger().debug("Create Namespace ...", 2)
self.nsp_name = nsp_name
self.id = id
self.vlan_iface_name = ""
self.vlan_iface_ip = "0.0.0.0"
self.ipdb = ipdb
self.ipdb_netns = None
try:
self.ipdb_netns = IPDB(nl=NetNS(nsp_name))
netns.setns(nsp_name)
self.ipdb_netns.interfaces['lo'].up().commit()
Logger().debug("[+] Namespace(" + nsp_name + ") successfully created", 3)
# self.encapsulate_interface()
except Exception as e:
Logger().debug("[-] Couldn't create Namespace(" + nsp_name + ")", 3)
for tb in traceback.format_tb(sys.exc_info()[2]):
Logger().error(tb, 3)
Logger().error(str(e), 3)
self.remove()
def import_firmwares(self, release_model: str):
"""
Imports the stored Firmwares, so the firmware_handler can use them.
:param release_model: stable, beta, experimental
"""
path = self.FIRMWARE_PATH + '/' + release_model + '/' + self.UPDATE_TYPE + '/'
Logger().debug("Import Firmwares from '" + path + "'", 2)
count = 0
try:
files = os.listdir(path)
except Exception:
Logger().debug(
"No Firmwares available for download at path '" + path + "'",
3)
return
for firmware_name in files:
try:
freifunk_verein = firmware_name.split('-')[1]
firmware_version = firmware_name.split('-')[2]
file = path + firmware_name
url = self.url + '/' + release_model + '/' + self.UPDATE_TYPE + '/' + firmware_name
self.firmwares.append(
Firmware(firmware_name, firmware_version, freifunk_verein,
release_model, file, url))
count += 1
except Exception:
Logger().warning("[-] Couldn't import " + firmware_name, 3)
continue
Logger().debug(str(count) + " Firmwares imported", 3)
def run(self):
Logger().info("Start WebServer on port " + str(WebServer.PORT_WEBSERVER) + " ...", 1)
try:
self.httpd.serve_forever()
except Exception as e:
Logger().debug("[-] WebServer couldn't get started", 2)
raise e
def _verified_download(self, firmware: Firmware, hash_firmware: str,
max_attempts: int) -> bool:
"""
Downloads the given Firmware and checks if the download was correct and if the hash is matching
:param firmware:
:param hash_firmware:
:param max_attempts: max number of attemps to download a firmware
:return: bool
"""
valid_download = False
count = 0
while (not valid_download) & (count < max_attempts):
valid_download = self._download_file(firmware.url, firmware.file,
firmware.release_model)
if valid_download:
valid_download = firmware.check_hash(hash_firmware)
count += 1
if count >= max_attempts:
Logger().debug(
"[-] The Firmware(" + firmware.name +
") couldn't be downloaded", 3)
return False
else:
Logger().debug(
"[+] The Firmware(" + firmware.name +
") was successfully downloaded", 3)
return True
def send_data(self, local_file: str, remote_file: str):
"""
Sends Data via sftp to the RemoteSystem
:param local_file: Path to the local file
:param remote_file: Path on the Router, where the file should be saved
"""
try:
# TODO: If sftp is installed on the Router
'''
sftp = self.ssh.open_sftp()
sftp.put(local_file, remote_file)
sftp.close()
'''
command = 'sshpass -p' + str(self.remote_system.usr_password) + ' scp ' + local_file + ' ' + \
str(self.remote_system.usr_name) + '@' + str(self.remote_system.ip) + ':' + remote_file
os.system(command)
# TODO: Paramiko_scp have to installed
'''
scp = SCPClient(self.ssh.get_transport())
scp.put(local_file, remote_file)
'''
Logger().debug("[+] Sent data '" + local_file + "' to RemoteSystem '" +
str(self.remote_system.usr_name) + "@" + str(self.remote_system.ip) +
":" + remote_file + "'", 2)
except Exception as e:
Logger().error("[-] Couldn't send '" + local_file + "' to RemoteSystem '" +
str(self.remote_system.usr_name) + "@" + str(self.remote_system.ip) +
":" + remote_file + "'", 2)
Logger().error(str(e), 2)
def get_router_manual_list() -> []:
"""
Read the Router Manual Config file
:return: List with any Router objects from the file
"""
output = ConfigManager.get_router_manual_config()
router_list = []
for i in range(0, len(output)):
router_info = output[i]
if not len(router_info) == 9:
Logger().error(
"List must be length of 9 but has a length of {0}".format(
len(output)))
return
try:
v = Router(i, router_info['Name'], router_info['Id'],
router_info['IP'], router_info['IP_Mask'],
router_info['CONFIG_IP'],
router_info['CONFIG_IP_MASK'],
router_info['Username'], router_info['Password'],
router_info['PowerSocket'])
router_list.append(v)
except Exception as ex:
Logger().error(
"Error at building the list of Router's\nError: {0}".
format(ex))
return router_list
def create_interface(self,
vlan_iface_ip: str = None,
vlan_iface_ip_mask: int = None):
"""
Creats a virtual interface on a existing interface (like eth0)
:param vlan_iface_ip: ip of the virtual interface
:param vlan_iface_ip_mask: network-mask of the virtual interface
"""
Logger().debug("Create VLAN Interface ...", 2)
try:
link_iface = self.ipdb.interfaces[self.link_iface_name]
with self.ipdb.create(kind="vlan",
ifname=self.vlan_iface_name,
link=link_iface,
vlan_id=self.vlan_iface_id).commit() as i:
if vlan_iface_ip:
i.add_ip(vlan_iface_ip, vlan_iface_ip_mask)
i.mtu = 1400
if not vlan_iface_ip:
self._wait_for_ip_assignment()
vlan_iface_ip = self._get_ipv4_from_dictionary(
self.ipdb.interfaces[self.vlan_iface_name])
Logger().debug(
"[+] " + self.vlan_iface_name + " created with: Link=" +
self.link_iface_name + ", VLAN_ID=" + str(self.vlan_iface_id) +
", IP=" + vlan_iface_ip, 3)
except Exception as e:
Logger().debug(
"[-] " + self.vlan_iface_name + " couldn't be created", 3)
Logger().error(str(e), 3)
def test_warning(self):
"""
Tests warning
:return: Tests results
"""
Logger().warning("Warning from Logger() {0}".format(Logger()))
Logger().close()
self.assertEqual(True, True)
def test_log(self):
"""
Tests critical
:return: Tests results
"""
Logger().log(20, "Log from Logger() {0}".format(Logger()))
Logger().close()
self.assertEqual(True, True)
def test_error(self):
"""
Tests error
:return: Tests results
"""
Logger().error("Error from Logger() {0}".format(Logger()))
Logger().close()
self.assertEqual(True, True)
def test_debug(self):
"""
Tests debug
:return: Tests results
"""
Logger().debug("Debug from Logger() {0}".format(Logger()))
Logger().close()
self.assertEqual(True, True)
def test_info(self):
"""
Tests info
:return: Tests results
"""
Logger().info("Hello from logger: {0}".format(Logger()))
Logger().close()
self.assertEqual(True, True)
def test_log_level_tab(self):
"""
Test log level tab
:return: Test results
"""
tabs = Logger().get_log_level_tab(2)
Logger().close()
self.assertEqual(True, tabs == "\t\t")
def join(self):
Logger().info("Stop WebServer ...", 1)
time.sleep(2)
try:
self.httpd.shutdown()
Logger().debug("[+] WebServer successfully stoped", 2)
except Exception as e:
Logger().debug("[-] WebServer couldn't stoped", 2)
Logger().error(str(e), 1)
def test_logger(self):
"""
Tests logger
:return: Tests results
"""
result = Logger().logger
self.assertEqual(True, Logger().logger is result)
Logger().close()
def test_is_loaded(self):
"""
Tests is loaded
:return: Tests results
"""
Logger().setup()
result = Logger().is_loaded
Logger().close()
self.assertEqual(True, result)
def get_router_auto_list(count: int = 0) -> []:
"""
Read the Router Manual Config file
:param count: Count of the Router
:return: List with any Router objects from the file
"""
output = ConfigManager.get_router_auto_config()
if not len(output) == 10:
Logger().error(
"List must be length of 10 but has a length of {0}".format(
len(output)))
return
try:
router_count = output[0]
name = output[1]
identifier = output[2]
ip = output[3]
ip_mask = output[4]
config_ip = output[5]
config_ip_mask = output[6]
username = output[7]
password = output[8]
power_socket = output[9]
i = identifier['default_Start_Id']
socket_id = power_socket['powerSocket_Start_Id']
router_list = []
if count <= 0:
count = router_count['router_Count']
else:
count = count
for x in range(0, count):
v = Router(x, name['default_Name'] + "{0}".format(i), i,
ip['default_IP'], ip_mask['default_IP_Mask'],
config_ip['default_CONFIG_IP'],
config_ip_mask['default_CONFIG_IP_MASK'],
username['default_Username'],
password['default_Password'], socket_id)
router_list.append(v)
i += 1
socket_id += 1
return router_list
except Exception as ex:
Logger().error(
"Error at building the list of Router's\nError: {0}".format(
ex))
def _create_path(self, path: str):
"""
Creates directories if necessary
:param path:
"""
try:
Logger().debug("Create path " + path + " ...", 3)
os.makedirs(path)
Logger().debug("[+] Path successfully created", 4)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
Logger().debug("[+] Path allready exists", 4)
def down(self, port_id):
"""
Turns off power on port
:param port_id: ID of power strip port
:return: bool for failure or success
"""
cmd = "echo 0 > /dev/output" + str(port_id)
self.__command(cmd)
if self.port_status(port_id) == 0:
Logger().info("Power turned off: port " + str(port_id))
return True
else:
Logger().error("Could not turn off power on port " + str(port_id))
return False
def _parse_router_model(self, router_model: str):
"""
Transform a str like "TP-LINK TL-WR841N/ND v9" into "tp-link-tl-wr841n-nd-v9"
:param router_model:
:return: router_model_name and router_model_version
"""
Logger().debug("Parse RouterModel ...", 2)
tmp = router_model.split(' v')
router_model_name = tmp[0]
router_model_version = 'v'+tmp[1]
router_model_name = router_model_name.lower()
# replace all symbols with a minus
router_model_name = re.sub(r'(?:[^\w])', '-', router_model_name)
Logger().debug("'" + router_model + "' into '" + router_model_name + " " + router_model_version + "'", 3)
return [router_model_name, router_model_version]
def send_command(self, command) -> str:
"""
Sends the given command via SSH to the RemoteSystem.
:param command: like "ping 8.8.8.8"
:return: The output of the command given by the RemoteSystem
"""
try:
stdin, stdout, stderr = self.ssh.exec_command(command)
output = stdout.readlines()
Logger().debug("[+] Sent the command (" + command + ") to the RemoteSystem", 2)
return str(output)
except Exception as e:
Logger().error("[-] Couldn't send the command (" + command + ")", 2)
raise e
def connect_with_remote_system(self):
"""
Connects to the remote_system via SSH(Paramiko).
Ignores a missing signatur.
"""
Logger().info("Connect with RemoteSystem ...", 1)
try:
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(str(self.remote_system.ip), port=22,
username=str(self.remote_system.usr_name),
password=str(self.remote_system.usr_password))
Logger().debug("[+] Successfully connected", 2)
except Exception as e:
Logger().error("[-] Couldn't connect", 2)
Logger().error("" + str(e), 1)
def __init__(self, meta, request_body, logger=None):
logger = logger if logger else Logger(log_level="info", vendor_key=-1, retailer_key=-1, module_name="MobileNotifierNanny")
__debug = request_body.get("debug", 'N')
__log_level = 'DEBUG' if str(__debug).upper() in ('Y', 'YES', 'TRUE', 'T') else 'INFO'
logger.set_level(log_level=__log_level)
MobileNotifier.__init__(self, meta={**meta}, params={**request_body}, logger=logger)
def check_hash(self, excep_hash: str) -> bool:
"""
Checks whether the excepted Hash equals the actual Hash of the Firmware.
:param excep_hash:
:return:
"""
Logger().debug("Check Hash of the Firmware(" + self.name + ") ...", 3)
self.calc_hash()
if self.hash == excep_hash:
Logger().debug("[+] The Hash is correct", 4)
return True
Logger().debug("[-] The Hash is incorrect", 4)
Logger().debug("Hash of the Firmware: " + self.hash, 4)
Logger().debug("Excepted Hash: " + excep_hash, 4)
return False
def test_create_namespace(self):
Logger().debug("TestNamespace: test_create_namespace ...")
# Create VLAN
vlan = Vlan('eth0', 'vlan1', 10, '192.168.1.10', 24)
assert isinstance(vlan, Vlan)
vlan.create_interface("192.168.1.11", 24)
# Create Namespace
namespace = Namespace('nsp1', vlan.ipdb)
assert isinstance(namespace, Namespace)
# encapsulate VLAN
namespace.encapsulate_interface(vlan.vlan_iface_name)
# Test if the namespace now exists
process = Popen(["ip", "netns"], stdout=PIPE, stderr=PIPE)
stdout, sterr = process.communicate()
assert sterr.decode('utf-8') == ""
assert namespace.nsp_name in stdout.decode('utf-8')
# Remove the Namespace
namespace.remove()
process = Popen(["ip", "netns"], stdout=PIPE, stderr=PIPE)
stdout, sterr = process.communicate()
assert stdout.decode('utf-8') == ""
def __init__(self, meta, request_body, logger=None):
logger = logger if logger else Logger(
log_level="info",
vendor_key=-1,
retailer_key=-1,
module_name="dataCompleteAGNanny")
logger.set_keys(log_id="{}_{}".format(request_body["jobId"],
request_body["stepId"]))
logger.debug(request_body)
request_body["header"] = {
"Content-Type": "application/json",
"hour_offset": "-6",
"module_name": logger.format_dict["module_name"]
}
request_body["log_level"] = request_body.get("log_level", 20)
logger.set_level(request_body["log_level"])
DataCompleteAG.__init__(self, {
**request_body,
**{
"meta": meta
},
**{
"logger": logger
}
})
def svr_xml_converter(raw_data):
'''
This method converts the supplied xml file-object to a python dictionary.
@raw_data, generally a file (or json string) containing the raw dataset(s),
to be used when computing a corresponding model. If this argument is a
file, it needs to be closed.
@list_observation_label, is a list containing dependent variable
labels.
'''
feature_count = None
list_dataset = []
list_observation_label = []
logger = Logger(__name__, 'error', 'error')
# convert xml file to python 'dict'
dataset = xmltodict.parse(raw_data)
# build 'list_dataset'
for observation in dataset['dataset']['observation']:
for key in observation:
if key == 'criterion':
observation_label = observation['criterion']
list_observation_label.append(observation[key])
elif key == 'predictor':
for predictor in observation[key]:
predictor_label = predictor['label']
predictor_value = predictor['value']
validate_value = Validate_Dataset(predictor_value)
validate_value.validate_value()
list_error_value = validate_value.get_errors()
if list_error_value:
logger.log(list_error_value)
return None
else:
list_dataset.append({
'dep_variable_label':
str(observation_label),
'indep_variable_label':
str(predictor_label),
'indep_variable_value':
predictor_value
})
# generalized feature count in an observation
if not feature_count:
feature_count = len(observation['predictor'])
# save observation labels, and return
raw_data.close()
return {
'dataset': list_dataset,
'observation_labels': list_observation_label,
'feature_count': feature_count
}
def __init__(self, meta, params, logger=None):
self.meta = meta
self.params = params
self.file_ext = self.params.get("fileExt") if self.params.get(
"fileExt") else "txt"
self.zip_flag = True if str(self.params.get(
"zipFlag", "N")).upper() in ["Y", "YES", "T", "TRUE"] else False
self._debug = "Y" if str(self.params.get(
'debug', 'N')).upper() in ["Y", "YES", "T", "TRUE"] else 'N'
__log_level = 'DEBUG' if self._debug == "Y" else 'INFO'
self.logger = logger if logger else Logger(log_level="info",
target="console",
vendor_key=-1,
retailer_key=-1,
sql_conn=None)
self.logger.set_level(log_level=__log_level)
self.logger.set_keys(log_id="{}".format(self.params["jobId"]))
self.provision_url = self.meta[
"api_provision_str"] + "/filesDelivery/layout/attributes?cycle_key="
self.capacity = Capacity(meta=self.meta)
self.rsi_username = self.meta.get("fileshare_username")
self.rsi_folder = self.meta.get("fileshare_folder")
self.cmn_schema = self.meta.get("db_conn_vertica_common_schema",
"common")
self.app_conn = MSOperation(meta=self.meta)
self.vertica_conn = DWOperation(meta=self.meta)
self.dumper_context = self.get_context()
self._layout_payload = {}
def main():
args = parser()
cfg = Config.fromfile(args.config)
log = Logger('./cache/log/' + args.net + '_trainlog.txt', level='info')
log.logger.info('Preparing data')
train_loader , val_loader = dataLoad(cfg)
start_epoch = 0
if args.pretrain:
log.logger.info('Loading Pretrain Data')
net = get_network(args).cuda()
model_params(net, log)
criterion = CrossEntropy().cuda()
# criterion = MeanSquaredError().cuda()
optimizer = optim.SGD(net.parameters(), lr=cfg.PARA.train.LR, momentum=cfg.PARA.train.momentum, weight_decay=cfg.PARA.train.wd)
net = torch.nn.DataParallel(net, device_ids=cfg.PARA.train.device_ids)
torch.backends.cudnn.benchmark = True
if args.resume:
log.logger.info('Resuming from checkpoint')
weighted_file = os.path.join('./cache/checkpoint/'+args.net, args.epoch + 'ckpt.pth')
checkpoint = torch.load(weighted_file)
net.load_state_dict(checkpoint['net'])
start_epoch = checkpoint['epoch']
train(start_epoch, train_loader, val_loader, cfg, net, criterion, optimizer, args, log)
log.logger.info("Training Finished, Total EPOCH=%d" % cfg.PARA.train.EPOCH)
def __init__(self, remote_system: RemoteSystem, link_iface_name='eth0'):
"""
Creats a VLAN and a Namespace for the specific RemoteSystem(Router,PowerStrip) and 'eth0' as the link-interface.
The VLAN will be encapsulate in the Namespace.
Also the a SSHClient will be created.
:param remote_system: Could e a Router or a powerstrip object
"""
Logger().info("Create Network Controller ...", 1)
self.remote_system = remote_system
# TODO: ausgelagert in NVAssisten. soll beides aber in Zukunft gelöscht/ausgelagert werden
'''
self.vlan = Vlan(link_iface_name, router.vlan_iface_name, router.vlan_iface_id,
vlan_iface_ip=None, vlan_iface_ip_mask=None)
self.vlan.create_interface()
self.namespace = Namespace(self.router.namespace_name, self.vlan.ipdb)
self.namespace.encapsulate_interface(self.vlan.vlan_iface_name)
'''
self.nv_assisten = NVAssistent()
self.nv_assisten.create_namespace_vlan(str(self.remote_system.namespace_name), link_iface_name,
str(self.remote_system.vlan_iface_name),
int(self.remote_system.vlan_iface_id))
self.ssh = paramiko.SSHClient()
