def getTraining():
# Import needed modules.
import pandas as pd
import scipy.io as io
from subprocess import check_output
from GetData import getData
from random import choice
dirs = ['Dog_1', 'Dog_2', 'Dog_3', 'Dog_4', 'Patient_1', 'Patient_2', 'Patient_3', 'Patient_4', 'Patient_5', 'Patient_6', 'Patient_7', 'Patient_8']
data = {}
for x in dirs:
print 'Getting ' + x + '...'
interfiles = check_output('find ' + x + ' -name *_interictal*', shell=True)
interfiles = interfiles.split('\n')
interfiles.pop()
ictfiles = check_output('find ' + x + ' -name *_ictal*', shell=True)
ictfiles = ictfiles.split('\n')
ictfiles.pop()
tempinter = getData(x, 'interictal', 1, len(interfiles))
tempict = getData(x, 'ictal', 1, len(ictfiles))
tempict2 = []
count = 0
ictstart = 0
ictbounds = []
for y in tempict:
if(y[1] is 1 and ictstart == 0):
print '1'
ictbounds.append([count,0])
ictstart = 1
elif(y[1] is 0):
print '2'
ictstart = 0
count += 1
print 'ictstart = ' + str(ictstart) + ' ictbounds = ' + str(len(ictbounds)) + ' count = ' + str(count)
for y in range(len(ictbounds)-1):
ictbounds[y][1] = ictbounds[y+1][0]
ictbounds[len(ictbounds)-1][1] = len(tempict)
for y in ictbounds:
tempict2.append(tempict[y[0]:y[1]])
data[x] = [tempinter, tempict2]
reserve = len(dirs)/3
reserved = {} # Holds the reserved set. Not sure what to do with these...
for x in range(reserve):
temp = choice(data.keys())
reserved[temp] = data[temp]
data.pop(temp)
return data
def getRouterIDList(RouterIDNetwork):
RouterIDList = [0]
IDs = ipaddress.ip_network(Network,strict=False)
data = getData('variable.json')
Number_of_instances = data["number_of_instances"]
for i in IDs.hosts():
if Number_of_instances > 0:
RouterIDList.append(i)
data = getData('variable.json')
Number_of_instances = data["number_of_instances"]
Number_of_instances = Number_of_instances -1
else:
break
return RouterIDList
def getPeerDetails(Router):
data = getData('variable.json')
Number_of_instances = data["number_of_instances"]
PeerAS = []
start = 'DUT1'
end = 'DUT%d' %(Number_of_instances)
data = getData('ProtocolSpecific.json')
topodata = getData('topo.json')
PeerDetails = {}
data['BGP_Parameters'][0]['DUT1'].update({"PeerDetails" : []})
if DUT == start:
DUT = 'DUT2'
PeerASNum = data["BGP_Parameters"][0][DUT]["ASNum"]
PeerAddress = topodata["Device_details"][0][DUT]["IP_address"]
PeerInterface = topodata["Device_details"][0][DUT]["Interface"]
PeerDetails = {"Peer1":{"ASNum" : PeerASNum, "IP_Address" : PeerAddress, "Interface" : PeerInterface}}
data['BGP_Parameters'][0][DUT]['PeerDetails'].append(PeerDetails)
putData(data,'ProtocolSpecific.json')
elif DUT == end:
DUT = 'DUT%d' % (Number_of_instances-1)
PeerASNum = data["BGP_Parameters"][0][DUT]["ASNum"]
PeerAddress = topodata["Device_details"][0][DUT]["IP_address"]
PeerInterface = topodata["Device_details"][0][DUT]["Interface"]
PeerDetails = {"Peer1":{"ASNum" : PeerASNum, "IP_Address" : PeerAddress, "Interface" : PeerInterface}}
data['BGP_Parameters'][0][DUT]['PeerDetails'].append(PeerDetails)
putData(data,'ProtocolSpecific.json')
else:
devno = DUT[3:]
Peers = ['DUT%d','DUT%d'] %(devno-1,devno+1)
count = 1
dictionary = {}
for i in Peers:
PeerASNum = data["BGP_Parameters"][0][i]["ASNum"]
PeerAddress = topodata["Device_details"][0][i]["IP_address"]
PeerInterface = topodata["Device_details"][0][i]["Interface"]
dictionary = {"Peer"+str(count) : {"ASNum" : PeerASNum, "IP_Address" : PeerAddress, "Interface" : PeerInterface}}
count = count+1
PeerDetails.update(dictionary)
data['BGP_Parameters'][0][DUT]['PeerDetails'].append(PeerDetails)
putData(data,'ProtocolSpecific.json')
return
def checkBGPNeighbors(self, Router): #Check if BGP neighbors are learnt
data = getData('ProtocolSpecific.json')
if (isinstance(Router, list)):
for i in Router:
logToFile.info("Checking if BGP neighbors are set in %s", i)
RouterInst = switchToRouter(self.device, i)
RouterInst = BGPConfiguration.checkAllBGPNeighbors(RouterInst)
RouterInst = BGPConfiguration.checkIPV4Route(RouterInst)
RouterInst.sendline('exit')
logToFile.info("Verification of BGP Neighbors done in %s", i)
elif Router == 'all':
for i in data['BGP_Parameters'].keys():
logToFile.info("Checking if BGP neighbors are set in %s", i)
RouterInst = BGPConfiguration.switchToRouter(self.device, i)
RouterInst = BGPConfiguration.checkAllBGPNeighbors(RouterInst)
RouterInst = BGPConfiguration.checkIPV4Route(RouterInst)
#RouterInst = BGPConfiguration.checkBGPRoute(RouterInst)
RouterInst.sendline('exit')
logToFile.info("Verification of BGP neighbors done in %s", i)
return
def censusSoftMax():
from GetData import getData
X_train, X_val, X_test, y_train, y_val, y_test = getData()
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.optimizers import SGD
import keras
# Generate dummy data
y_train = keras.utils.to_categorical(y_train, num_classes=2)
y_test = keras.utils.to_categorical(y_train, num_classes=2)
model = Sequential()
# Dense(64) is a fully-connected layer with 64 hidden units.
# in the first layer, you must specify the expected input data shape:
# here, 20-dimensional vectors.
model.add(Dense(64, activation='relu', input_dim=103))
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(2, activation='softmax'))
# 动量batch下降,lr学习率,decay:每轮学习后学习率的衰减,nesterov使用牛顿动量
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
model.fit(X_train, y_train)
# epochs=20,
# batch_size=128)
scores = model.evaluate(X_train, y_train, batch_size=128)
print('Test loss:', scores[0])
print('Test accuracy:', scores[1])
def pingTest(self): #Ping Test to verify IP address configuration
topodata = getData('Topology.json')
Routers = topodata['Device_details'].keys()
Routers.sort()
for Router in Routers:
if Router == "Router1":
RouterInst = basicConfiguration.switchToRouter(self.device,Router)
for i in range(1,self.NumberOfDevices):
NDUT = 'Router' + str(i+1)
Link = "Link_Router1_"+ NDUT
PeerInterface = topodata["Link_details"][Link][NDUT]
IPAddress = topodata["Device_details"][NDUT][PeerInterface]
basicConfiguration.ping(RouterInst,IPAddress)
else:
RouterInst = basicConfiguration.switchToRouter(self.device,Router)
NDUT = 'Router1'
Link = "Link_Router1_" + Router
PeerInterface = topodata["Link_details"][Link][NDUT]
IPAddress = topodata["Device_details"][NDUT][PeerInterface]
basicConfiguration.ping(RouterInst,IPAddress)
logToFile.info('Ping Test successful on all the devices')
return
def enableBGPGlobal(self, Device):
data = getData('ProtocolSpecific.json')
GlobalAS = self.data["globalAS"]
RouterID = data['BGP_Parameters'][Device]['RouterID']
RouterInst = BGP.switchToRouter(self.child, Device)
RouterInst = BGP.BGPglobal(RouterInst, GlobalAS, RouterID)
return RouterInst
def enableBGPGlobal(self, Device): #Enables BGP global
logToFile.info(" Enabling BGP Global")
data = getData('ProtocolSpecific.json')
RouterID = data['BGP_Parameters'][Device]['RouterID']
AS = data['BGP_Parameters'][Device]['ASNum']
RouterInst = BGPConfiguration.switchToRouter(self.device, Device)
RouterInst = BGPConfiguration.BGPglobal(RouterInst, AS, RouterID)
return RouterInst
def getRouterIDList():
#Gets RouterId for each created node from the given routerID Network
data = getData('variable.json')
NumberOfDevices = data["NumberOfDevices"]
RouterIDList = []
for i in range(1, (NumberOfDevices + 1)):
ID = "%s.%s.%s.%s" % (i, i, i, i)
RouterIDList.append(ID)
return RouterIDList
def setPeerDetails(
Router): #Sets the Peer details into the ProtocolSpecific JSON File
vardata = getData('variable.json')
NumberOfDevices = vardata["NumberOfDevices"]
start = 'Router1'
data = getData('ProtocolSpecific.json')
topodata = getData('Topology.json')
data["BGP_Parameters"][Router].update({"PeerDetails": {}})
if Router == start:
for i in range(1, NumberOfDevices):
NDUT = 'Router' + str(i + 1)
Link = "Link_Router1_" + NDUT
PeerASNum = data["BGP_Parameters"][NDUT]["ASNum"]
PeerInterface = topodata["Link_details"][Link][NDUT]
PeerAddress = topodata["Device_details"][NDUT][PeerInterface]
Peer = "Peer" + str(i)
data["BGP_Parameters"][Router]['PeerDetails'].update({
Peer: {
"ASNum": PeerASNum,
"IP_Address": PeerAddress,
"Interface": PeerInterface
}
})
putData(data, 'ProtocolSpecific.json')
else:
NDUT = 'Router1'
Link = "Link_Router1_" + Router
PeerInterface = topodata["Link_details"][Link][NDUT]
PeerAddress = topodata["Device_details"][NDUT][PeerInterface]
PeerASNum = data["BGP_Parameters"][Router]["ASNum"]
data["BGP_Parameters"][Router]['PeerDetails'].update({
"Peer1": {
"ASNum": PeerASNum,
"IP_Address": PeerAddress,
"Interface": PeerInterface
}
})
putData(data, 'ProtocolSpecific.json')
return data
def getASNumList(AS_Start):
ASNumList = [0]
data = getData('variable.json')
Number_of_instances = data["number_of_instances"]
#count = Number_of_instances // 2
for i in range(1,(Number_of_instances+1)):
# if i <= count:
ASNumList.append(AS_Start)
# else :
# ASNumList.append(AS_Start + 1)
return ASNumList
def main():
Data = getData()
AddDate(Data)
AddUserName(Data)
move_list = DeleteMoveListException(GetAllMove_FasterVersion(Data))
day_m = AggMoveListByTime(move_list, way='day')
week_m = AggMoveListByTime(move_list, way='week')
month_m = AggMoveListByTime(move_list, way='month')
Save_Obj(day_m, 'day_move')
Save_Obj(week_m, 'week_move')
Save_Obj(month_m, 'month_move')
def getIPaddList(Network):
IPaddList = [0]
Hosts_IP = ipaddress.ip_network(Network,strict = False)
data = getData('variable.json')
Number_of_instances = data["number_of_instances"]
for i in Hosts_IP.hosts():
if Number_of_instances > 0:
#print i
IPaddList.append(str(i))
Number_of_instances = Number_of_instances -1
else:
break
return IPaddList
def createBGPNeighbor(self, Router): #Creates BGP Neighbor
data = getData('ProtocolSpecific.json')
if (isinstance(Router, list)):
for i in Router:
logToFile.info("Configuring BGP in %s", i)
RouterInst = self.enableBGPGlobal(i)
Peers = data["BGP_Parameters"][i]['PeerDetails']
logToFile.info(" Setting up the neighbors")
for peer in Peers.keys():
PeerAS = data["BGP_Parameters"][i]['PeerDetails'][peer][
'ASNum']
PeerAddress = data["BGP_Parameters"][i]['PeerDetails'][
peer]['IP_Address']
PeerInterface = data["BGP_Parameters"][i]['PeerDetails'][
peer]['Interface']
RouterInst = BGPConfiguration.createBGPV4Neighbor(
RouterInst, PeerAS, PeerAddress, PeerInterface)
RouterInst.sendline('exit')
logToFile.info("BGP configuration donein %s", i)
elif Router == 'all':
Routers = data['BGP_Parameters'].keys()
Routers.sort()
for Router in Routers:
logToFile.info("Configuring BGP in %s", Router)
self.device.expect(['/w+@.*/#', pexpect.EOF, pexpect.TIMEOUT],
timeout=3)
RouterInst = self.enableBGPGlobal(str(Router))
Peers = data["BGP_Parameters"][str(Router)]['PeerDetails']
logToFile.info(" Setting up the neighbors")
for peer in Peers.keys():
PeerAS = str(data["BGP_Parameters"][str(Router)]
['PeerDetails'][str(peer)]['ASNum'])
PeerAddress = str(data["BGP_Parameters"][str(Router)]
['PeerDetails'][str(peer)]['IP_Address'])
PeerInterface = str(data["BGP_Parameters"][str(
Router)]['PeerDetails'][str(peer)]['Interface'])
NeighborAddress = PeerAddress.split('/')
RouterInst = BGPConfiguration.createBGPV4Neighbor(
RouterInst, PeerAS, NeighborAddress[0], PeerInterface)
RouterInst.sendline('exit')
logToFile.info("BGP configuration done in %s", Router)
return
def boot(device): #Boots network devices, Start the flexswitch services
data = getData('variable.json')
NumberOfDevices = data["NumberOfDevices"]
for i in range(0,NumberOfDevices):
Router = "Router"+str(i+1)
#RouterInst = switchToRouter(device,Router)
device.expect(['/w+@.*/#',pexpect.EOF,pexpect.TIMEOUT],timeout=1)
cmd = "sudo docker exec %s sh -c \"\/etc\/init\.d\/flexswitch restart\"" %(Router)
device.sendline(cmd)
time.sleep(8)
device.expect(['/w+@.*/#',pexpect.EOF,pexpect.TIMEOUT],timeout=1)
logger.info(device.before,also_console=True)
return
def preSetup(image):
data = getData('variable.json') #Get the json data from file "variable.json"
NumberOfDevices = data["NumberOfDevices"]
device = pexpect.spawn("/bin/bash") #Spawn the process ("/bin/bash")- Terminal
logToFile.info(" Pulling the flexswitch base image")
device.expect(['/$',pexpect.EOF,pexpect.TIMEOUT],timeout=1)
cmd = "sudo docker pull %s" %(image)
device.sendline(cmd) #Pull the flexswitch image from docker hub
device.expect(['/w+@.*/#',pexpect.EOF,pexpect.TIMEOUT],timeout=2)
device.expect(['/$',pexpect.EOF,pexpect.TIMEOUT],timeout=2)
logToFile.info(" Creating directory for storing network namespace")
device.sendline("mkdir -m 777 -p /var/run/netns") #Create a directory
device.expect(['netns',pexpect.EOF,pexpect.TIMEOUT],timeout=2)
logger.info(device.before,also_console=True)
return device
def getASNumList(AS): #Gets the AS Num for each created node
ASNumList = []
data = getData('variable.json')
NumberOfDevices = data["NumberOfDevices"]
BGPType = data["BGPType"]
if BGPType == "IBGP":
for i in range(0, NumberOfDevices):
ASNumList.append(AS)
else:
ASNumList.append(AS)
for i in range(1, NumberOfDevices):
ASNumList.append(AS + 1)
return ASNumList
def configureIPaddress(self):
data = getData('Topology.json')
Devices = data['Device_details']
for Device in Devices.keys():
print "*** Device_Name = %s ***" % (Device)
print "*** Setting up initial configurations in docker instance -- %s ***" % (
Device)
RouterInst = basic_conf.switchToRouter(self.child, Device)
RouterInst = basic_conf.preliminaryInstalls(RouterInst)
RouterInst = basic_conf.configureIP(
RouterInst, data['Device_details'][Device]['Interface'],
data['Device_details'][Device]['IP_address'])
RouterInst.sendline('exit')
self.child.expect(['/w+@.*/#', pexpect.EOF, pexpect.TIMEOUT],
timeout=10)
print self.child.before
print "*** IP address for %s interface of device %s is set ***" % (
data['Device_details'][Device]['Interface'], Device)
print "***** Initial Setup to start with flexswitch is ready *****"
return
def configureIPaddress(self): #Configures IP address
data = getData('Topology.json')
Devices = data['Device_details']
Routers = Devices.keys()
Routers.sort()
for Router in Routers:
logToFile.info('Configuring ip address on Device %s',Router)
RouterInst = basicConfiguration.switchToRouter(self.device,Router)
RouterInst = basicConfiguration.preliminaryInstalls(RouterInst)
#RouterInst = basicConfiguration.boot(RouterInst)
interfaces = Devices[Router].keys()
interfaces.sort()
for intf in interfaces:
interface = intf
ipadd = Devices[Router][intf]
RouterInst = basicConfiguration.configureIP(RouterInst,interface,ipadd)
logToFile.info(' IP address for %s interface of device %s is configured',interface,Router)
RouterInst.sendline('exit')
logToFile.info('IP address configured on all of the interfaces in %s',Router)
return
def createBGPNeighbor(self, Router):
data = getData('ProtocolSpecific.json')
if (isinstance(Router, list)):
for i in Router:
print "Configuring BGP in %s" % (i)
RouterInst = self.enableBGPGlobal(i)
Peers = data["BGP_Parameters"][i]['PeerDetails']
for peer in Peers.keys():
PeerAS = data["BGP_Parameters"][i]['PeerDetails'][peer][
'ASNum']
PeerAddress = data["BGP_Parameters"][i]['PeerDetails'][
peer]['IP_Address']
RouterInst = BGP.createBGPV4Neighbor(
RouterInst, PeerAS, PeerAddress)
RouterInst.sendline('exit')
print "BGP neighbors done in %s" % (i)
elif Router == 'all':
for i in data['BGP_Parameters'].keys():
print "Configuring BGP in %s" % (i)
RouterInst = self.enableBGPGlobal(i)
Peers = data["BGP_Parameters"][i]['PeerDetails']
#print Peers.keys()
for peer in Peers.keys():
#print
PeerAS = str(data["BGP_Parameters"][i]['PeerDetails'][str(
peer)]['ASNum'])
PeerAddress = str(data["BGP_Parameters"][i]['PeerDetails'][
str(peer)]['IP_Address'])
NeighborAddress = PeerAddress.split('/')
RouterInst = BGP.createBGPV4Neighbor(
RouterInst, PeerAS, NeighborAddress[0])
RouterInst.sendline('exit')
print "BGP neighbor set in %s" % (i)
return
def fullConnection():
from GetData import getData
from keras.layers import Input, Dense
from keras.models import Model
import keras
X_train, X_val, X_test, y_train, y_val, y_test = getData()
y_train = keras.utils.to_categorical(y_train, num_classes=2)
y_test = keras.utils.to_categorical(y_train, num_classes=2)
inputs = Input(shape=(103, ))
x = Dense(64, activation='relu')(inputs)
x = Dense(64, activation='relu')(x)
predictions = Dense(2, activation="softmax")(x)
# a layer instance is callable on a tensor, and returns a tensor
model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer="rmsprop",
loss="categorical_crossentropy",
metrics=['accuracy'])
model.fit(X_train, y_train)
x1 = X_test.iloc[0, :]
y1 = y_test[0, :]
def btnNer():
text = getData(False)
lang = detect(text)
if (lang == "en"):
TaggedOrganizations, TaggedPersons, TaggedGeographicalEntities = nerEng(
text)
jres = {
'org': "Organizations: ",
'org2': TaggedOrganizations,
'per': 'Persons: ',
'per2': TaggedPersons,
'loc': 'Locations: ',
'loc2': TaggedGeographicalEntities
}
return jsonify(jres)
elif (lang == "tr"):
orglist, namelist, loclist, timelist = nertr(text)
jres = {
'org': "Organizations: ",
'org2': orglist,
'per': 'Persons: ',
'per2': namelist,
'loc': 'Locations:',
'loc2': loclist,
'time': "Time:",
'time2': timelist
}
return jsonify(jres)
else:
result = "This language is not supported for NER"
jres = {"org": result}
return jsonify(jres)
from PowLaw import *
import pandas as pd
from GetData import getData
# https://pypi.python.org/pypi/louvain/
import community
import matplotlib.pyplot as plt
def Import_Obj(File):
import pickle
File_Name = File + '.pkl'
pkl_file = open(File_Name, 'rb')
return pickle.load(pkl_file)
Data = getData(with_geo_tag=True)
IdNameDict = Import_Obj('./Data/IdNameDict')
def addUserName(Data):
user_name = [IdNameDict[Id] for Id in Data.uid]
Data['user_name'] = user_name
def addPlace(Data):
Place = []
for loc in Data.location_name:
try:
Place.append(loc.split(',')[1])
except:
print "userdata : " + str(userdata)
mqttc = paho.Client()
awshost = "ec2-54-202-225-155.us-west-2.compute.amazonaws.com"
awsport = 1883
mqttc.connect(awshost, awsport, keepalive=60)
mqttc.on_connect = on_connect
mqttc.on_message = on_message
mqttc.on_subscribe = on_subscribe
mqttc.loop_start()
while True:
picontrol = getData()
print picontrol
latt = str(picontrol).split("::")
longitude = latt[0]
print(latt[0])
print cloud
if (latt[0] >= str(cloud) and cloud != 0.0):
#if (latt[0] >= str(96.766098)):
print "In loop"
print latt[0]
print "turning on light"
GPIO.output(17, True)
#sleep(1)
break
else:
mqttc.publish("ack", picontrol)
def __init__(self): #Initializes class object with below variables
self.device = pexpect.spawn("/bin/bash")
self.data = getData('variable.json')
self.NumberOfDevices = self.data["NumberOfDevices"]
self.Network = self.data["Network"]
def btnlangDetect():
text = getData(False)
result, lang = languagedetect(text)
jres = {'language': result, 'language2': lang}
return jsonify(jres)
def btnTerm():
text = getData(True)
lang = detect(getData(False))
term = termWeigthing(text, lang)
jres = {'detail': 'Descriptive Terms: ', 'detail2': term}
return jsonify(jres)
def btnCon():
text = getData(False)
jres = {'detail': text}
return jsonify(jres)
def btnWord():
text = getData(True)
plt = wordCloud(text)
jres = {'detail': plt.show()}
return jsonify(jres)
def __init__(self):
self.child = pexpect.spawn("/bin/bash")
self.data = getData('variable.json')
self.Number_of_instances = self.data["number_of_instances"]
self.Network = self.data["Network"]
