def __init__(self, input_B_Di, input_shape, output_shape, initializer):
assert len(input_shape) == len(output_shape) == 1
util.header('Affine(in=%d, out=%d)' %
(input_shape[0], output_shape[0]))
self._output_shape = (output_shape[0], )
with variable_scope(type(self).__name__) as self.__varscope:
if initializer is None:
# initializer = np.random.randn(input_shape[0], output_shape[0]) * np.sqrt(2./input_shape[0])
# Glorot/Bengio 2010
s = np.sqrt(6. / (input_shape[0] + output_shape[0]))
initializer = np.random.uniform(low=-s,
high=s,
size=(input_shape[0],
output_shape[0]))
else:
assert initializer.shape == (input_shape[0], output_shape[0])
self.W_Di_Do = get_variable(
'W', initializer.astype(theano.config.floatX))
self.b_1_Do = get_variable('b',
np.zeros((1, output_shape[0]),
dtype=theano.config.floatX),
broadcastable=(True, False))
self._output_B_Do = input_B_Di.dot(self.W_Di_Do) + self.b_1_Do
def sendPacket(self, curr_recv=0, fin=False):
if fin:
h = header(0, 0, 0, 1, 0, 1)
else:
h = header(0, 0, curr_recv, 0, 0, 1)
print("send ack #%s" % curr_recv)
packet = Packet(h, (c_ubyte * 1000)(*(b'')))
self.s.sendto(packet, self.agent)
def __init__(self, input_B_Di, output_shape, func):
util.header('Nonlinearity(func=%s)' % func)
self._output_shape = output_shape
with variable_scope(type(self).__name__) as self.__varscope:
self._output_B_Do = {
'relu': tensor.nnet.relu,
'lrelu': lambda x: tensor.nnet.relu(x, .01),
'elu': tensor.nnet.elu,
'tanh': tensor.tanh,
}[func](input_B_Di)
def __init__(self, input_B_Di, output_shape, func):
util.header('Nonlinearity(func=%s)' % func)
self._output_shape = output_shape
with variable_scope(type(self).__name__) as self.__varscope:
self._output_B_Do = {
'relu': tensor.nnet.relu,
'lrelu': lambda x: tensor.nnet.relu(x, .01),
'elu': tensor.nnet.elu,
'tanh': tensor.tanh,
}[func](input_B_Di)
def main(model, metrics):
if metrics:
old_metrics = {}
new_metrics = {}
files = f'data/predict/{model}/*_0000.nii.gz'
files = f'data/predict/{model}/*/*.nii.gz'
for file in glob.glob(files):
sample, i = os.path.basename(file).split('_')
i = i[:4]
header = util.header(file)
volume = util.read_vol(file)
if metrics:
label = util.read_vol(
glob.glob(f'data/labels/{sample}/{sample}_0_all_brains.nii.gz')
[0])
old_metrics[sample] = util.dice_coef(label, volume)
n = 2 if sample in constants.TWINS else 1
volume = remove_artifacts(volume, n)
if metrics:
new_metrics[sample] = util.dice_coef(label, volume)
os.makedirs(f'data/predict_cleaned/{model}/{sample}', exist_ok=True)
util.save_vol(
volume,
f'data/predict_cleaned/{model}/{sample}/{sample}_{i}.nii.gz',
header)
if metrics:
print('before: ', old_metrics)
print('after: ', new_metrics)
def predict(self, generator, subj_dirs=False):
n = len(generator)
for i in range(n):
#TODO: why is this hard coded here.
input_file = generator.input_files[i]
subj_dir = input_file.split("/")[-2]
path = f'{self.output_location}/{subj_dir}'
os.makedirs(path, exist_ok=True)
fname = input_file.split('/')[-1]
header = util.header(input_file)
#vol = nib.load(input_file)
vol = generator[i]
#vol = vol[0,:,:,:]
sub_vols, sub_codes = simple_preproc.get_subvolumes(
vol[0], self.input_size)
preds = []
for sub_vol in sub_vols:
preds.append(
self.model.predict(sub_vol[np.newaxis, :, :, :, :])[0])
pred = simple_preproc.sub_vols_to_original_shape(
preds, sub_codes, vol.shape[1:])
util.save_vol(
pred,
os.path.join(path, fname),
)
def df2str(df, s_format='%.4g', s_null=''):
df2 = df.copy()
for s in util.header(df2):
df2[s] = [
",".join([s_format % x for x in X]) for X in df[s].values
]
return df2
def __init__(self, input_B_T_Di, mask_B_T, hidden_B_Dh, cell_B_Dh, input_shape, layerspec_json):
'''
Args:
layerspec (string): JSON string describing LSTM
'''
assert len(input_shape) >= 1
self._input_shape = input_shape
layerspec = json.loads(layerspec_json)
util.header('Loading LSTM net specification')
print (json.dumps(layerspec, indent=2, separators=(',', ': ')))
self.layers = []
B, T = input_B_T_Di.shape[0], input_B_T_Di.shape[1]
with variable_scope(type(self).__name__) as self.__varscope:
prev_output, prev_output_shape = input_B_T_Di, input_shape
for i_layer, ls in enumerate(layerspec):
with variable_scope('layer_%d' % i_layer):
# not implemented yet
if ls['type'] == 'reshape':
_check_keys(ls, ['type', 'new_shape'], [])
self.layers.append(ReshapeLayer(prev_output, ls['new_shape']))
# add sequential input setting
elif ls['type'] == 'fc':
_check_keys(ls, ['type', 'n'], ['initializer'])
self.layers.append(AffineLayer(
prev_output, prev_output_shape, output_shape=(ls['n'],), initializer=_parse_initializer(ls), sequential_input=True))
elif ls['type'] == 'nonlin':
_check_keys(ls, ['type', 'func'], [])
self.layers.append(NonlinearityLayer(prev_output, prev_output_shape, ls['func']))
# add lstm layer
elif ls['type'] == 'lstm':
_check_keys(ls, ['type', 'n'], [])
self.layers.append(LSTMLayer(prev_output, mask_B_T, hidden_B_Dh, cell_B_Dh, prev_output_shape, output_shape=(ls['n'],), initializer=None))
else:
raise NotImplementedError('Unknown layer type %s' % ls['type'])
prev_output, prev_output_shape = self.layers[-1].output, self.layers[-1].output_shape
self._output, self._output_shape = prev_output, prev_output_shape
def parse(sysv, loader):
""" Modules can set their own CLI options. Right now we only
load services and scanners, as these represent a majority of
the 'typical' use case for something you want to pull off quickly.
loader is a Loader object with all loaded modules.
"""
parser = argparse.ArgumentParser(description=util.header())
# add standard options
parser.add_argument('-q', help='Generic network sniff', action='store',
dest='filter')
parser.add_argument('--update', help='Update Zarp', action='store_true',
default=False, dest='update')
service_group = parser.add_argument_group('Services')
scanner_group = parser.add_argument_group('Scanners')
# iterate through loaded modules and build the argument parser
for service in loader.services:
if hasattr(service, 'cli'):
service().cli(service_group)
for scanner in loader.scanner:
if hasattr(scanner, 'cli'):
scanner().cli(scanner_group)
options = parser.parse_args()
option_dict = options.__dict__
# first handle standard options
if options.filter:
util.Msg("Sniffing with filter [%s]...(ctrl^c to exit)" %
options.filter)
try:
sniff(filter=options.filter, store=0, prn=lambda x: x.summary())
except Exception:
util.Msg("Exiting sniffer..")
except Scapy_Exception as msg:
util.Error(msg)
sys.exit(1)
elif options.update:
update()
sys.exit(1)
# we can only launch one module at a time, so grab the first
usr_mod = [x for x in option_dict.keys() if option_dict[x] is True][0]
# see what it is
if usr_mod in [x().which for x in loader.services]:
module = [x for x in loader.services if x().which == usr_mod][0]
util.Msg('Starting %s...' % module().which)
mod = module()
mod.dump_data = True
mod.initialize()
elif usr_mod in [x().which for x in loader.scanner]:
module = [x for x in loader.scanner if x().which == usr_mod][0]
module().initialize()
sys.exit(1)
def __init__(self, input_B_Di, input_shape, output_shape, initializer):
assert len(input_shape) == len(output_shape) == 1
util.header('Affine(in=%d, out=%d)' % (input_shape[0], output_shape[0]))
self._output_shape = (output_shape[0],)
with variable_scope(type(self).__name__) as self.__varscope:
if initializer is None:
# initializer = np.random.randn(input_shape[0], output_shape[0]) * np.sqrt(2./input_shape[0])
# Glorot/Bengio 2010
s = np.sqrt(6. / (input_shape[0] + output_shape[0]))
initializer = np.random.uniform(low=-s, high=s, size=(input_shape[0],output_shape[0]))
else:
assert initializer.shape == (input_shape[0], output_shape[0])
self.W_Di_Do = get_variable('W', initializer.astype(theano.config.floatX))
self.b_1_Do = get_variable('b', np.zeros((1, output_shape[0]), dtype=theano.config.floatX), broadcastable=(True,False))
self._output_B_Do = input_B_Di.dot(self.W_Di_Do) + self.b_1_Do
def str2df(df):
df2 = df.copy()
for s in util.header(df2):
df2[s] = [
np.array(util.sarray2rarray(x.split(",")))
for x in df[s].values
]
return df2
def predict(self, generator, path):
preds = self.model.predict_generator(generator, verbose=1)
# FIXME
for i in range(preds.shape[0]):
fname = generator.inputs[i].split('/')[-1]
header = util.header(generator.inputs[i])
util.save_vol(uncrop(preds[i], generator.shape),
os.path.join(path, fname), header)
def __init__(self, input_B_Di, input_shape, layerspec_json):
'''
Args:
layerspec (string): JSON string describing layers
'''
assert len(input_shape) >= 1
self.input_B_Di = input_B_Di
layerspec = json.loads(layerspec_json)
util.header('Loading feedforward net specification')
print(json.dumps(layerspec, indent=2, separators=(',', ': ')))
self.layers = []
with variable_scope(type(self).__name__) as self.__varscope:
prev_output, prev_output_shape = input_B_Di, input_shape
for i_layer, ls in enumerate(layerspec):
with variable_scope('layer_%d' % i_layer):
if ls['type'] == 'reshape':
_check_keys(ls, ['type', 'new_shape'], [])
self.layers.append(
ReshapeLayer(prev_output, ls['new_shape']))
elif ls['type'] == 'fc':
_check_keys(ls, ['type', 'n'], ['initializer'])
self.layers.append(
AffineLayer(prev_output,
prev_output_shape,
output_shape=(ls['n'], ),
initializer=_parse_initializer(ls)))
elif ls['type'] == 'nonlin':
_check_keys(ls, ['type', 'func'], [])
self.layers.append(
NonlinearityLayer(prev_output, prev_output_shape,
ls['func']))
else:
raise NotImplementedError('Unknown layer type %s' %
ls['type'])
prev_output, prev_output_shape = self.layers[
-1].output, self.layers[-1].output_shape
self._output, self._output_shape = prev_output, prev_output_shape
def get_default_exp_cols(self, S_col=None):
'Filter S_col for numerical columns that are not in key columns (ID, Description)'
S_col = S_col or []
S_key=set([self.input_opt['ID']]+self.input_opt['DESCRIPTION']+[self.input_opt['WEIGHT_COL']])
if len(S_col)==0: S_col=util.header(self.table)
# filter out key columns
S_col=[s for s in S_col if s not in S_key]
# filter out non-numeric columns
return [s for s in S_col if (self.table[s].dtype is not np.dtype(object))]
def restore_distance(s_file, max_dist=1.0):
'No longer needed, as we use cwc.new'
df=pd.read_table(s_file, header=None)
s_col=util.header(df)[-1]
R=(-df[s_col]+1.0).abs()*max_dist
if R.max()<=1.0:
df[s_col]=util.rarray2sarray((-R+1.0).abs(), s_format='%.3f') # convert to similarity score and output
df.to_csv(s_file, sep='\t', index=False, header=False)
else:
util.warn_msg('Cannot restore distance to similarity, as the max distance '+str(R.max())+' > 1.0, restore skipped!')
def save_prediction(pred, input_file, tile, path, scale=False):
fname = input_file.split('/')[-1]
sample = fname.split('_')[0]
path = os.path.join(path, sample)
os.makedirs(path, exist_ok=True)
shape = util.shape(input_file)
header = util.header(input_file)
vol = process.postprocess(pred, shape, resize=True, tile=tile)
util.save_vol(vol, os.path.join(path, fname), header, scale)
print(fname, flush=True)
def parse(sysv):
""" Provides an interface for parsing CLI options.
As of now (v.10) this is set manually; eventually
it will be refactored to allow modules to set their
own CLI interfaces.
"""
# parse debug first so the header isn't dumped twice
if 'debug' in sysv[1]:
util.isDebug = True
util.debug('Zarp debug session started.')
return
parser = argparse.ArgumentParser(description=util.header())
# other options
parser.add_argument('-q', help='Quick network sniff with filter', action='store', dest='filter')
parser.add_argument('--debug', help='Launch Zarp with error logging',action='store_true',default=False,dest='debug')
parser.add_argument('--update', help='Update Zarp',action='store_true', default=False,dest='update')
# scanners
scan_group = parser.add_argument_group("Scanners")
scan_group.add_argument('-s', help='Quick network map', action='store', dest='scan')
scan_group.add_argument('--finger', help='Fingerprint scan packets', action='store_true', default=False,dest='finger')
scan_group.add_argument('-a', help='Service scan', action='store_true', default=False, dest='service')
scan_group.add_argument('-w', help='Wireless AP scan', action='store_true', default=False,dest='wifind')
scan_group.add_argument('--channel',help='Set channel to scan on',action='store', dest='channel')
# spoof
spoof_group = parser.add_argument_group("Services")
spoof_group.add_argument('--ssh',help='SSH server', action='store_true',default=False,dest='ssh')
spoof_group.add_argument('--ftp',help='FTP server', action='store_true',default=False,dest='ftp')
spoof_group.add_argument('--http',help='HTTP server', action='store_true',default=False,dest='http')
spoof_group.add_argument('--smb', help='SMB listener',action='store_true',default=False,dest='smb')
spoof_group.add_argument('--wap', help='Wireless Access Point', action='store_true',default=False,dest='wap')
options = parser.parse_args()
# initiate
if options.scan is not None:
tmp = net_map()
tmp.net_mask = options.scan
tmp.fingerprint = options.finger
tmp.scan_block()
elif options.service:
tmp = service_scan()
tmp.initialize()
elif options.filter is not None:
util.Msg("Sniffing with filter [%s]...(ctrl^c to exit)"%options.filter)
try:
sniff(filter=options.filter,store=0, prn=lambda x: x.summary())
except KeyboardInterrupt,Exception:
util.Msg("Exiting sniffer..")
except Scapy_Exception as msg:
util.Error(msg)
sys.exit(1)
def __init__(self, input_B_Di, input_shape, layerspec_json):
'''
Args:
layerspec (string): JSON string describing layers
'''
assert len(input_shape) >= 1
self.input_B_Di = input_B_Di
layerspec = json.loads(layerspec_json)
util.header('Loading feedforward net specification')
print json.dumps(layerspec, indent=2, separators=(',', ': '))
self.layers = []
with variable_scope(type(self).__name__) as self.__varscope:
prev_output, prev_output_shape = input_B_Di, input_shape
for i_layer, ls in enumerate(layerspec):
with variable_scope('layer_%d' % i_layer):
if ls['type'] == 'reshape':
_check_keys(ls, ['type', 'new_shape'], [])
self.layers.append(ReshapeLayer(prev_output, ls['new_shape']))
elif ls['type'] == 'fc':
_check_keys(ls, ['type', 'n'], ['initializer'])
self.layers.append(AffineLayer(
prev_output, prev_output_shape, output_shape=(ls['n'],), initializer=_parse_initializer(ls)))
elif ls['type'] == 'nonlin':
_check_keys(ls, ['type', 'func'], [])
self.layers.append(NonlinearityLayer(prev_output, prev_output_shape, ls['func']))
else:
raise NotImplementedError('Unknown layer type %s' % ls['type'])
prev_output, prev_output_shape = self.layers[-1].output, self.layers[-1].output_shape
self._output, self._output_shape = prev_output, prev_output_shape
def createPackets(self, _file):
packets = []
count = 1
with open(_file, "rb") as f:
while True:
data = f.read(1000)
#print("ori data:",data)
h = header(len(data), count, 0, 0, 0, 0)
packet = Packet(h, (c_ubyte * 1000)(*data))
#print("data:", packet.data)
if not data:
break
packets.append(packet)
count += 1
return packets
def save_predictions(preds, generator, path, scale=False):
os.makedirs(path, exist_ok=True)
if generator.tile_inputs:
preds = np.reshape(preds, (-1, 8) + preds.shape[1:])
for i in range(preds.shape[0]):
input_file = generator.input_files[i]
fname = input_file.split('/')[-1]
header = util.header(input_file)
shape = util.shape(input_file)
volume = process.postprocess(preds[i],
shape,
resize=True,
tile=generator.tile_inputs)
util.save_vol(volume, os.path.join(path, fname), header, scale)
def general_statistics():
"afficher des statistiques sur la data "
desc = data.describe()
description = np.asarray(desc)
# créer la page html correspont a cette page
html = ''
html += header()
html += '<table><thead><tr><th colspan="2">table de desciption d age</th></tr></thead><tbody>'
html += "<tr> <td>champs</td> <td>valeurs</td></tr>"
html += "<tr> <td>total contenus</td> <td>" + str(
description[0]) + "</td></tr>"
html += "<tr> <td>moyenne des champs </td> <td>" + str(
description[1]) + "</td></tr>"
html += "<tr> <td>average </td> <td>" + str(description[2]) + "</td></tr>"
html += "<tr> <td>minimum d'age </td> <td>" + str(
description[3]) + "</td></tr>"
html += "<tr> <td>premier quartile 25%</td> <td>" + str(
description[4]) + "</td></tr>"
html += "<tr> <td>median 50% </td> <td>" + str(
description[5]) + "</td></tr>"
html += "<tr> <td>3eme quartile </td> <td>" + str(
description[6]) + "</td></tr>"
html += "<tr> <td>maximum d'age</td> <td>" + str(
description[7]) + "</td></tr>"
html += "</tbody></table>"
male = data[data['gender'] == 'm']
female = data[data['gender'] == 'f']
number_of_male = len(male)
number_of_female = len(female)
html += "hommes: " + str(number_of_male) + " femmes: " + str(
number_of_female) + "</br>"
male_percent = (number_of_male * 100) / (number_of_male + number_of_female)
female_percent = (number_of_female * 100) / (number_of_male +
number_of_female)
html += "porcentage des hommes: " + str(
male_percent) + "% porcentage des femmes: " + str(
female_percent) + "%</br>"
html += '<a href="http://localhost:5001/" > retourner a la page d accueil </a>'
html += "</body>"
return html
def anova2_unbalanced(T):
'''Unbalanced two-way ANOVA
#First two way model
# Y = B0+a1*A1+a2*A2+...+aa-1*Aa-1+b1*B1+b2*B2+...+bb-1*Bb-1
# J.D. Jobson 91, page 458, Applied multivariate data analysis'''
a = len(util.header(T))
b = len(T)
Y = list()
for i in range(b):
for j in range(a):
Y.append(T.values[i, j])
Y = np.hstack(Y)
N = len(Y)
Y = Y.reshape(N, 1)
X = np.zeros([N, a + b - 1])
cnt = 0
for i in range(b):
for j in range(a):
for k in range(len(T.values[i, j])):
X[cnt, 0] = 1 #B0
if j < a - 1: X[cnt, j + 1] = 1 #Aj
if i < b - 1: X[cnt, a + i] = 1 #Bi
cnt += 1
# Jobson page 226
C2 = la.inv(np.dot(X.T, X))
Xb = np.dot(X, np.dot(C2, np.dot(X.T, Y)))
SSE1 = np.dot((Y - Xb).T, (Y - Xb)).sum()
dfE1 = N - (a + b - 1)
# Now try drop the A factor from the model
# Y=B0+b1*B1+b2*B2+...+bb-1*Bb-1;
X = np.zeros([N, b])
cnt = 0
for i in range(b):
for j in range(a):
for k in range(len(T.values[i, j])):
X[cnt, 0] = 1 #B0
if i < b - 1: X[cnt, i + 1] = 1 #Bi
cnt += 1
C2 = la.inv(np.dot(X.T, X))
Xb = np.dot(X, np.dot(C2, (np.dot(X.T, Y))))
SSE2 = np.dot((Y - Xb).T, (Y - Xb)).sum()
dfE2 = N - b
df = dfE2 - dfE1
F = max(SSE2 - SSE1, 0.0) / df / ((SSE1 + np.finfo(float).eps) / dfE1)
return ss.beta.cdf(dfE1 / (dfE1 + df * F), dfE1 * 0.5, df * 0.5)
def main(model):
folder = options.sample if options.sample else '*'
files = f'data/predict/{model}/{folder}/*.nii.gz'
for file in glob.glob(files):
sample, i = os.path.basename(file).split('_')
i = i[:4]
header = util.header(file)
volume = util.read_vol(file)
n = 2 if sample in constants.TWINS else 1
volume = remove_artifacts(volume, n)
os.makedirs(f'data/predict_cleaned/{model}/{sample}', exist_ok=True)
util.save_vol(
volume,
f'data/predict_cleaned/{model}/{sample}/{sample}_{i}.nii.gz',
header)
def send(self, _file):
packets = self.createPackets(_file)
n_packets = len(packets)
resend = [False] * n_packets
print("number of packets:", n_packets)
print("start sending packets...")
curr_packet = 0
packet_have_sent = 0
success = False
while packet_have_sent < n_packets:
end = min(self.windowSize + packet_have_sent, n_packets)
while curr_packet < end:
self.sendPacket(packets[curr_packet])
if resend[curr_packet]:
print("resnd data #%s, winSize = %s" %
(curr_packet + 1, self.windowSize))
else:
print("send data #%s, winSize = %s" %
(curr_packet + 1, self.windowSize))
resend[curr_packet] = True
curr_packet += 1
success, ack_return = self.checkACK(packet_have_sent, end)
if success:
packet_have_sent = end
if self.windowSize > self.threshold:
self.windowSize += 1
else:
self.windowSize *= 2
else:
packet_have_sent = ack_return
curr_packet = ack_return
self.windowSize = 1
finPacket = Packet(header(len(b''), n_packets + 1, 0, 1, 0, 0),
(c_ubyte * 1000)(*(b'')))
print("send fin")
self.sendPacket(finPacket)
p = Packet.from_buffer_copy(self.s.recvfrom(1024)[0])
if p.head.fin == 1:
print("recv finack")
else:
print("Error: expect finack")
def mackskillings2(T):
'''Two-way Mack-Skillings
# T is a table, each row is a block (a probe)
# each column is a treatment
# each cell is a Rarray()
# See Hollander & Wolfe, Nonparametric statistical methods'''
n = len(T)
k = len(util.header(T))
T2 = T.copy()
for i in range(n):
Rr = rank(np.hstack(T.values[i, :]))
cnt = 0
for j in range(k):
T2.values[i, j] = Rr[cnt:cnt + len(T.values[i, j])]
cnt += len(T.values[i, j])
q = np.zeros(n)
for i in range(n):
for j in range(k):
q[i] += len(T2.values[i, j])
V0 = np.zeros([k - 1, k - 1])
for s in range(k - 1):
for t in range(s, k - 1):
if s == t:
for i in range(n):
cis = len(T2.values[i, s])
V0[s, t] += cis * (q[i] - cis) * (q[i] +
1) / 12.0 / q[i] / q[i]
else:
for i in range(n):
cis = len(T2.values[i, s])
cit = len(T2.values[i, t])
V0[s, t] -= cis * cit * (q[i] + 1) / 12.0 / q[i] / q[i]
V0[t, s] = V0[s, t]
V = np.zeros([1, k - 1])
for j in range(k - 1):
V[0, j] = 0.0
for i in range(n):
V[0, j] += np.sum(T2.values[i, j]) / q[i]
V[0, j] -= len(T2.values[i, j]) * (q[i] + 1) / 2.0 / q[i]
MS = np.dot(np.dot(V, la.inv(V0)), V.T).sum()
return ss.gamma.sf(MS / 2, (k - 1.0) / 2.0)
def anova2(T, l_interaction=False):
'''Sokal 95 Chapter 11. Two-way analysis of variance'''
b = len(T)
a = len(util.header(T))
n = len(T.values[0, 0])
if n == 1 and l_interaction: return None
N = a * b * n
Ya = np.zeros(a)
Yb = np.zeros(b)
for i in range(a):
for j in range(b):
Ya[i] += np.sum(T.values[j, i])
Ya[i] /= n * b
for i in range(b):
for j in range(a):
Yb[i] += np.sum(T.values[i, j])
Yb[i] /= n * a
Yab = np.mean(Ya)
SSa = n * b * np.var(Ya) * a
SSb = n * a * np.var(Yb) * b
SSab = 0.0
for j in range(a):
for i in range(b):
SSab += (np.mean(T.values[i, j]) - Ya[j] - Yb[i] + Yab)**2
SSab *= n
SSwn = 0.0
if n > 1:
for j in range(a):
for i in range(b):
SSwn += np.var(T.values[i, j]) * n
dfa = a - 1.0
MSa = SSa / dfa
if l_interaction:
df = a * b * (n - 1.0)
F = max(MSa / (SSwn / df + np.finfo(float).eps), 0.0)
else:
df = (a - 1.0) * (b - 1.0) + a * b * (n - 1.0)
F = max(MSa / ((SSwn + SSab) / df + np.finfo(float).eps), 0.0)
return ss.beta.cdf(df / (df + dfa * F), df * 0.5, dfa * 0.5)
def parse(sysv, loader):
""" Modules can set their own CLI options. Right now we only
load services and scanners, as these represent a majority of
the 'typical' use case for something you want to pull off quickly.
loader is a Loader object with all loaded modules.
"""
parser = argparse.ArgumentParser(description=util.header())
# add standard options
parser.add_argument('-q', help='Generic network sniff', action='store', dest='filter')
parser.add_argument('--update', help='Update Zarp', action='store_true',
default=False, dest='update')
service_group = parser.add_argument_group('Services')
scanner_group = parser.add_argument_group('Scanners')
# iterate through loaded modules and build the argument parser
for service in loader.services:
if hasattr(service, 'cli'):
service().cli(service_group)
for scanner in loader.scanner:
if hasattr(scanner, 'cli'):
scanner().cli(scanner_group)
options = parser.parse_args()
option_dict = options.__dict__
# first handle standard options
if options.filter:
util.Msg("Sniffing with filter [%s]...(ctrl^c to exit)"%options.filter)
try:
sniff(filter=options.filter,store=0, prn=lambda x: x.summary())
except KeyboardInterrupt,Exception:
util.Msg("Exiting sniffer..")
except Scapy_Exception as msg:
util.Error(msg)
def home():
# initialiser les fonctions et créer les plots visuals
gender_ville(data)
population_ville(data)
age_histogram(data)
gender_distribution(data)
gender_ville(data)
porcentage_par_ville(data)
bars_gender_ville(data)
esperance_vie(data)
pyramide_ages(data)
"la page d'acceuil de l'API"
html = ''
html += header()
html += '<h1>API pour naviguer dans des données démographique visualement </h1>'
html += '<a href="http://localhost:5001/statistic"> Cliquer pour voir les stats</a></br>'
html += '<a href="http://localhost:5001/basic_visual"> Cliquer pour voir les charts des features basiques</a></br>'
html += '<a href="http://localhost:5001/stat_avancee"> Cliquer pour voir les charts des statistique </a></br>'
html += '<a href="http://localhost:5001/demographie"> Cliquer pour voir les charts démographie </a></br>'
html += '<a href="http://localhost:5001/form"> Cliquer pour rajouter des champs dans la dataset </a></br>'
return html
def mackskillings(T):
'''Non-parametric multi-group test with multiple measurements
# T is a table, each row is a block (a probe)
# each column is a treatment
# each cell is a Rarray()
# See Hollander & Wolfe, Nonparametric statistical methods'''
n = len(T)
c = len(T.values[0, 0])
# nof measurements, assume all share the same # of measurements
k = len(util.header(T))
S = np.zeros(k)
N = n * c * k
T2 = T.copy()
Rc = None
for i in range(n):
Rr = rank(np.hstack(T.values[i, :]))
for j in range(k):
T2.values[i, j] = Rr[j * c:(j + 1) * c]
for j in range(k):
for i in range(n):
S[j] += np.sum(T2.values[i, j]) / c
MS = max(np.sum((S - (N + n) / 2.0)**2) * 12.0 / k / (N + n), 0.0)
return ss.gamma.sf(MS / 2, (k - 1.0) / 2.0)
def main():
# handle command line options first
if len(sys.argv) > 1:
parse_cmd.parse(sys.argv)
# set up configuration
config.initialize()
# menus
main_menu = [ 'Poisoners', 'DoS Attacks', 'Sniffers', 'Scanners',
'Parameter','Services','Sessions']
poison_menu = [ 'ARP Poison', 'DNS Poison', 'DHCP Poison', 'NBNS Spoof']
dos_menu = [ 'NDP', 'Nestea', 'LAND', 'TCP SYN', 'SMB2',
'DHCP Starve'
]
sniffer_menu = [ 'HTTP Sniffer', 'Password Sniffer', 'Traffic Sniffer']
service_menu = [ 'HTTP Server', 'SSH Server', 'FTP Server', 'SMB Listener', 'Access Point' ]
scanner_menu = [ 'NetMap', 'Service Scan', 'AP Scan']
parameter_menu = [ 'WEP Crack', 'WPA2 Crack', 'WPS Crack', 'Router Pwn', 'Pass-The-Hash' ]
running = True
choice = -1
while running:
header()
choice = print_menu(main_menu)
if choice == 0:
# check if they've got running sessions!
cnt = stream.get_session_count()
if cnt > 0:
Msg('You have %d sessions running. Are you sure?'%cnt)
choice = raw_input('> ')
if 'y' in choice.lower():
stream.stop_session('all', -1)
running = False
else:
debug("Exiting with session count: %d"%(cnt))
Msg("Exiting...")
running = False
elif choice == 1:
while True:
choice = print_menu(poison_menu)
if choice == 0:
break
elif choice == 1:
stream.initialize('arp')
elif choice == 2:
stream.initialize('dns')
elif choice == 3:
stream.initialize('dhcp')
elif choice == 4:
stream.initialize('nbns')
elif choice == -1:
pass
elif choice == 2:
while True:
choice = print_menu(dos_menu)
if choice == 1:
stream.initialize('ndp')
elif choice == 2:
stream.initialize('nestea')
elif choice == 3:
stream.initialize('land')
elif choice == 4:
stream.initialize('tcp_syn')
elif choice == 5:
stream.initialize('smb2')
elif choice == 6:
stream.initialize('dhcp_starv')
elif choice == 0:
break
elif choice == -1:
pass
else:
os.system('clear')
elif choice == 3:
while True:
choice = print_menu(sniffer_menu)
if choice == 0:
break
elif choice == 1:
stream.initialize('http_sniffer')
elif choice == 2:
stream.initialize('password_sniffer')
elif choice == 3:
stream.initialize('traffic_sniffer')
elif choice == -1:
pass
elif choice == 4:
while True:
choice = print_menu(scanner_menu)
if choice == 0:
break
elif choice == 1:
stream.initialize('net_map')
elif choice == 2:
stream.initialize('service_scan')
elif choice == 3:
stream.initialize('ap_scan')
elif choice == -1:
pass
elif choice == 5:
while True:
choice = print_menu(parameter_menu)
if choice == 0:
break
elif choice == 1:
stream.initialize('wep_crack')
elif choice == 2:
stream.initialize('wpa_crack')
elif choice == 3:
stream.initialize('wps_crack')
elif choice == 4:
stream.initialize('router_pwn')
elif choice == 5:
util.Error('Module not yet implemented')
#stream.initialize('pass_the_hash')
elif choice == -1:
pass
elif choice == 6:
while True:
choice = print_menu(service_menu)
if choice == 0:
break
elif choice == 1:
stream.initialize('http_server')
elif choice == 2:
stream.initialize('ssh_server')
elif choice == 3:
stream.initialize('ftp_server')
elif choice == 4:
stream.initialize('smb')
elif choice == 5:
stream.initialize('access_point')
elif choice == -1:
pass
elif choice == 7:
session_manager.menu()
elif choice == -1:
pass
def make_table(self):
opt=self.input_opt
self.table=pd.read_table(self.input+".input")
self.input_opt['EXP_WEIGHT']=self.table.iloc[0][3:].astype(float).values
self.table=self.table.drop([0], axis=0)
opt['DATA_COLS']=util.header(self.table)[3:]
def print_trainable_variables(self):
for v in self.get_trainable_variables():
util.header('- %s (%d parameters)' % (v.name, v.get_value().size))
util.header('Total: %d parameters' % (self.get_num_params(),))
def __init__(self, input_, new_shape):
self._output_shape = tuple(new_shape)
util.header('Reshape(new_shape=%s)' % (str(self._output_shape),))
with variable_scope(type(self).__name__) as self.__varscope:
self._output = input_.reshape((-1,)+self._output_shape)
def __init__(self, input_, new_shape):
self._output_shape = tuple(new_shape)
util.header('Reshape(new_shape=%s)' % (str(self._output_shape),))
with variable_scope(type(self).__name__) as self.__varscope:
self._output = input_.reshape((-1,)+self._output_shape)
import glob
import numpy as np
import util
for sample in [
'010918L', '010918S', '012115', '013018L', '013018S', '013118L',
'013118S', '021015', '021218L', '021218S', '022318L', '022318S',
'022415', '022618', '030217', '030315', '031317L', '031317T', '031516',
'031615', '031616', '031716', '032217', '032318a', '032318b',
'032318c', '032318d', '032818', '040218', '040417'
]:
files = glob.glob('data/labels/{}/{}_0_*_brain.nii.gz'.format(
sample, sample))
volume = np.zeros(
util.shape('data/raw/{}/{}_0.nii.gz'.format(sample, sample)))
header = util.header(files[0])
for file in files:
volume += util.read_vol(file)
util.save_vol(
volume, 'data/labels/{}/{}_0_all_brains.nii.gz'.format(sample, sample),
header)
def main():
""" Zarp entry point
"""
# set up configuration
config.initialize()
# set up database
database.initialize()
# load modules
loader = LoadedModules()
loader.load()
Msg('Loaded %d modules.' % loader.total)
# handle command line options first
if len(argv) > 1:
parse_cmd.parse(argv, loader)
# menus
main_menu = ['Poisoners', 'DoS Attacks', 'Sniffers', 'Scanners',
'Parameter', 'Services', 'Attacks', 'Sessions']
running = True
choice = -1
while running:
header()
choice = print_menu(main_menu)
if choice == 0:
# check if they've got running sessions!
cnt = stream.get_session_count()
if cnt > 0:
choice = raw_input(color.YELLOW +
'You have %d sessions running. Are you sure? [Y/n] ' % cnt
+ color.END)
if 'y' in choice.lower() or choice == '':
Msg('Shutting all sessions down...')
stream.stop_session('all', -1)
running = False
else:
debug("Exiting with session count: %d" % (cnt))
Msg("Exiting...")
running = False
elif choice == 1:
while True:
choice = print_menu([x().which for x in loader.poison])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.poison):
continue
else:
stream.initialize(loader.poison[choice - 1])
elif choice == 2:
while True:
choice = print_menu([x().which for x in loader.dos])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.dos):
continue
else:
stream.initialize(loader.dos[choice - 1])
elif choice == 3:
while True:
choice = print_menu([x().which for x in loader.sniffers])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.sniffers):
continue
else:
stream.initialize(loader.sniffers[choice - 1])
elif choice == 4:
while True:
choice = print_menu([x().which for x in loader.scanner])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.scanner):
continue
else:
stream.initialize(loader.scanner[choice - 1])
elif choice == 5:
while True:
choice = print_menu([x().which for x in loader.parameter])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.parameter):
continue
else:
stream.initialize(loader.parameter[choice - 1])
elif choice == 6:
while True:
choice = print_menu([x().which for x in loader.services])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.services):
continue
else:
stream.initialize(loader.services[choice - 1])
elif choice == 7:
while True:
choice = print_menu([x().which for x in loader.attacks])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.attacks):
continue
else:
stream.initialize(loader.attacks[choice - 1])
elif choice == 8:
session_manager.menu()
elif choice == -1:
pass
def main():
""" Zarp entry point
"""
# set up configuration
config.initialize()
# set up database
database.initialize()
# load modules
loader = LoadedModules()
loader.load()
util.Msg('Loaded %d modules.' % loader.total)
# handle command line options first
if len(argv) > 1:
parse_cmd.parse(argv, loader)
# menus
main_menu = ['Poisoners', 'DoS Attacks', 'Sniffers', 'Scanners',
'Parameter', 'Services', 'Attacks', 'Sessions']
running = True
choice = -1
while running:
util.header()
choice = util.print_menu(main_menu)
if choice == 0:
# check if they've got running sessions!
cnt = stream.get_session_count()
if cnt > 0:
display = color.B_YELLOW + 'You have %d sessions running. ' + \
'Are you sure? ' + color.B_GREEN + '[' + color.B_YELLOW + \
'Y' + color.B_GREEN + '/' + color.B_YELLOW + 'n' + \
color.B_GREEN + '] ' + color.END
choice = raw_input(display % cnt)
if 'y' in choice.lower() or choice == '':
util.Msg('Shutting all sessions down...')
stream.stop_session('all', -1)
running = False
else:
util.debug("Exiting with session count: %d" % (cnt))
util.Msg("Exiting...")
running = False
# remove zarp temporary directory
util.init_app('rm -fr /tmp/.zarp/')
# recheck that all sessions are down
cnt = stream.get_session_count()
if cnt <= 0:
# some libs dont clean up their own threads, so
# we need to hard quit those to avoid hanging; FIXME
_exit(1)
elif choice == 1:
while True:
choice = util.print_menu([x().which for x in loader.poison])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.poison):
continue
else:
stream.initialize(loader.poison[choice - 1])
elif choice == 2:
while True:
choice = util.print_menu([x().which for x in loader.dos])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.dos):
continue
else:
stream.initialize(loader.dos[choice - 1])
elif choice == 3:
while True:
choice = util.print_menu([x().which for x in loader.sniffers])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.sniffers):
continue
else:
stream.initialize(loader.sniffers[choice - 1])
elif choice == 4:
while True:
choice = util.print_menu([x().which for x in loader.scanner])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.scanner):
continue
else:
stream.initialize(loader.scanner[choice - 1])
elif choice == 5:
while True:
choice = util.print_menu([x().which for x in loader.parameter])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.parameter):
continue
else:
stream.initialize(loader.parameter[choice - 1])
elif choice == 6:
while True:
choice = util.print_menu([x().which for x in loader.services])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.services):
continue
else:
stream.initialize(loader.services[choice - 1])
elif choice == 7:
while True:
choice = util.print_menu([x().which for x in loader.attacks])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.attacks):
continue
else:
stream.initialize(loader.attacks[choice - 1])
elif choice == 8:
session_manager.menu()
elif choice == -1:
pass
from util import header
# How to run a python program
# 1. `python <file>`
# 2. python console
# 3. ipython
# =======================
# Basics
# ======================
print("Hello world!")
# Variables
header("Variables")
name = "Mate"
print(name)
# Types:
# boolean
# integer
# long
# float
# string
# list
# object
# None
# Python is a typed language
type(name)
# The commented part doesn't work as they have different types.
def main():
""" Zarp entry point
"""
# handle command line options first
if len(sys.argv) > 1:
parse_cmd.parse(sys.argv)
# set up configuration
config.initialize()
# menus
main_menu = [
'Poisoners', 'DoS Attacks', 'Sniffers', 'Scanners', 'Parameter',
'Services', 'Sessions'
]
# load modules
loader = LoadedModules()
loader.load()
Msg('Loaded %d modules.' % loader.total)
running = True
choice = -1
while running:
header()
choice = print_menu(main_menu)
if choice == 0:
# check if they've got running sessions!
cnt = stream.get_session_count()
if cnt > 0:
Msg('You have %d sessions running. Are you sure?' % cnt)
choice = raw_input('> ')
if 'y' in choice.lower():
Msg('Shutting all sessions down...')
stream.stop_session('all', -1)
running = False
else:
debug("Exiting with session count: %d" % (cnt))
Msg("Exiting...")
running = False
elif choice == 1:
while True:
choice = print_menu([x().which for x in loader.poison])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.poison):
continue
else:
stream.initialize(loader.poison[choice - 1], 'POISON')
elif choice == 2:
while True:
choice = print_menu([x().which for x in loader.dos])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.dos):
continue
else:
stream.initialize(loader.dos[choice - 1], 'DOS')
elif choice == 3:
while True:
choice = print_menu([x().which for x in loader.sniffers])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.sniffers):
continue
else:
stream.initialize(loader.sniffers[choice - 1], 'SNIFFER')
elif choice == 4:
while True:
choice = print_menu([x().which for x in loader.scanner])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.scanner):
continue
else:
stream.initialize(loader.scanner[choice - 1], 'SCANNER')
elif choice == 5:
while True:
choice = print_menu([x().which for x in loader.parameter])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.parameter):
continue
else:
stream.initialize(loader.parameter[choice - 1],
'PARAMETER')
elif choice == 6:
while True:
choice = print_menu([x().which for x in loader.services])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.services):
continue
else:
stream.initialize(loader.services[choice - 1], 'SERVICE')
elif choice == 7:
session_manager.menu()
elif choice == -1:
pass
def main():
""" Zarp entry point
"""
# set up configuration
config.initialize()
# set up database
database.initialize()
# load modules
loader = LoadedModules()
loader.load()
util.Msg('Loaded %d modules.' % loader.total)
# handle command line options first
if len(argv) > 1:
parse_cmd.parse(argv, loader)
# menus
main_menu = [
'Poisoners', 'DoS Attacks', 'Sniffers', 'Scanners', 'Parameter',
'Services', 'Attacks', 'Sessions'
]
running = True
choice = -1
while running:
util.header()
choice = util.print_menu(main_menu)
if choice == 0:
# check if they've got running sessions!
cnt = stream.get_session_count()
if cnt > 0:
display = color.B_YELLOW + 'You have %d sessions running. ' + \
'Are you sure? ' + color.B_GREEN + '[' + color.B_YELLOW + \
'Y' + color.B_GREEN + '/' + color.B_YELLOW + 'n' + \
color.B_GREEN + '] ' + color.END
choice = raw_input(display % cnt)
if 'y' in choice.lower() or choice == '':
util.Msg('Shutting all sessions down...')
stream.stop_session('all', -1)
running = False
else:
util.debug("Exiting with session count: %d" % (cnt))
util.Msg("Exiting...")
running = False
# remove zarp temporary directory
util.init_app('rm -fr /tmp/.zarp/')
# recheck that all sessions are down
cnt = stream.get_session_count()
if cnt <= 0:
# some libs dont clean up their own threads, so
# we need to hard quit those to avoid hanging; FIXME
_exit(1)
elif choice == 1:
while True:
choice = util.print_menu([x().which for x in loader.poison])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.poison):
continue
else:
stream.initialize(loader.poison[choice - 1])
elif choice == 2:
while True:
choice = util.print_menu([x().which for x in loader.dos])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.dos):
continue
else:
stream.initialize(loader.dos[choice - 1])
elif choice == 3:
while True:
choice = util.print_menu([x().which for x in loader.sniffers])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.sniffers):
continue
else:
stream.initialize(loader.sniffers[choice - 1])
elif choice == 4:
while True:
choice = util.print_menu([x().which for x in loader.scanner])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.scanner):
continue
else:
stream.initialize(loader.scanner[choice - 1])
elif choice == 5:
while True:
choice = util.print_menu([x().which for x in loader.parameter])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.parameter):
continue
else:
stream.initialize(loader.parameter[choice - 1])
elif choice == 6:
while True:
choice = util.print_menu([x().which for x in loader.services])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.services):
continue
else:
stream.initialize(loader.services[choice - 1])
elif choice == 7:
while True:
choice = util.print_menu([x().which for x in loader.attacks])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.attacks):
continue
else:
stream.initialize(loader.attacks[choice - 1])
elif choice == 8:
session_manager.menu()
elif choice == -1:
pass
def print_trainable_variables(self):
for v in self.get_trainable_variables():
util.header('- %s (%d parameters)' % (v.name, v.get_value().size))
util.header('Total: %d parameters' % (self.get_num_params(),))
def main():
""" Zarp entry point
"""
# set up configuration
config.initialize()
# set up database
database.initialize()
# load modules
loader = LoadedModules()
loader.load()
Msg("Loaded %d modules." % loader.total)
# handle command line options first
if len(argv) > 1:
parse_cmd.parse(argv, loader)
# menus
main_menu = ["Poisoners", "DoS Attacks", "Sniffers", "Scanners", "Parameter", "Services", "Attacks", "Sessions"]
running = True
choice = -1
while running:
header()
choice = print_menu(main_menu)
if choice == 0:
# check if they've got running sessions!
cnt = stream.get_session_count()
if cnt > 0:
choice = raw_input(
color.YELLOW + "You have %d sessions running. Are you sure? [Y/n] " % cnt + color.END
)
if "y" in choice.lower() or choice == "":
Msg("Shutting all sessions down...")
stream.stop_session("all", -1)
running = False
# recheck that all sessions are down
cnt = stream.get_session_count()
if cnt <= 0:
# some libs dont clean up their own threads, so
# we need to hard quit those to avoid hanging; FIXME
_exit(1)
else:
debug("Exiting with session count: %d" % (cnt))
Msg("Exiting...")
running = False
elif choice == 1:
while True:
choice = print_menu([x().which for x in loader.poison])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.poison):
continue
else:
stream.initialize(loader.poison[choice - 1])
elif choice == 2:
while True:
choice = print_menu([x().which for x in loader.dos])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.dos):
continue
else:
stream.initialize(loader.dos[choice - 1])
elif choice == 3:
while True:
choice = print_menu([x().which for x in loader.sniffers])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.sniffers):
continue
else:
stream.initialize(loader.sniffers[choice - 1])
elif choice == 4:
while True:
choice = print_menu([x().which for x in loader.scanner])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.scanner):
continue
else:
stream.initialize(loader.scanner[choice - 1])
elif choice == 5:
while True:
choice = print_menu([x().which for x in loader.parameter])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.parameter):
continue
else:
stream.initialize(loader.parameter[choice - 1])
elif choice == 6:
while True:
choice = print_menu([x().which for x in loader.services])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.services):
continue
else:
stream.initialize(loader.services[choice - 1])
elif choice == 7:
while True:
choice = print_menu([x().which for x in loader.attacks])
if choice == 0:
break
elif choice == -1:
pass
elif choice > len(loader.attacks):
continue
else:
stream.initialize(loader.attacks[choice - 1])
elif choice == 8:
session_manager.menu()
elif choice == -1:
pass
