def __init__(self, pkts_window_size=10, single_csv=True):
self.pkts_window_size = pkts_window_size
assert self.pkts_window_size >= 1, "Valore per la finestra non valido"
self.single_csv = single_csv
assert (self.single_csv is
True) or (self.single_csv is
False), "Valore non valido per il flag single_csv"
self.featuresCalc = FeaturesCalc(flow_type="malware",
min_window_size=pkts_window_size)
ip_to_ignore = ["127.0.0.1"]
self.filter_1 = PacketFilter(ip_whitelist_filter=[],
ip_blacklist_filter=ip_to_ignore,
TCP=True)
self.filter_2 = PacketFilter(ip_whitelist_filter=[],
ip_blacklist_filter=ip_to_ignore,
UDP=True)
self.filter_3 = PacketFilter(ip_whitelist_filter=[],
ip_blacklist_filter=ip_to_ignore,
ICMP=True)
self.filters = [self.filter_1, self.filter_2, self.filter_3]
if (self.single_csv):
self.csv = CSV(file_name="features")
self.csv.create_empty_csv()
self.csv.add_row(self.featuresCalc.get_features_name())
class TestCSV(unittest.TestCase):
def setUp(self):
self.c = CSV()
self.f1 = self.c.openFile('f1.csv', 'rt')
self.f2 = self.c.openFile('f2.csv', 'rt')
def testFileNotFoundError(self):
self.assertRaises(FileNotFoundError, self.c.openFile, "f3.csv", "rt")
def testSniffDialect(self):
assert(self.c.sniffDialect(self.f1).delimiter == ';')
def testReadFile(self):
list = self.c.readFile(self.f1,self.c.sniffDialect(self.f1))
assert(list==[['1', '2', '3', '5', '']])
def testWriteFile(self):
out = self.c.openFile('o.csv', "wt")
self.c.writeFile(out, [['1', '2', '3', '5', '']],'Pipe','|')
out = self.c.openFile('o.csv', "rt")
l2 = self.c.readFile(out,self.c.sniffDialect(out))
print(l2)
assert(l2==[['1', '2', '3', '5', ''],[]])
def testCloseFile(self):
self.assertRaises(AttributeError, self.c.closeFile, "f3.csv")
def __init__(self, depth, sampleSize, keyword):
self.depth = depth
self.contents = []
self.crawlContents = []
self.titles = []
self.linksCrawled = []
self.datecreated = {}
self.keyword = keyword
self.sampleSize = sampleSize
self.voidedTitles = ["BBC Homepage", "Sign in"]
self.results = {}
self.knownlinks = [".mp4"]
today = str(date.today())
self.csvFile = CSV(keyword + "," + today + ".csv",
["Word", "Tag", "Weight", "Link", "DateCreated"])
def __init__(self,split):
# self.coco=MSCOCO(split)
self.map_path = 'map.csv'
self.csv_path = 'detections.csv'
self.coco=CSV(self.csv_path, self.map_path)
self.data_rng = cfg.data_rng
self.num_image = len(self.coco.get_all_img())
self.categories = cfg.categories
self.input_size = cfg.input_size
self.output_size = cfg.output_sizes[0]
self.border = cfg.border
#self.lighting = cfg.lighting
self.rand_crop = cfg.rand_crop
print(self.rand_crop)
self.rand_color = cfg.rand_color
self.rand_scales = cfg.rand_scales
self.gaussian_bump = cfg.gaussian_bump
self.gaussian_iou = cfg.gaussian_iou
self.gaussian_rad = cfg.gaussian_radius
def start():
csv = CSV(args)
database = Database(csv.get_schema(), csv.get_table_name())
if not args.file:
print('Please include a filename using the -f flag')
try:
print('Inserting Entries')
if args.copy:
database.copy_csv()
else:
for entry in csv.get_entries():
database.insert(entry)
print('\033[92mDone Inserting\033[0m')
except (Exception) as e:
print(e)
print('Failed to Insert Data')
# When Finished, Disconnect
finally:
print('done')
database.close_connection()
def tokenizing(csv_import_path, csv_export_path):
csv_obj = CSV(csv_import_path)
csv_data = csv_obj.get_data()
sentence_arr = []
for row in csv_data:
for cell in row:
sentence_arr.append(cell)
tokenizer = MeCabTokenizer(tagger='-Ochasen')
output_arr = []
stop_words = ['。', '、', '･']
for sentence in sentence_arr:
tokens = tokenizer.parse_to_node(sentence)
surface = []
while tokens:
if tokens.surface and tokens.surface not in stop_words:
surface.append(tokens.surface)
tokens = tokens.next
if len(surface) > 0:
output_arr.append([sentence, " ".join(surface)])
csv_obj.export(csv_export_path, output_arr)
def _parserCSVFile(self):
# I get first argument passed in command line, in this case,
# the PATH of CSV file
csvFile = File(self._file)
# If file does not exist, then print a message warning the-
# user and stop running program
if csvFile.exists() == False:
print("File not found: %s", csvFile._file)
sys.exit(os.EX_SOFTWARE)
# Parser CSV File
self._log.logger.info('Parser CSV File')
return CSV(csvFile._file)
def legitimate_features():
folder_name = "Pcaps_Legitimate"
flow_type = "legitimate"
if (self.featuresCalc.get_flow_type() == flow_type):
pass
else:
self.featuresCalc.set_flow_type(flow_type)
for filter in self.filters:
filter.set_ip_whitelist_filter([])
for pcap in glob.glob(folder_name + "/" + "*.pcap"):
if (self.single_csv):
csv = self.csv
else:
pcap_name = pcap.split("/")
pcap_name = pcap_name[len(pcap_name) - 1].replace(
".pcap", "")
csv = CSV(file_name=pcap_name,
folder_name="Legitimate_Features")
csv.create_empty_csv()
csv.add_row(self.featuresCalc.get_features_name())
array_of_pkts = []
filter_res = []
print("\nCalculation of features " + pcap + "\n")
try:
pkts = rdpcap(pcap)
except:
sys.exit()
for pkt in pkts:
for filter in self.filters:
if (filter.check_packet_filter(pkt)):
filter_res.append(True)
else:
filter_res.append(False)
if (True in filter_res):
array_of_pkts.append(pkt)
if (len(array_of_pkts) >=
self.featuresCalc.get_min_window_size()):
features = self.featuresCalc.compute_features(
array_of_pkts)
csv.add_row(features)
array_of_pkts.clear()
filter_res.clear()
def malware_features():
folder_name = "Pcaps_Malware"
flow_type = "malware"
if (self.featuresCalc.get_flow_type() == flow_type):
pass
else:
self.featuresCalc.set_flow_type(flow_type)
for pcap in glob.glob(folder_name + "/" + "*.pcap"):
if (self.single_csv):
csv = self.csv
else:
pcap_name = pcap.split("/")
pcap_name = pcap_name[len(pcap_name) - 1].replace(
".pcap", "")
csv = CSV(file_name=pcap_name,
folder_name="Malware_Features")
csv.create_empty_csv()
csv.add_row(self.featuresCalc.get_features_name())
array_of_pkts = []
print("\nCalcolo features di " + pcap + "\n")
attacker = AttackerCalc(pcap=pcap)
ip_to_consider = attacker.compute_attacker()
for filter in self.filters:
filter.set_ip_whitelist_filter(ip_to_consider)
pkts = rdpcap(pcap)
filter_res = []
for pkt in pkts:
for filter in self.filters:
if (filter.check_packet_filter(pkt)):
filter_res.append(True)
else:
filter_res.append(False)
if (True in filter_res):
array_of_pkts.append(pkt)
if (len(array_of_pkts) >=
self.featuresCalc.get_min_window_size()):
features = self.featuresCalc.compute_features(
array_of_pkts)
csv.add_row(features)
array_of_pkts.clear()
filter_res.clear()
def full_report(self):
N = self.adsorbophore_db.session.query(SQL_Adsorbophore).count()
del self._adsorbophores
del self._active_sites
csv = CSV('full_report.csv')
csv.set_headings("rank","N_sites", "N_unq_MOFs", "rmsd_error", "av_elstat", "stdev_elstat", "av_vdw", "stdev_vdw")
for rank in range(N):
adsorbophore = self.adsorbophore_from_sql(rank)
nsites = len(adsorbophore.active_sites)
mofs = {}
for site in adsorbophore.active_sites:
mofname = '.'.join(site.name.split('.')[:-1])
mofs.setdefault(mofname, 0)
mofs[mofname] += 1
error, vdw_mean, vdw_std, el_mean, el_std = self.obtain_error(rank)
csv.add_data(**{'rank.1': rank, 'N_sites.1':nsites, 'N_unq_MOFs.1':len(mofs.keys()),
'rmsd_error.1':error, 'av_elstat.1':el_mean, 'stdev_elstat.1':el_std,
'av_vdw.1':vdw_mean, 'stdev_vdw.1':vdw_std})
csv.write()
class Image_data():
def __init__(self,split):
# self.coco=MSCOCO(split)
self.map_path = 'map.csv'
self.csv_path = 'detections.csv'
self.coco=CSV(self.csv_path, self.map_path)
self.data_rng = cfg.data_rng
self.num_image = len(self.coco.get_all_img())
self.categories = cfg.categories
self.input_size = cfg.input_size
self.output_size = cfg.output_sizes[0]
self.border = cfg.border
#self.lighting = cfg.lighting
self.rand_crop = cfg.rand_crop
print(self.rand_crop)
self.rand_color = cfg.rand_color
self.rand_scales = cfg.rand_scales
self.gaussian_bump = cfg.gaussian_bump
self.gaussian_iou = cfg.gaussian_iou
self.gaussian_rad = cfg.gaussian_radius
def read_from_disk(self,queue):
# allocating memory
max_tag_len = 128
image = np.zeros((self.input_size[0], self.input_size[1],3), dtype=np.float32)
heatmaps_tl = np.zeros((self.output_size[0], self.output_size[1],self.categories), dtype=np.float32)
heatmaps_br = np.zeros((self.output_size[0], self.output_size[1],self.categories), dtype=np.float32)
offsets_tl = np.zeros((max_tag_len, 2), dtype=np.float32)
offsets_br = np.zeros((max_tag_len, 2), dtype=np.float32)
tags_tl = np.zeros((max_tag_len), dtype=np.int64)
tags_br = np.zeros((max_tag_len), dtype=np.int64)
tags_mask = np.zeros((max_tag_len), dtype=np.float32)
boxes = np.zeros((max_tag_len,4), dtype=np.int64)
ratio = np.ones((max_tag_len,2), dtype=np.float32)
tag_lens = 0
# reading image
image=self.coco.read_img(queue[0])
# reading detections
detections = self.coco.detections(queue[0])
# cropping an image randomly
if self.rand_crop:
image, detections = random_crop(image, detections, self.rand_scales, self.input_size, border=self.border)
else:
image, detections = full_image_crop(image, detections)
image, detections = resize_image(image, detections, self.input_size)
detections = clip_detections(image, detections)
width_ratio = self.output_size[1] / self.input_size[1]
height_ratio = self.output_size[0] / self.input_size[0]
# flipping an image randomly
if np.random.uniform() > 0.5:
image[:] = image[:, ::-1, :]
width = image.shape[1]
detections[:, [0, 2]] = width - detections[:, [2, 0]] - 1
image = image.astype(np.float32) / 255.
# if rand_color:
# color_jittering_(data_rng, image)
# if lighting:
# lighting_(data_rng, image, 0.1, db.eig_val, db.eig_vec)
#normalize_(image, self.coco.mean, self.coco.std)
for ind, detection in enumerate(detections):
category = int(detection[-1]) - 1
xtl_ori, ytl_ori = detection[0], detection[1]
xbr_ori, ybr_ori = detection[2], detection[3]
fxtl = (xtl_ori * width_ratio)
fytl = (ytl_ori * height_ratio)
fxbr = (xbr_ori * width_ratio)
fybr = (ybr_ori * height_ratio)
xtl = int(fxtl)
ytl = int(fytl)
xbr = int(fxbr)
ybr = int(fybr)
if self.gaussian_bump:
width = detection[2] - detection[0]
height = detection[3] - detection[1]
width = math.ceil(width * width_ratio)
height = math.ceil(height * height_ratio)
if self.gaussian_rad == -1:
radius = gaussian_radius((height, width), self.gaussian_iou)
radius = max(0, int(radius))
else:
radius = self.gaussian_rad
draw_gaussian(heatmaps_tl[:,:,category], [xtl, ytl], radius)
draw_gaussian(heatmaps_br[:,:,category], [xbr, ybr], radius)
else:
heatmaps_tl[ytl, xtl, category] = 1
heatmaps_br[ybr, xbr, category] = 1
tag_ind = tag_lens
offsets_tl[tag_ind, :] = [fxtl - xtl, fytl - ytl]
offsets_br[tag_ind, :] = [fxbr - xbr, fybr - ybr]
tags_tl[tag_ind] = ytl * self.output_size[1] + xtl
tags_br[tag_ind] = ybr * self.output_size[1] + xbr
boxes[tag_ind] = [xtl_ori,ytl_ori,xbr_ori,ybr_ori]
ratio[tag_ind] = [width_ratio,height_ratio]
tag_lens += 1
tags_mask[:tag_lens] = 1
return image, tags_tl, tags_br,heatmaps_tl, heatmaps_br, tags_mask, offsets_tl, offsets_br,boxes,ratio
def get_single_data(self,queue):
images, tags_tl, tags_br,heatmaps_tl, heatmaps_br, tags_mask, offsets_tl, offsets_br,boxes,ratio=tf.py_func(self.read_from_disk,[queue],
[tf.float32,tf.int64,tf.int64,tf.float32,tf.float32,tf.float32,tf.float32,tf.float32,tf.int64,tf.float32])
return images, tags_tl, tags_br,heatmaps_tl, heatmaps_br, tags_mask, offsets_tl, offsets_br,boxes,ratio
def inupt_producer(self):
quene_train=tf.train.slice_input_producer([self.coco.get_all_img()],shuffle=True)
self.images, self.tags_tl, self.tags_br,self.heatmaps_tl, self.heatmaps_br, self.tags_mask, self.offsets_tl, self.offsets_br,self.boxes,self.ratio=self.get_single_data(quene_train)
def get_batch_data(self,batch_size):
images, tags_tl, tags_br,heatmaps_tl, heatmaps_br, tags_mask, offsets_tl, offsets_br,boxes,ratio=tf.train.shuffle_batch([self.images,
self.tags_tl, self.tags_br,self.heatmaps_tl, self.heatmaps_br, self.tags_mask, self.offsets_tl, self.offsets_br,self.boxes,self.ratio],
batch_size=batch_size,shapes=[(self.input_size[0], self.input_size[1],3),(128),(128),
(self.output_size[0], self.output_size[1],self.categories),(self.output_size[0], self.output_size[1],self.categories),
(128),(128,2),(128,2),(128,4),(128,2)],capacity=100,min_after_dequeue=batch_size,num_threads=16)
return images, tags_tl, tags_br,heatmaps_tl, heatmaps_br, tags_mask, offsets_tl, offsets_br,boxes,ratio
"""
Created on Mon Feb 10 22:26:14 2020
@author: Asif
"""
from GCM import GCM
from formats import FORMAT
from features import FEATURE
from CSV import CSV
import numpy as np
import os
for filename in os.listdir(os.getcwd()+"\\raw_data"):
#print(filename)
csv = CSV(filename)
csv.generate_csv()
for filename in os.listdir(os.getcwd()+"\\csv"):
#print(filename)
frmt = FORMAT(filename)
frmt.format()
ctr = 0
data = np.array([[]])
for filename in os.listdir(os.getcwd()+"\\graph"):
ctr = ctr + 1
print(filename)
feature = FEATURE(filename)
dt = feature.generate_feature()
print(dt.shape)
def __init__(self, source, log_file=None, debug=False):
CSV.__init__(self, source, log_file, debug)
s.z,s.zErr_noqso,cModelMag_r,p.cModelMagErr_r,p.cModelMag_u,p.cModelMagErr_u,p.petroRad_r,p.petroRadErr_r,p.modelMag_r,p.modelMag_u, p.petroMag_r ,p.petroMagErr_r, p.petroMagErr_u, p.petroMag_u, p.petroR90_r
FROM PhotoObj AS p
JOIN SpecObj AS s ON s.bestobjid = p.objid
WHERE
(s.BOSS_TARGET1 & 1) != 0 and s.bossprimary = 1 and ZWARNING_NOQSO = 0 and s.z between 0.002 and 0.5 and s.plateID >= 10324
''' # redoanlaod with new query
from CSV import CSV
from calc_kcor import calc_kcor
from astropy.cosmology import Planck15 as cosmo
import numpy as np
import matplotlib.pyplot as pl
SDSS = CSV('data_all_2')
SDSS_data = SDSS.read_all()
# -1 because of column titles
print(SDSS.row_count_data)
calculated = np.zeros((SDSS.row_count_data, 3)) #add area
print('read data')
for index, row in enumerate(SDSS_data):
if index % 1000 == 0:
pass
z, zErr_noqso, cModelMag_r, cModelMagErr_r, cModelMag_u, cModelMagErr_u, petroRad_r, petroRadErr_r, modelMag_r, modelMag_u, petroMag_r, petroMagErr_r, petroMagErr_u, petroMag_u, Pr90 = row
dist = cosmo.luminosity_distance(z).value # in Mpc
k_corr = calc_kcor('r', z, 'u - r', (petroMag_u - petroMag_r))
def __init__(self):
self.my_csv = CSV()
self.error = Error()
def legitimate_train(line):
global legitimate_train_nb
if (check_if_already_trained(line) is False):
flow_type = "legitimate"
filter_1.set_ip_whitelist_filter([])
filter_2.set_ip_whitelist_filter([])
filter_3.set_ip_whitelist_filter([])
featuresCalc = FeaturesCalc(flow_type=flow_type, min_window_size=5)
csv = CSV(file_name="features_" + flow_type, folder_name="Features")
csv.create_empty_csv()
csv.add_row(featuresCalc.get_features_name())
argument = {
"features_calc": featuresCalc,
"packets": [],
'filter': [filter_1, filter_2, filter_3],
'csv_obj': csv
}
sniffer = Sniffer(iface_sniffer,
callback_prn=callback_sniffer,
callback_prn_kwargs=argument)
sniffer.start()
while (sniffer.get_started_flag() is False):
pass
try:
sender = Sender(iface_sender,
fast=False,
verbose=False,
time_to_wait=10)
sender.send(lg.get_folder_name() + "/" + line)
sniffer.stop()
except Exception as e:
print(e)
csv.close_csv()
env.set_csv(csv.get_folder_name() + "/" + csv.get_current_file_name())
agent.train_agent(steps=csv.get_number_of_rows() - 1,
log_interval=csv.get_number_of_rows() - 1,
verbose=2,
nb_max_episode_steps=csv.get_number_of_rows() - 1)
legitimate_train_nb -= 1
trained_file.write(line + "\n")
else:
print("\nPcap gia' utilizzato in passato. Saltato.\n")
subpath = os.path.join(args_opt.rootDir,'output',re.sub(r'_epoch_size_[0-9.]*','',key))
if os.path.exists(subpath):
import shutil
# if exited, remove it!
shutil.rmtree(subpath)
os.makedirs(subpath,exist_ok=True)
strategy_path = os.path.join(subpath,'strategy')
csv_abspath = re.sub(r'_epoch_size_[0-9.]*_device_num_\d', '-log.csv', key)
csv_path = os.path.join(subpath,csv_abspath)
with open(strategy_path,'w') as fp:
for item in value['strategy']:
fp.write(item+'\n')
if 'step' in value['performance'].keys() and\
'speed' in value['performance'].keys() and\
'throughput' in value['performance'].keys() and\
'loss' in value['performance'].keys():
csvfile = CSV(
path=csv_path,
columns=['step','step_cost_time(ms)','samples/second','loss'],
values=[
value['performance']['step'],
value['performance']['speed'],
value['performance']['throughput'],
value['performance']['loss'],
])
csvfile.dump()
# print(blocks[1])
def _build_structures(self):
"""Pass the sbu combinations to a MOF building algorithm."""
run = Generate(self.options, self.sbu_pool)
# generate the combinations of SBUs to build
if self.options.sbu_combinations:
combinations = run.combinations_from_options()
else:
# remove SBUs if not listed in options.organic_sbus or options.metal_sbus
combinations = run.generate_sbu_combinations()
csvinfo = CSV(name='%s_info'%(self.options.jobname))
csvinfo.set_headings('topology', 'sbus', 'edge_count', 'time', 'space_group', 'net_charge')
csvinfo.set_headings('edge_length_err', 'edge_length_std', 'edge_angle_err', 'edge_angle_std')
self.options.csv = csvinfo
# generate the MOFs.
if self.options.count_edges_along_lattice_dirs:
lattfile = open("edge_counts.csv","w")
lattfile.writelines("topology,Na,Nb,Nc\n")
inittime = time()
for combo in combinations:
node_degree = [i.degree for i in set(combo)]
node_lin = [i.linear for i in set(combo)]
degree = sorted([j for i, j in zip(node_lin, node_degree) if not i])
# find degrees of the sbus in the combo
if not self._topologies:
warning("No topologies found! Exiting.")
Terminate()
debug("Trying "+self.combo_str(combo))
for top, graph in self._topologies.items():
if self.options.use_builds:
try:
build = self._stored_builds[top]
except:
build = Build(self.options)
build.net = (top, graph, self._topologies.voltages[top])
else:
build = Build(self.options)
build.net = (top, graph, self._topologies.voltages[top])
build.sbus = list(set(combo))
#build.get_automorphisms()
if self.options.count_edges_along_lattice_dirs:
info("Computing Edge lengths along each lattice direction for %s"%(top))
n = Net(graph)
n.voltage = self._topologies.voltages[top]
n.simple_cycle_basis()
n.get_lattice_basis()
n.get_cocycle_basis()
edge_str = n.print_edge_count()
lattfile.writelines("%s,%s\n"%(top, edge_str))
elif self.options.show_barycentric_net_only:
info("Preparing barycentric embedding of %s"%(top))
#print("CHECK", top, build.net.graph.number_of_selfloops())
self._check_barycentric_embedding(graph, self._topologies.voltages[top])
else:
if build.check_net:
# check node incidence
if build.met_met_bonds and run.linear_sbus_exist and not run.linear_in_combo(combo):
# add linear organics
debug("Metal-type nodes attached to metal-type nodes. "+
"Attempting to insert 2-c organic SBUs between these nodes.")
for comb in run.yield_linear_org_sbu(combo):
if self.options.use_builds:
try:
build = self._stored_builds[top]
except:
build = Build(self.options)
else:
build = Build(self.options)
build.sbus = list(set(comb))
build.net = (top, graph, self._topologies.voltages[top])
self.embed_sbu_combo(top, comb, build)
elif build.met_met_bonds and run.linear_in_combo(combo):
self.embed_sbu_combo(top, combo, build)
elif build.met_met_bonds and not run.linear_sbus_exist:
debug("Metal-type nodes are attached to metal-type nodes. "+
"No linear SBUs exist in database, so the structure "+
"will have metal - metal SBUs joined")
self.embed_sbu_combo(top, combo, build)
elif not build.met_met_bonds:
self.embed_sbu_combo(top, combo, build)
else:
debug("Net %s does not support the same"%(top)+
" connectivity offered by the SBUs")
if self.options.count_edges_along_lattice_dirs:
lattfile.close()
finaltime = time() - inittime
info("Topcryst completed after %f seconds"%finaltime)
if self.options.get_run_info:
info("Writing run information to %s"%self.options.csv.filename)
self.options.csv.write()
if self.options.store_net and self._stored_nets:
info("Writing all nets to nets_%s.pkl"%self.options.jobname)
f = open("nets_%s.pkl"%self.options.jobname, 'wb')
p = pickle.dump(self._stored_nets, f)
f.close()
Terminate()
running = False
break
# Main()
if __name__ == '__main__':
'''
Main iteration menu is given to the users the the options to add an entry,
look up and old entry, or exit the program. Looking up an old entry yields even
more options on how to look up the entry.
'''
# Run set to True, create other class instances
run = True
error = Error()
entry = Entry()
csv = CSV()
# While to correct for mistakes
while run:
# Print work log menu with options, check for errors and clear screen
print("Welcome to the work log! Options are listed below!")
print(
"1: Add new entry\n2: Lookup previous entry\n3: Exit the program")
main_choice = error.error(1, 4)
clear_screen()
# Choice is to add Entry, use entry.add() to add
if main_choice == 1:
entry.add()
clear_screen()
# Choice is to look up Entry
elif main_choice == 2:
# Print menu of choices to look up by, check for errors
def obtain_rdfs(self, rank):
"""Return the radial distribution functions of the cliques in the original MOFs,
with the cliques cut-out."""
adsorbophore = self.adsorbophore_from_sql(rank)
nconfig = 0 # configurations counted
nparticles = {} # number of particles counted
distances = {} # keep distances in lists.
densities = {}
for site in adsorbophore.active_sites:
indices = [i.index for i in site.indices]
act_site = self.active_site_from_sql(site.name)
ads_atoms = [act_site.atoms[i] for i in indices]
site_eng = act_site.vdweng + act_site.eleng
if site_eng <= self.options.en_max and site_eng >= self.options.en_min:
mofpath = act_site.mofpath
# get mof from cif
mofname = os.path.split(mofpath)[-1][:-4]
mof = Structure(mofname)
mof.from_cif(mofpath)
# get atoms to cut out of mof
cut_inds = [atom.mof_id for atom in ads_atoms]
cut_coords = np.array([np.array((atom.x, atom.y, atom.z)) for atom in ads_atoms])
# get co2
co2 = self.return_co2_array(act_site)
rdf_centre = co2[0] # or self.centre_of_atoms(np.array([np.array([a.x, a.y, a.z]) for a in ads_atoms]))
# compute the minimum image convention of a supercell around the rdf centre.
# cut out the atoms in the active site SQL_ActiveSiteAtoms.mof_id
for atom in ads_atoms:
rdf_centre = np.array([atom.x, atom.y, atom.z])
original_indices, coordinates = self.min_img(mof, rdf_centre, (3,3,3), cut_coords)#cut_inds)
dists = distance.cdist(np.column_stack(rdf_centre), coordinates)
# debug
#f = open('debug.xyz', 'a')
#f.writelines("%i\n%s\n"%(len(coordinates)+1, "test"))
#for id, coord in enumerate(coordinates):
# orig_ind = original_indices[id]
# element = mof.atoms[orig_ind].type
# f.writelines("%s %9.5f %9.5f %9.5f\n"%(element, coord[0], coord[1], coord[2]))
#f.writelines("%s %9.5f %9.5f %9.5f\n"%("As", co2[0][0], co2[0][1], co2[0][2]))
#f.close()
nconfig += 1
for (x,y), val in np.ndenumerate(dists):
orig_ind = original_indices[y]
element = mof.atoms[orig_ind].uff_type
distances.setdefault(element, []).append(val)
nparticles.setdefault(element, 0)
nparticles[element] += 1
# add number densities from unit cell.
counts = {}
for atom in mof.atoms:
counts.setdefault(atom.uff_type, 0)
counts[atom.uff_type] += 1
for element, count in counts.items():
densities.setdefault(element, []).append(count/mof.cell.volume)
# compute RDFs by atom type? element? general?
for element, dis in distances.items():
rho = np.mean(densities[element])
hist, bin_edges = np.histogram(dis, bins=self.options.rdf_bins,
range=(0., self.options.rdf_dist))
dr = np.diff(bin_edges)[0]
norm = rho * float(nconfig)
#shell_volume = 4/3*pi*pow(r, 2)*dr
#rho, norm = 1., 1.
rdf = [hist[i]/norm/(4./3.*pi*(pow((i+0.5)*dr, 3) - pow((i-0.5)*dr, 3)))
for i in range(self.options.rdf_bins)]
Emax = self.options.en_max
Emin = self.options.en_min
el_name = '.'.join([j for j in element.split('_') if j])
csv = CSV("RDF_rank%i_%s_Emax_%0.2f_Emin_%0.2f"%(rank, el_name, Emax, Emin))
csv.set_headings("r", "g(r)")
for i, val in enumerate(rdf):
csv.add_data(**{"r.1":i*dr, "g(r).1":val})
csv.write()
def setUp(self):
self.c = CSV()
self.f1 = self.c.openFile('f1.csv', 'rt')
self.f2 = self.c.openFile('f2.csv', 'rt')
class Crawler:
def __init__(self, depth, sampleSize, keyword):
self.depth = depth
self.contents = []
self.crawlContents = []
self.titles = []
self.linksCrawled = []
self.datecreated = {}
self.keyword = keyword
self.sampleSize = sampleSize
self.voidedTitles = ["BBC Homepage", "Sign in"]
self.results = {}
self.knownlinks = [".mp4"]
today = str(date.today())
self.csvFile = CSV(keyword + "," + today + ".csv",
["Word", "Tag", "Weight", "Link", "DateCreated"])
def start(self):
urlsToCrawl = []
json = CustomJSONFormatter(self.keyword)
articles = json.getValues("articles")
for item in articles:
urlsToCrawl.append(item["url"])
self.contents.append(item["description"])
self.titles.append(item["title"])
self.datecreated[item["url"]] = item["publishedAt"]
# for item in urlsToCrawl:
# # print(item)
# self.crawl(item, 0 , self.keyword, self.datecreated[item])
print("")
print("Links crawled:")
posDict = {}
# [(), ""] : weight
for i in range(len(self.titles)):
item = self.titles[i]
url = urlsToCrawl[i]
if "\\" not in item:
tempDictForTitle = POS.POS(item)
keys = tempDictForTitle.keys()
for i in keys:
urlList = (i, url)
posDict[urlList] = tempDictForTitle[i]
for i in range(len(self.contents)):
title = self.titles[i]
datecreated = self.datecreated[url]
item2 = self.contents[i]
url = urlsToCrawl[i]
listOfKeysToUpdateUrl = []
if item2 is not None:
for node in item2:
if not isinstance(node, str):
sentence = node.find_all(text=True)
for minisentence in sentence:
tempDictForContents = POS.POS(minisentence, True)
keys = tempDictForContents.keys()
for i in keys:
listUrl = (i, url)
if listUrl not in posDict:
posDict[listUrl] = tempDictForContents[i]
else:
posDict[listUrl] = posDict[
listUrl] + tempDictForContents[i]
listOfKeysToUpdateUrl.append(listUrl)
break
elif isinstance(node, str):
tempDictForContents = POS.POS(item2)
keys = tempDictForContents.keys()
for i in keys:
listUrl = (i, url)
if listUrl not in posDict:
posDict[listUrl] = tempDictForContents[i]
else:
posDict[listUrl] = posDict[
listUrl] + tempDictForContents[i]
listOfKeysToUpdateUrl.append(listUrl)
break
self.crawlContents = self.contents
keys = posDict.keys()
for i in keys:
self.crawl(i[1], 0, self.keyword, posDict)
for i in keys:
print(i)
print(posDict[i])
print(self.datecreated[i[1]])
newDict = {}
newDict["Word"] = self.lowerCase(i[0][0])
newDict["Tag"] = i[0][1]
newDict["Weight"] = posDict[i]
newDict["Link"] = i[1]
date = self.datecreated[i[1]]
dates = date.split("T")
dates2 = dates[0].split("-")
newDict[
"DateCreated"] = dates2[2] + "/" + dates2[1] + "/" + dates2[0]
self.csvFile.push(newDict)
self.csvFile.save()
def getContents(self):
return self.contents
def lowerCase(self, sentence):
return sentence.lower()
def crawl(self, link, depth, keyword, posDict):
print("Depth of " + str(depth) + " : " + link)
if link in self.linksCrawled:
return
for item in self.linksCrawled:
if link[:100] == item[:100]:
return
if depth >= self.depth:
return
self.linksCrawled.append(link)
scraper = Scraper(link, keyword)
hyperLinkList = scraper.scrapeLinks(link)
articleDictionary = scraper.scrape('p')
print(articleDictionary)
for item in articleDictionary:
if item not in self.crawlContents:
self.crawlContents.append(item)
if isinstance(item, str):
tempDictForContents = POS.POS(item)
keys = tempDictForContents.keys()
for i in keys:
listUrl = (i, link)
if listUrl not in posDict:
posDict[listUrl] = tempDictForContents[i]
else:
posDict[listUrl] = posDict[
listUrl] + tempDictForContents[i]
print(item)
if len(self.crawlContents) + len(
self.contents) <= self.sampleSize and len(
articleDictionary) != 0 and len(
self.linksCrawled) <= self.sampleSize:
self.linksCrawled.append(link)
if hyperLinkList is not None:
for link in hyperLinkList:
if link not in self.linksCrawled:
if "ad" not in link:
_thread.start_new_thread(self.crawl, (
link,
depth + 1,
self.keyword,
posDict,
))
# self.crawl(link, depth + 1, self.keyword, posDict)
def getAbsoluteLink(self, link):
pathList = link.split("/")
protocol = pathList[0]
host = pathList[2]
return protocol + "//" + host
def _build_structures_from_top(self):
if not self._topologies:
warning("No topologies found!")
Terminate()
csvinfo = CSV(name='%s_info'%(self.options.jobname))
csvinfo.set_headings('topology', 'sbus', 'edge_count', 'time', 'space_group')
csvinfo.set_headings('edge_length_err', 'edge_length_std', 'edge_angle_err', 'edge_angle_std')
self.options.csv = csvinfo
run = Generate(self.options, self.sbu_pool)
inittime = time()
if self.options.count_edges_along_lattice_dirs:
lattfile = open("edge_counts.csv","w")
lattfile.writelines("topology,Na,Nb,Nc\n")
for top, graph in self._topologies.items():
if self.options.count_edges_along_lattice_dirs:
info("Computing Edge lengths along each lattice direction for %s"%(top))
n = Net(graph)
n.voltage = self._topologies.voltages[top]
n.simple_cycle_basis()
n.get_lattice_basis()
n.get_cocycle_basis()
edge_str = n.print_edge_count()
lattfile.writelines("%s,%s"%(top, edge_str))
elif self.options.show_barycentric_net_only:
info("Preparing barycentric embedding of %s"%(top))
self._check_barycentric_embedding(graph, self._topologies.voltages[top])
else:
build = Build(self.options)
build.net = (top, graph, self._topologies.voltages[top])
if self.options.sbu_combinations:
combinations = run.combinations_from_options()
else:
combinations = run.generate_sbu_combinations(incidence=build.net_degrees())
if not list(combinations):
debug("Net %s does not support the same"%(top)+
" connectivity offered by the SBUs")
for combo in combinations:
build.sbus = list(set(combo))
# check node incidence
if build.met_met_bonds and run.linear_sbus_exist:
# add linear organics
debug("Metal-type nodes attached to metal-type nodes. "+
"Attempting to insert 2-c organic SBUs between these nodes.")
for comb in run.yield_linear_org_sbu(combo):
build.sbus = list(set(comb))
self.embed_sbu_combo(top, comb, build)
elif build.met_met_bonds and not run.linear_sbus_exist:
debug("Metal-type nodes are attached to metal-type nodes. "+
"No linear SBUs exist in database, so the structure "+
"will have metal - metal SBUs joined")
self.embed_sbu_combo(top, combo, build)
else:
self.embed_sbu_combo(top, combo, build)
if self.options.count_edges_along_lattice_dirs:
lattfile.close()
finaltime = time() - inittime
info("Topcryst completed after %f seconds"%finaltime)
Terminate()
def _build_structures_from_top(self):
if not self._topologies:
warning("No topologies found!")
Terminate()
csvinfo = CSV(name='%s_info' % (self.options.jobname))
csvinfo.set_headings('topology', 'sbus', 'edge_count', 'time',
'space_group')
csvinfo.set_headings('edge_length_err', 'edge_length_std',
'edge_angle_err', 'edge_angle_std')
self.options.csv = csvinfo
run = Generate(self.options, self.sbu_pool)
inittime = time()
if self.options.count_edges_along_lattice_dirs:
lattfile = open("edge_counts.csv", "w")
lattfile.writelines("topology,Na,Nb,Nc\n")
for top, graph in self._topologies.items():
if self.options.count_edges_along_lattice_dirs:
info(
"Computing Edge lengths along each lattice direction for %s"
% (top))
n = Net(graph)
n.voltage = self._topologies.voltages[top]
n.simple_cycle_basis()
n.get_lattice_basis()
n.get_cocycle_basis()
edge_str = n.print_edge_count()
lattfile.writelines("%s,%s" % (top, edge_str))
elif self.options.show_barycentric_net_only:
info("Preparing barycentric embedding of %s" % (top))
self._check_barycentric_embedding(
graph, self._topologies.voltages[top])
else:
build = Build(self.options)
build.net = (top, graph, self._topologies.voltages[top])
if self.options.sbu_combinations:
combinations = run.combinations_from_options()
else:
combinations = run.generate_sbu_combinations(
incidence=build.net_degrees())
if not list(combinations):
debug("Net %s does not support the same" % (top) +
" connectivity offered by the SBUs")
for combo in combinations:
build.sbus = list(set(combo))
# check node incidence
if build.met_met_bonds and run.linear_sbus_exist:
# add linear organics
debug(
"Metal-type nodes attached to metal-type nodes. " +
"Attempting to insert 2-c organic SBUs between these nodes."
)
for comb in run.yield_linear_org_sbu(combo):
build.sbus = list(set(comb))
self.embed_sbu_combo(top, comb, build)
elif build.met_met_bonds and not run.linear_sbus_exist:
debug(
"Metal-type nodes are attached to metal-type nodes. "
+
"No linear SBUs exist in database, so the structure "
+ "will have metal - metal SBUs joined")
self.embed_sbu_combo(top, combo, build)
else:
self.embed_sbu_combo(top, combo, build)
if self.options.count_edges_along_lattice_dirs:
lattfile.close()
finaltime = time() - inittime
info("Topcryst completed after %f seconds" % finaltime)
Terminate()
def _sbu_report(self):
"""Compute the surface areas and report them to a .csv file."""
# WARNING - this assumes that SBUs with the same name but in
# different topologies are the same, and will take the last instance
met_sbus = {}
org_sbus = {}
for sbu in self.sbu_pool:
if sbu.is_metal:
met_sbus[sbu.name] = sbu
else:
org_sbus[sbu.name] = sbu
filename = os.path.join(self.options.job_dir,
self.options.jobname + ".SBU_report.csv")
report = CSV(name=filename)
report.set_headings("sbu_id")
if self.options.calc_sbu_surface_area:
report.set_headings("surface_area")
if self.options.calc_max_sbu_span:
report.set_headings("sbu_span")
# metal sbus first.
for name, sbu in met_sbus.items():
info("Computing data for %s"%name)
report.add_data(**{"sbu_id.1": sbu.identifier})
if self.options.calc_sbu_surface_area:
report.add_data(**{"surface_area.1": sbu.surface_area})
if self.options.calc_max_sbu_span:
report.add_data(**{"sbu_span.1":sbu.max_span})
# list organic SBUs second.
for name, sbu in org_sbus.items():
info("Computing data for %s"%name)
report.add_data(**{"sbu_id.1": sbu.identifier})
if self.options.calc_sbu_surface_area:
report.add_data(**{"surface_area.1": sbu.surface_area})
if self.options.calc_max_sbu_span:
report.add_data(**{"sbu_span.1": sbu.max_span})
report.write()
def _build_structures(self):
"""Pass the sbu combinations to a MOF building algorithm."""
run = Generate(self.options, self.sbu_pool)
# generate the combinations of SBUs to build
if self.options.sbu_combinations:
combinations = run.combinations_from_options()
else:
# remove SBUs if not listed in options.organic_sbus or options.metal_sbus
combinations = run.generate_sbu_combinations()
csvinfo = CSV(name='%s_info' % (self.options.jobname))
csvinfo.set_headings('topology', 'sbus', 'edge_count', 'time',
'space_group', 'net_charge')
csvinfo.set_headings('edge_length_err', 'edge_length_std',
'edge_angle_err', 'edge_angle_std')
self.options.csv = csvinfo
# generate the MOFs.
if self.options.count_edges_along_lattice_dirs:
lattfile = open("edge_counts.csv", "w")
lattfile.writelines("topology,Na,Nb,Nc\n")
inittime = time()
for combo in combinations:
node_degree = [i.degree for i in set(combo)]
node_lin = [i.linear for i in set(combo)]
degree = sorted(
[j for i, j in zip(node_lin, node_degree) if not i])
# find degrees of the sbus in the combo
if not self._topologies:
warning("No topologies found! Exiting.")
Terminate()
debug("Trying " + self.combo_str(combo))
for top, graph in self._topologies.items():
if self.options.use_builds:
try:
build = self._stored_builds[top]
except:
build = Build(self.options)
build.net = (top, graph,
self._topologies.voltages[top])
else:
build = Build(self.options)
build.net = (top, graph, self._topologies.voltages[top])
build.sbus = list(set(combo))
#build.get_automorphisms()
if self.options.count_edges_along_lattice_dirs:
info(
"Computing Edge lengths along each lattice direction for %s"
% (top))
n = Net(graph)
n.voltage = self._topologies.voltages[top]
n.simple_cycle_basis()
n.get_lattice_basis()
n.get_cocycle_basis()
edge_str = n.print_edge_count()
lattfile.writelines("%s,%s\n" % (top, edge_str))
elif self.options.show_barycentric_net_only:
info("Preparing barycentric embedding of %s" % (top))
#print("CHECK", top, build.net.graph.number_of_selfloops())
self._check_barycentric_embedding(
graph, self._topologies.voltages[top])
else:
if build.check_net:
# check node incidence
if build.met_met_bonds and run.linear_sbus_exist and not run.linear_in_combo(
combo):
# add linear organics
debug(
"Metal-type nodes attached to metal-type nodes. "
+
"Attempting to insert 2-c organic SBUs between these nodes."
)
for comb in run.yield_linear_org_sbu(combo):
if self.options.use_builds:
try:
build = self._stored_builds[top]
except:
build = Build(self.options)
else:
build = Build(self.options)
build.sbus = list(set(comb))
build.net = (
top, graph,
self._topologies.voltages[top])
self.embed_sbu_combo(top, comb, build)
elif build.met_met_bonds and run.linear_in_combo(
combo):
self.embed_sbu_combo(top, combo, build)
elif build.met_met_bonds and not run.linear_sbus_exist:
debug(
"Metal-type nodes are attached to metal-type nodes. "
+
"No linear SBUs exist in database, so the structure "
+ "will have metal - metal SBUs joined")
self.embed_sbu_combo(top, combo, build)
elif not build.met_met_bonds:
self.embed_sbu_combo(top, combo, build)
else:
debug("Net %s does not support the same" % (top) +
" connectivity offered by the SBUs")
if self.options.count_edges_along_lattice_dirs:
lattfile.close()
finaltime = time() - inittime
info("Topcryst completed after %f seconds" % finaltime)
if self.options.get_run_info:
info("Writing run information to %s" % self.options.csv.filename)
self.options.csv.write()
if self.options.store_net and self._stored_nets:
info("Writing all nets to nets_%s.pkl" % self.options.jobname)
f = open("nets_%s.pkl" % self.options.jobname, 'wb')
p = pickle.dump(self._stored_nets, f)
f.close()
Terminate()
class CreateFeaturesHandler():
def __init__(self, pkts_window_size=10, single_csv=True):
self.pkts_window_size = pkts_window_size
assert self.pkts_window_size >= 1, "Valore per la finestra non valido"
self.single_csv = single_csv
assert (self.single_csv is
True) or (self.single_csv is
False), "Valore non valido per il flag single_csv"
self.featuresCalc = FeaturesCalc(flow_type="malware",
min_window_size=pkts_window_size)
ip_to_ignore = ["127.0.0.1"]
self.filter_1 = PacketFilter(ip_whitelist_filter=[],
ip_blacklist_filter=ip_to_ignore,
TCP=True)
self.filter_2 = PacketFilter(ip_whitelist_filter=[],
ip_blacklist_filter=ip_to_ignore,
UDP=True)
self.filter_3 = PacketFilter(ip_whitelist_filter=[],
ip_blacklist_filter=ip_to_ignore,
ICMP=True)
self.filters = [self.filter_1, self.filter_2, self.filter_3]
if (self.single_csv):
self.csv = CSV(file_name="features")
self.csv.create_empty_csv()
self.csv.add_row(self.featuresCalc.get_features_name())
def compute_features(self):
def malware_features():
folder_name = "Pcaps_Malware"
flow_type = "malware"
if (self.featuresCalc.get_flow_type() == flow_type):
pass
else:
self.featuresCalc.set_flow_type(flow_type)
for pcap in glob.glob(folder_name + "/" + "*.pcap"):
if (self.single_csv):
csv = self.csv
else:
pcap_name = pcap.split("/")
pcap_name = pcap_name[len(pcap_name) - 1].replace(
".pcap", "")
csv = CSV(file_name=pcap_name,
folder_name="Malware_Features")
csv.create_empty_csv()
csv.add_row(self.featuresCalc.get_features_name())
array_of_pkts = []
print("\nCalcolo features di " + pcap + "\n")
attacker = AttackerCalc(pcap=pcap)
ip_to_consider = attacker.compute_attacker()
for filter in self.filters:
filter.set_ip_whitelist_filter(ip_to_consider)
pkts = rdpcap(pcap)
filter_res = []
for pkt in pkts:
for filter in self.filters:
if (filter.check_packet_filter(pkt)):
filter_res.append(True)
else:
filter_res.append(False)
if (True in filter_res):
array_of_pkts.append(pkt)
if (len(array_of_pkts) >=
self.featuresCalc.get_min_window_size()):
features = self.featuresCalc.compute_features(
array_of_pkts)
csv.add_row(features)
array_of_pkts.clear()
filter_res.clear()
def legitimate_features():
folder_name = "Pcaps_Legitimate"
flow_type = "legitimate"
if (self.featuresCalc.get_flow_type() == flow_type):
pass
else:
self.featuresCalc.set_flow_type(flow_type)
for filter in self.filters:
filter.set_ip_whitelist_filter([])
for pcap in glob.glob(folder_name + "/" + "*.pcap"):
if (self.single_csv):
csv = self.csv
else:
pcap_name = pcap.split("/")
pcap_name = pcap_name[len(pcap_name) - 1].replace(
".pcap", "")
csv = CSV(file_name=pcap_name,
folder_name="Legitimate_Features")
csv.create_empty_csv()
csv.add_row(self.featuresCalc.get_features_name())
array_of_pkts = []
filter_res = []
print("\nCalcolo features di " + pcap + "\n")
pkts = rdpcap(pcap)
for pkt in pkts:
for filter in self.filters:
if (filter.check_packet_filter(pkt)):
filter_res.append(True)
else:
filter_res.append(False)
if (True in filter_res):
array_of_pkts.append(pkt)
if (len(array_of_pkts) >=
self.featuresCalc.get_min_window_size()):
features = self.featuresCalc.compute_features(
array_of_pkts)
csv.add_row(features)
array_of_pkts.clear()
filter_res.clear()
malware_features()
legitimate_features()
def _sbu_report(self):
"""Compute the surface areas and report them to a .csv file."""
# WARNING - this assumes that SBUs with the same name but in
# different topologies are the same, and will take the last instance
met_sbus = {}
org_sbus = {}
for sbu in self.sbu_pool:
if sbu.is_metal:
met_sbus[sbu.name] = sbu
else:
org_sbus[sbu.name] = sbu
filename = os.path.join(self.options.job_dir,
self.options.jobname + ".SBU_report.csv")
report = CSV(name=filename)
report.set_headings("sbu_id")
if self.options.calc_sbu_surface_area:
report.set_headings("surface_area")
if self.options.calc_max_sbu_span:
report.set_headings("sbu_span")
# metal sbus first.
for name, sbu in met_sbus.items():
info("Computing data for %s" % name)
report.add_data(**{"sbu_id.1": sbu.identifier})
if self.options.calc_sbu_surface_area:
report.add_data(**{"surface_area.1": sbu.surface_area})
if self.options.calc_max_sbu_span:
report.add_data(**{"sbu_span.1": sbu.max_span})
# list organic SBUs second.
for name, sbu in org_sbus.items():
info("Computing data for %s" % name)
report.add_data(**{"sbu_id.1": sbu.identifier})
if self.options.calc_sbu_surface_area:
report.add_data(**{"surface_area.1": sbu.surface_area})
if self.options.calc_max_sbu_span:
report.add_data(**{"sbu_span.1": sbu.max_span})
report.write()
def malware_train(line):
global malware_train_nb
if (mta.check_if_link_is_in_downloaded_file(line) is False):
pcap_file_name = mta.download_pcap([line])
for pcap in pcap_file_name:
if (pcap is not None):
if (check_if_already_trained(pcap) is False):
attacker = AttackerCalc(pcap=mta.get_folder_name() + "/" +
pcap)
ip_to_consider = attacker.compute_attacker()
flow_type = "malware"
filter_1.set_ip_whitelist_filter(ip_to_consider)
filter_2.set_ip_whitelist_filter(ip_to_consider)
filter_3.set_ip_whitelist_filter(ip_to_consider)
featuresCalc = FeaturesCalc(flow_type=flow_type,
min_window_size=5)
csv = CSV(file_name="features_" + flow_type,
folder_name="Features")
csv.create_empty_csv()
csv.add_row(featuresCalc.get_features_name())
argument = {
"features_calc": featuresCalc,
"packets": [],
'filter': [filter_1, filter_2, filter_3],
'csv_obj': csv
}
sniffer = Sniffer(iface_sniffer,
callback_prn=callback_sniffer,
callback_prn_kwargs=argument)
sniffer.start()
while (sniffer.get_started_flag() is False):
pass
try:
sender = Sender(iface_sender,
fast=False,
verbose=False,
time_to_wait=10)
sender.send(mta.get_folder_name() + "/" + pcap)
sniffer.stop()
except Exception as e:
print(e)
csv.close_csv()
env.set_csv(csv.get_folder_name() + "/" +
csv.get_current_file_name())
agent.train_agent(
steps=csv.get_number_of_rows() - 1,
log_interval=csv.get_number_of_rows() - 1,
verbose=2,
nb_max_episode_steps=csv.get_number_of_rows() - 1)
malware_train_nb -= 1
trained_file.write(pcap + "\n")
else:
print("\nPcap gia' utilizzato in passato. Saltato.\n")
else:
pass
else:
pass
class Blockchain:
def __init__(self):
self.csv_operator = CSV()
def get_hash(self, block):
encoded_block = json.dumps(block, sort_keys=True).encode()
return hashlib.sha256(encoded_block).hexdigest()
def proof_of_work(self, previous_nonce):
new_nonce = 1
check_nonce = True
while check_nonce:
hash_operation = hashlib.sha256(
str(new_nonce**2 - previous_nonce**2).encode()).hexdigest()
if hash_operation[:4] == '0000':
check_nonce = False
else:
new_nonce += 1
return new_nonce
def chain_is_valid(self):
for previous_block, current_block in self.csv_operator.read_chain():
if self.get_hash(previous_block) != current_block["hash"]["block"]:
return False
hash_operation = hashlib.sha256(
str(current_block["nonce"]**2 -
previous_block["nonce"]**2).encode()).hexdigest()
if hash_operation[:4] != '0000':
return False
return True
def add_genesis_block(self):
block = {
"_id": 0,
"car_id": "None",
"nonce": 0,
"hash": {
"block": "None",
"car": "None"
},
"details": {
"block_type": "None"
}
}
self.chain.append(block)
def mine_block(self, block_type, block_data):
client = DBConnect()
last_block = self.csv_operator.get_last_block()
_id = self.csv_operator.get_chain_length()
car_id = block_data[0]
nonce = self.proof_of_work(last_block["nonce"])
last_hash_block = self.get_hash(last_block)
last_hash_car = "None"
if block_type == "Production":
block = {
"_id": _id,
"car_id": car_id,
"nonce": nonce,
"hash": {
"block": last_hash_block,
"car": last_hash_car
},
"details": {
"block_type": "Production"
}
}
elif block_type == "NewRegister":
block = {
"_id": _id,
"car_id": car_id,
"nonce": nonce,
"hash": {
"block": last_hash_block,
"car": last_hash_car
},
"details": {
"block_type": "NewRegister"
}
}
elif block_type == "Repair":
last_car_entry = client.get_car_history(car_id)[-1]
last_hash_car = self.get_hash(last_car_entry)
block = {
"_id": _id,
"car_id": car_id,
"nonce": nonce,
"hash": {
"block": last_hash_block,
"car": last_hash_car
},
"details": {
"block_type": "Repair",
}
}
elif block_type == "Sale":
last_car_entry = client.get_car_history(car_id)[-1]
last_hash_car = self.get_hash(last_car_entry)
block = {
"_id": _id,
"car_id": car_id,
"nonce": nonce,
"hash": {
"block": last_hash_block,
"car": last_hash_car
},
"details": {
"block_type": "Sale"
}
}
else:
return "wrong input"
client.ingest_block(block)
self.csv_operator.add_block(block)
def car_history_is_valid(self, car_id):
"""
car history is valid if
1. chain is valid
2. entries in data base match corresponding chain entries
3. the hashes that link the car history are valid
"""
if self.chain_is_valid():
client = DBConnect()
car_history = client.get_car_history(car_id)
for stage in car_history:
if stage != self.chain[stage["_id"]]:
return False
if len(car_history) <= 1:
return True
index = 1
while index < len(car_history):
if self.get_hash(
car_history[index -
1]) != car_history[index]["hash"]["car"]:
return False
index += 1
return True
else:
return False
def replace_chain(self):
longest_chain = None
max_length = len(self.chain)
for node in self.nodes:
response = requests.get(f'http://{node}/get_chain')
if response.status_code == 200:
length = response.json()['length']
chain = response.json()['chain']
if length > max_length and self.is_chain_valid(chain):
max_length = length
longest_chain = chain
if longest_chain:
self.chain = longest_chain
return True
return False
}
# Grabbing most used Pokemon online ranked 1 - 25 according to Pikalytics
most_popular = format_data(soup.find(id='min_list'), num_spaces=2)
for i in range(int(len(most_popular)/2)):
temp_list = [most_popular[i][0], most_popular[i + 1][0]]
most_popular[i] = temp_list
del (most_popular[i + 1])
# Path needs to be specified by the user
path = ''
# For each Pokemon in the top 25, creates a CSV file for common moves, teammates, items, abilities, and ev spreads
# Checks to make sure the directories exist, and create new ones if they don't
for i in range(len(most_popular)):
directory = path + most_popular[i][0].title()
for key, value in id_dict.items():
URL_pikalytics = "https://pikalytics.com/pokedex/ss/" + most_popular[i][0].lower()
pikalytics_page = requests.get(URL_pikalytics)
soup = BeautifulSoup(pikalytics_page.text, 'html.parser')
data = format_data(soup.find(id=key), value[0], value[1], value[2], value[3])
filepath = directory + '/' + most_popular[i][0] + '_' + value[4][0] + '.csv'
file = CSV(data, filepath)
if not os.path.exists(directory):
os.makedirs(directory)
else:
if not os.path.exists(filepath):
file.create_file(value[4][1:len(value[4])])
file.csv_write()
def __init__(self):
self.csv_operator = CSV()
class Entry:
"""
Entry class is used for interaction with an Entry between the work_log and the
CSV class. Most of the functions in this class simply pass down variables or
display them in a neater way. Init initializes the CSV and Errors class, and
the entry class uses user input to create a new entry.
"""
# Init initializes CSV/Error class to use in other functions
def __init__(self):
self.my_csv = CSV()
self.error = Error()
# Add an entry to CSV file
def add(self):
# Set date, name, time, and notes, check for errors on first 3 (Notes are optional)
date = self.error.time_error()
name = self.error.empty_task_error()
time = self.error.time_spent_error()
notes = input("Enter any additional notes on the entry (optional)\n>>")
# Pass down variables to writer in CSV, print confirmation
self.my_csv.writer(date, name, time, notes)
input("This entry has been added. Press enter to return to the main menu!")
# Delete an Entry (pass info)
def delete(self, to_delete):
self.my_csv.delete_entry(to_delete)
# Edit an Entry (pass info)
def edit(self, list_edit, item_edit):
self.my_csv.edit_entry(list_edit, item_edit)
# Search by date Range (pass info)
def range(self, date1, date2):
self.my_csv.reader5(date1, date2)
# Used for appending whole row while looking in entire row
def list_display(self, look):
new_list = [x for x in self.my_csv.reader(look)]
return new_list
# Used for appending whole row while looking at Task Name and Additional Notes
def list_display2(self, look):
new_list = [x for x in self.my_csv.reader2(look)]
return new_list
# Used for appending whole row while looking at Task Name and Additional Notes with REGEX
def list_display3(self, look):
new_list = [x for x in self.my_csv.reader3(look)]
return new_list
# Used for appending whole row while looking at Time Spent
def list_display4(self, look):
new_list = [x for x in self.my_csv.reader4(look)]
return new_list
# Used to display available dates
def display_dates(self):
for x in self.my_csv.date_reader():
print(x)
from CSV import CSV
# load data in from csv file (from https://catalog.data.gov/dataset/local-weather-archive)
weather_data = CSV().DictRead("rdu-weather-history.csv",delimiter=";")
# print headers
print(weather_data[0].keys())
# write data to a new file
CSV().DictWrite(weather_data,"weather_data.csv")
# convert the csv file to an excel file
CSV().CsvToXlsx("weather_data.csv")
# test converting the excel file data back to csv
CSV().XlsxToCsv("Workbook.xlsx",sheet="Sheet")
