def process_data(jList,
categories_to_filter_out,
model_file,
filter_flag=True):
return_list = []
num_entries = len(jList)
print("-------------------------------")
print("Total entries: %d" % num_entries)
print("-------------------------------")
cnt = 0
all_algos = get_all_algorithms()
cooccuring_counts_dict = {}
counts_dict = {}
for algo_name in all_algos:
cooccuring_counts_dict[algo_name] = {}
counts_dict[algo_name] = {}
for j in jList:
cnt += 1
if cnt % 10 == 0:
display_progress(cnt, num_entries)
try:
payloadJ = Payload(j)
cleaned_entities_text = get_text_to_process(
payloadJ, categories_to_filter_out)
if not cleaned_entities_text:
continue
doc = spacy_parser(cleaned_entities_text)
j['id'] = cnt
j['cleanedText'] = cleaned_entities_text
for algo_name in all_algos:
algo = get_algorithm_by_name(algo_name)
entities = algo.get_entities(cleaned_entities_text)
(j[algo_name + 'stemmed_entities'],
j[algo_name +
'non_stemmed_entities']) = clean_and_filter_entities(
entities, model_file, filter_flag)
update_cooccuring_dict(cooccuring_counts_dict[algo_name],
j[algo_name + 'non_stemmed_entities'],
list(doc.sents))
update_dict(counts_dict[algo_name],
j[algo_name + 'non_stemmed_entities'])
return_list.append(Payload(j))
except:
print("Encountered error: " + str(sys.exc_info()[0]))
print("While processing the following data: ")
print(cleaned_entities_text)
continue
display_progress(num_entries, num_entries)
return return_list, cooccuring_counts_dict, counts_dict
def extract_relationships(input_file, jar_path, descriptions_file,
relations_file, categories_to_filter_out):
all_text = []
(jar_directory, filename) = os.path.split(jar_path)
try:
with open(input_file) as data_file:
entitiesJsonList = json.loads(str(data_file.read()))
for jsonEntity in entitiesJsonList:
cleaned_text = get_text_to_process(Payload(jsonEntity),
categories_to_filter_out)
if cleaned_text:
all_text.append(cleaned_text)
except IOError:
print("Could not read file:" + input_file)
try:
with open(descriptions_file, 'w') as desc_file:
for text in all_text:
desc_file.write(text + "\n")
except IOError:
print("Could not read file:" + descriptions_file)
print("Extracting relations using open IE")
print("-----------------------------------")
subprocess.call([
"java", "-Xmx10g", "-XX:+UseConcMarkSweepGC", "-jar", jar_path,
"--ignore-errors", "-s", "--format", "column", descriptions_file,
relations_file
],
cwd=jar_directory)
print("-----------------------------------")
def createPayload(self):
try:
payloadBytes = self.curlJSONPayload()
decodedPayload = json.loads(payloadBytes.decode('utf-8'))
jsonPayload = json.dumps(decodedPayload)
initialDateTime = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
payloadID = str(uuid.uuid1())
size = sys.getsizeof(payloadBytes)
lastModified = initialDateTime
payloadData = jsonPayload
roundTripTime = None
payload = Payload(initialDateTime, payloadID, size, lastModified,
payloadData, roundTripTime)
log = self.createLog(self.agentID,
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
"createPayload", True)
self.sendLog(log)
return payload
except Exception:
print('Error creating payload.')
log = self.createLog(self.agentID,
datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
"createPayload", False)
self.sendLog(log)
def Transfer(apiKey, accountId, dtoTransfer):
url = host + "/accounts/v1/accounts/" + accountId + "/transfers"
#payload = jsonify(dtoTransfer)
payload_mock = Payload()
payload = payload_mock.GetPayloads(accountId)
headers = {
'Content-Type': 'application/json',
'Authorization': 'Bearer ' + apiKey
}
response = requests.request("POST", url, headers=headers, data = payload)
responseJson = json.loads(response.text)
return responseJson, response.status_code
def payload_request_out(parameters):
if len(parameters) != 3:
return
website = parameters[0]
username = parameters[1]
password = parameters[2]
payload = Payload(
website, username,
password) # just to make data as clear as possible we put in a object
Vars.user_interface.payload_request_out_handler(payload)
def processTree(featureOrientation, data):
# cutoffPayloads = getOrientationFromFile(featureOrientation)
# if(cutoffPayloads != None): return cutoffPayloads
# orientation not found, must create
length = len(featureOrientation) # arbritrary but for our case is 15
cutoffPayloads = [Payload()
for i in range(length)] # stores cutoffs of each feature
getCutoffs(0, length, data, data.shape[0], featureOrientation,
cutoffPayloads) # stores cutoffs in cutoffs
# storeNewPayloads(cutoffPayloads, featureOrientation)
return cutoffPayloads
def processTree(featureOrientation, data):
global SUCCESSES
SUCCESSES = 0
# cutoffPayloads = getOrientationFromFile(featureOrientation)
# if(cutoffPayloads != None): return cutoffPayloads
# orientation not found, must create
length = len(featureOrientation) # arbritrary but for our case is 15
cutoffPayloads = [Payload()
for i in range(length)] # stores cutoffs of each feature
print("SUCCBEFORE:", SUCCESSES)
getCutoffs(0, length, data, data.shape[0], featureOrientation,
cutoffPayloads) # stores cutoffs in cutoffs
print("SUCCAFTER", SUCCESSES)
accuracy = SUCCESSES / (1.0 * data.shape[0])
sendToMasterArduino(cutoffPayloads)
sendToAccuracyArduino(accuracy)
# storeNewPayloads(cutoffPayloads, featureOrientation)
return cutoffPayloads
def get_entities(text):
result = analyze_entities(text, get_native_encoding_type())
entitiesList = [Payload(ent) for ent in result['entities']]
filteredEntitiesList = list(
filter(lambda x: x.type == 'OTHER', entitiesList))
return set([x.name for x in filteredEntitiesList])
purpose: run map user interface
to run, enter into terminal:
>> python main.py <COM#>
'''
import sys
if len(sys.argv) != 2:
print('*ERROR*\n Usage: python {} <COM#>'.format(sys.argv[0]))
sys.exit(1)
from Map import Map
from Entity import Entity
from Payload import Payload
from Uart import Uart
COM_PORT = sys.argv[1]
uart = Uart(COM_PORT)
# starting center coordinate
coord = (38.0331584, -78.5098304)
map = Map(coord)
while (1):
binary = uart.uart_recv()
pld = Payload(binary)
map.add_point(pld)
print('Received Data: ' + str(pld.data))
print(pld.getCallout())
print(pld.coord)
print('\n')
def __init__(self):
self.agentID = '00002'
self.payload = Payload("", "", "", "", "", "")
def generate_payload_center():
print("[*] Start generate payload to fuzz the target !")
global TEST_CASE_ID, OUTPUT_DATA
s = "socket"
if SERVICETYPE == 'TCP':
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((RHOST, RPORT))
else:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
Csequence = 0
# custom payload class
rtsp_payload = Payload()
PARAMETERS = ['OPTIONS', 'DESCRIBE', 'SETUP', 'PLAY', 'TEARDOWN', 'PAUSE']
# PARAMETERS = ['OPTIONS']
target_csv = createcsv()
with open(target_csv, 'w') as output:
writer = csv.writer(output)
#Write header
writer.writerow(['#', 'Payload', 'Response'])
while TEST_CASE_ID < STOPAFTER:
try:
print(
"------------------Test case %s Start------------------------"
% (TEST_CASE_ID))
print("Start Time:%s" %
(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
print("Delay Time:%s/s" % (5))
# clone a clean format
payload_format = deepcopy(rtsp_payload.rtsp_payload_format)
rand_payload_size = randint(STARTSIZE, ENDSIZE)
# rand_payload_size = int(200)
payload_format = rtsp_payload.random_generator_payload(
payload_format, rand_payload_size, DEBUG)
# payload_format = rtsp_payload.specific_generator_payload(payload_format,"cseq",rand_payload_size)
# payload_format = rtsp_payload.specific_generator_payload(payload_format,"auth",rand_payload_size)
if Csequence > 100:
Csequence = 0
else:
Csequence += 1
rand_method = choice(PARAMETERS)
index = randint(0, 1)
payload = advance_payload_generator(
method=rand_method,
index=index,
Cseq=Csequence,
payload_format=payload_format)
if SERVICETYPE == 'TCP':
s.send(payload)
else:
s.sendto(payload)
print(
"[-->]Advance Fuzzing Test case Sented! \nPayload is :\n" +
payload)
result = s.recv(1500)
if result:
# while result:
result = result
print("[<--]Server Response:" + result)
else:
result = '[<--]Server no response:!!!\n'
print("[<--]Server no response:!!!\n")
buff_log = [TEST_CASE_ID, payload, result]
print("End Time:%s" %
(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
print(
"------------------Test case %s End------------------------"
% (TEST_CASE_ID))
TEST_CASE_ID += 1
# Write log to csv
writer.writerow(buff_log)
# DELAY
time.sleep(DELAY)
except KeyboardInterrupt:
sys.exit()
except socket.error, exc:
print "[*]Caught exception socket.error : %s" % exc
# create socket
s.close()
if SERVICETYPE == 'TCP':
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((RHOST, RPORT))
else:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
def workerThread2(self):
print("net working")
# read pre-trained model and config file
net = cv2.dnn.readNet("./_models/yolov3.weights",
"./_models/yolov3.cfg")
net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV)
net.setPreferableTarget(cv2.dnn.DNN_TARGET_OPENCL)
scale = 0.00392
"""
This is where we handle the asynchronous I/O. For example, it may be
a 'select( )'. One important thing to remember is that the thread has
to yield control pretty regularly, by select or otherwise.
"""
while self.running:
blob = cv2.dnn.blobFromImage(self.frame,
scale, (416, 416), (0, 0, 0),
True,
crop=False)
# To simulate asynchronous I/O, we create a random number at
# random intervals. Replace the following two lines with the real
# thing.
Width = self.frame.shape[1]
Height = self.frame.shape[0]
net.setInput(blob)
outs = net.forward(self.get_output_layers(net))
print("outs " + str(len(outs)) + str(type(outs)))
class_ids = []
confidences = []
boxes = []
conf_threshold = 0.5
nms_threshold = 0.4
for out in outs:
for detection in out:
scores = detection[5:]
class_id = np.argmax(scores)
confidence = scores[class_id]
if confidence > 0.5:
center_x = int(detection[0] * Width)
center_y = int(detection[1] * Height)
w = int(detection[2] * Width)
h = int(detection[3] * Height)
x = center_x - w / 2
y = center_y - h / 2
class_ids.append(class_id)
confidences.append(float(confidence))
boxes.append([x, y, w, h])
indices = cv2.dnn.NMSBoxes(boxes, confidences, conf_threshold,
nms_threshold)
payload = Payload(indices, class_ids, confidences, boxes)
print("outs " + str(len(outs)) + str(type(indices)))
time.sleep(.5)
# msg = rand.random( )
# print("put", msg)
self.queue.put(payload)