def save(self, translate, file=None):
'''
Permet d'enregistrer un fichier .thc
:param translate: (translate) Class translate du texte à enregistrer.
:param file: (facultatif) (string) lien vers l'emplacement du fichier.
:return: (boolean) si la sauvegarde s'est bien passée.
'''
binary = Binary()
if file is None:
self.file = "*.thc"
self.SaveAsFileDialog()
if self.file == "":
return False
else:
self.file = file
if not self.file.lower().endswith(".thc"):
self.file += ".thc"
dic = translate.getDict()
bin = translate.getBin()
savedic = binary.dictToSave(dic)
savebin = binary.to255lst(bin)
save = savedic + savebin
file = open(self.file, "wb")
file.write(bytearray(save))
file.close()
return True
def analyze(self):
try:
self.__offset = int(self.__options.offset,
16) if self.__options.offset else 0
except ValueError:
print "[Error] The offset must be in hexadecimal"
return False
if self.__options.console:
if self.__options.binary:
self.__binary = Binary(self.__options)
if self.__checksBeforeManipulations() == False:
return False
self.cmdloop()
return True
self.__binary = Binary(self.__options)
if self.__checksBeforeManipulations() == False:
return False
if self.__options.string:
return self.__lookingForAString(self.__options.string)
elif self.__options.opcode:
return self.__lookingForOpcodes(self.__options.opcode)
elif self.__options.memstr:
return self.__lookingForMemStr(self.__options.memstr)
else:
self.__getAllgadgets()
self.__lookingForGadgets()
if self.__options.ropchain:
ROPMaker(self.__binary, self.__gadgets, self.__offset)
return True
def __init__(self, ip, binary, name="noname", port=22):
self.name = name
self.ip = ip
self.port = port
listener_port = random.randint(20000, 25000)
self.listener = Listener(name, ip, listener_port, self)
self.binary = Binary(binary, self)
def analyze(self):
try:
self.__offset = int(self.__options.offset, 16) if self.__options.offset else 0
except ValueError:
print "[Error] The offset must be in hexadecimal"
return False
if self.__options.console:
if self.__options.binary:
self.__binary = Binary(self.__options)
if self.__checksBeforeManipulations() == False:
return False
self.cmdloop()
return True
self.__binary = Binary(self.__options)
if self.__checksBeforeManipulations() == False:
return False
if self.__options.string: return self.__lookingForAString(self.__options.string)
elif self.__options.opcode: return self.__lookingForOpcodes(self.__options.opcode)
elif self.__options.memstr: return self.__lookingForMemStr(self.__options.memstr)
else:
self.__getAllgadgets()
self.__lookingForGadgets()
if self.__options.ropchain:
ROPMaker(self.__binary, self.__gadgets, self.__offset)
return True
def binary(self):
"""Generates binaries"""
for sn in range(1, self.n_source_nodes+1):
for tn in range(1, self.n_transit_nodes+1):
for dn in range(1, self.n_destination_nodes+1):
binary = Binary()
binary.create_constraint_flow(sn, tn, dn)
self.add_line(binary)
def set(self, value):
binary = Binary(value)
upper, lower = binary.split(self.bits)
if upper != 0:
self.overflow = True
self._value = Binary(lower)._value
def __init__(self, binary, verbose=False, cflags=None):
self.bin = Binary(binary)
self.bin.verbose = verbose
self.bin.linker.cflags = cflags
self.patches = []
self.patchfiles = []
self.verbose = verbose
self.cflags = cflags
def set(self, value):
binary = Binary(value)
upper, lower = binary.split(self.bits)
if upper != 0:
self.overflow = True
self._value = Binary(lower)._value
def main():
args = get_args()
# only analyse the graph if it has not been done before
if os.path.isfile(args.graph):
return
config = Config()
config.MODE = config.TRAIN
config.BINARY_PATH = args.binary
config.BINARY_NAME = args.binary
config.DEBUG_INFO_PATH = args.debug_info
config.BAP_FILE_PATH = args.bap
config.GRAPH_PATH = args.graph
config.TWO_PASS = args.two_pass
config.FP_MODEL_PATH = args.fp_model
if config.TWO_PASS:
reg_dict = open(os.path.join(config.FP_MODEL_PATH, 'reg.dict'), 'rb')
reg_model = open(os.path.join(config.FP_MODEL_PATH, 'reg.model'), 'rb')
reg_support = open(os.path.join(config.FP_MODEL_PATH, 'reg.support'), 'rb')
config.REG_DICT = pickle.load(reg_dict, encoding='latin1')
config.REG_SUPPORT = pickle.load(reg_support, encoding='latin1')
config.REG_DICT.restrict(config.REG_SUPPORT.get_support())
config.REG_MODEL = pickle.load(reg_model, encoding='latin1')
config.REG_MODEL.n_jobs = 1
off_dict = open(os.path.join(config.FP_MODEL_PATH, 'off.dict'), 'rb')
off_model = open(os.path.join(config.FP_MODEL_PATH, 'off.model'), 'rb')
off_support = open(os.path.join(config.FP_MODEL_PATH, 'off.support'), 'rb')
config.OFF_DICT = pickle.load(off_dict, encoding='latin1')
config.OFF_SUPPORT = pickle.load(off_support, encoding='latin1')
config.OFF_DICT.restrict(config.OFF_SUPPORT.get_support())
config.OFF_MODEL = pickle.load(off_model, encoding='latin1')
config.OFF_MODEL.n_jobs = 1
with open(config.BINARY_PATH, 'rb') as elffile, open(config.DEBUG_INFO_PATH, 'rb') as debug_elffile:
b = Binary(config, elffile, debug_elffile)
if config.GRAPH_PATH != '':
b.dump_graph()
if config.TWO_PASS:
reg_dict.close()
reg_support.close()
reg_model.close()
off_dict.close()
off_support.close()
off_model.close()
def generate_feature(b, bin_dir, debug_dir, bap_dir):
try:
config = Config()
config.BINARY_NAME = b
config.BINARY_PATH = os.path.join(bin_dir, b)
config.DEBUG_INFO_PATH = os.path.join(debug_dir, b)
if bap_dir != '':
config.BAP_FILE_PATH = os.path.join(bap_dir, b)
with open(config.BINARY_PATH, 'rb') as elffile, open(config.DEBUG_INFO_PATH, 'rb') as debug_elffile:
b = Binary(config, elffile, debug_elffile)
return b.get_features()
except Exception as e:
print('Exception in binary anaylsis: ' + e)
return [], [], [], []
def main():
options = Arguments().get_args()
binary = Binary(options)
exec_sections = binary.get_exec_sections()
data_sections = binary.get_data_sections()
gadgets = Gadgets(exec_sections, options)
gadgets.pretty_print_gadgets()
strings = Strings(exec_sections + data_sections)
strings.pretty_print_strings()
ropchain = Ropchain(gadgets.get_gadgets(), binary.get_data_section_offset())
ropchain.pretty_print_chain()
def open(self):
'''
Permet d'ouvrir un fichier .thc ou .txt pour le lire et le modifier
:return: (translate) Class translate contenant le texte
'''
translate = Translate()
self.OpenFileDialog()
if self.file.lower().endswith(".txt"):
doc = open(self.file, "r", encoding='utf8')
text = doc.read()
translate.toBin(text)
return translate
binary = Binary()
if self.file == "":
return None
doc = open(self.file, "rb")
data = self.bytes2binstr(doc.read())
doc.close()
dic = ""
text = ""
isDic = True
countLenDic = 0
for i in range(len(data) // 7 + ceil((len(data) % 7) / 7)):
value = ""
for y in range(7):
b = ""
if isDic:
b = "0"
if len(data) >= i * 7 + y + 1:
b = data[i * 7 + y]
value += b
if value == cbch["[SUPERSEP¤]"]:
isDic = False
elif isDic:
countLenDic += 7
if isDic:
dic += value
else:
if value != cbch["[SUPERSEP¤]"]:
text += value
translate.toText(text[(len(text) % 8):], binary.SaveToDict(dic))
return translate
def __init__(self, binary, verbose=False, cflags=None):
self.bin = Binary(binary)
self.bin.verbose = verbose
self.bin.linker.cflags = cflags
self.patches = []
self.patchfiles = []
self.verbose = verbose
self.cflags = cflags
def main():
args = get_args()
if os.path.isfile(args.stat): return
config = Config()
config.MODE = config.TRAIN
config.BINARY_PATH = args.binary_with_symtab
config.BINARY_NAME = args.binary_with_symtab
config.BAP_FILE_PATH = args.bap
config.DEBUG_INFO_PATH = args.debug_info
config.OUTPUT_BINARY_PATH = args.output
config.MODIFY_ELF_LIB_PATH = args.elf_modifier
config.N2P_SERVER_URL = args.n2p_url
config.STAT_PATH = args.stat
config.TWO_PASS = args.two_pass
config.FP_MODEL_PATH = args.fp_model
if config.TWO_PASS:
reg_dict = open(os.path.join(config.FP_MODEL_PATH, 'reg.dict'), 'rb')
reg_model = open(os.path.join(config.FP_MODEL_PATH, 'reg.model'), 'rb')
reg_support = open(os.path.join(config.FP_MODEL_PATH, 'reg.support'),
'rb')
config.REG_DICT = pickle.load(reg_dict, encoding='latin1')
config.REG_SUPPORT = pickle.load(reg_support, encoding='latin1')
config.REG_DICT.restrict(config.REG_SUPPORT.get_support())
config.REG_MODEL = pickle.load(reg_model, encoding='latin1')
config.REG_MODEL.n_jobs = 1
off_dict = open(os.path.join(config.FP_MODEL_PATH, 'off.dict'), 'rb')
off_model = open(os.path.join(config.FP_MODEL_PATH, 'off.model'), 'rb')
off_support = open(os.path.join(config.FP_MODEL_PATH, 'off.support'),
'rb')
config.OFF_DICT = pickle.load(off_dict, encoding='latin1')
config.OFF_SUPPORT = pickle.load(off_support, encoding='latin1')
config.OFF_DICT.restrict(config.OFF_SUPPORT.get_support())
config.OFF_MODEL = pickle.load(off_model, encoding='latin1')
config.OFF_MODEL.n_jobs = 1
with open(config.BINARY_PATH,
'rb') as elffile, open(config.DEBUG_INFO_PATH,
'rb') as debug_elffile:
b = Binary(config, elffile, debug_elffile)
b.set_test_result_from_server(True)
b.modify_elf(args.binary_without_symtab)
if config.STAT_PATH is not None:
b.dump_stat()
if config.TWO_PASS:
reg_dict.close()
reg_support.close()
reg_model.close()
off_dict.close()
off_support.close()
off_model.close()
def do_binary(self, s, silent=False):
# Do not split the filename with spaces since it might contain
# whitespaces
if len(s) == 0:
if not silent:
return self.help_binary()
return False
binary = s
self.__options.binary = binary
self.__binary = Binary(self.__options)
if self.__checksBeforeManipulations() == False:
return False
if not silent:
print "[+] Binary loaded"
def set_documents(self, bundle, prefix=None):
"""Generates and appends a SmokingStatus entry to the transaction"""
if self.pid in Document.documents and GENERATION_MAP["Documents"]:
for d in [doc for doc in Document.documents[self.pid] if doc.type != 'photograph']:
data = fetch_document (self.pid, d.file_name)
# d.content = data['base64_content']
# d.size = data['size']
# d.hash = data['hash']
# b = d
# id = uid("Binary", "%s-document" % d.id, prefix)
# d.binary_id = id
# template = template_env.get_template('binary.xml')
# print >>pfile, template.render(dict(globals(), **locals()))
doc = Binary({
"mime_type": d.mime_type,
"content" : data['base64_content'],
"id" : uid(None, "%s-document" % d.id, prefix)
})
# self.appendEntry(bundle, doc)
patientFile = open(os.path.join(
self.path,
"patient-%s-document-%s.fhir-bundle.json" % (self.pid, d.id)
), "w")
print >> patientFile, pp_json({
"resourceType": "Bundle",
"type": "transaction",
"entry": [ doc ]
})
binary_id = uid(None, "%s-document-ref" % d.id, prefix)
docRef = Document({
'ID' : binary_id,
'PID' : self.pid,
'DATE' : datetime.now().strftime("%Y-%m-%dT%H:%M:%S+" + "05:00"), #.isoformat(),
'TITLE' : d.file_name,
'MIME_TYPE': d.mime_type,
'FILE_NAME': d.file_name,
'TYPE' : "Document",
'mime_type': d.mime_type
})
self.appendEntry(bundle, docRef.toJSON(data, binary_id, prefix))
# id = uid("DocumentReference", "%s-document" % d.id, prefix)
# d.system = 'http://smarthealthit.org/terms/codes/DocumentType#'
# d.code = d.type
# d.display = d.type
# template = template_env.get_template('document.xml')
# print >>pfile, template.render(dict(globals(), **locals()))
# if GENERATION_MAP["Documents"]:
# if self.pid in SocialHistory.socialHistories:
# self.appendEntry(bundle,
# SocialHistory.socialHistories[self.pid].toJSON(prefix)
# )
return bundle
def save_pre(n, pre_prob):
if None in (n, pre_prob):
raise PreventUpdate
if pre_prob == 'trav':
configs.params['prob_type'] = Single(configs.params)
configs.params['num_objs'] = 1
configs.params['objs'] = [min]
configs.params['obj_names'] = ['Distance']
configs.params['enc_name'] = 'Permutation'
configs.params['enc_type'] = Perm(configs.params)
configs.params['gene_size'] = 29
configs.params['enc_type'].params['min_value'] = 0
configs.params['enc_type'].params['max_value'] = 28
configs.params['eval_type'] = TSP
configs.params['cust_vis'] = network
elif pre_prob == 'knap':
configs.params['prob_type'] = Multi(configs.params)
configs.params['num_objs'] = 2
configs.params['objs'] = [max, min]
configs.params['obj_names'] = ['Value', 'Weight']
configs.params['enc_name'] = 'Binary String'
configs.params['enc_type'] = Binary(configs.params)
configs.params['gene_size'] = 64
configs.params['enc_type'].params['min_value'] = 0
configs.params['enc_type'].params['max_value'] = 1
configs.params['eval_type'] = knapsack
configs.params['cust_vis'] = selection
configs.params['n'] = 8
elif pre_prob == 'queens':
configs.params['prob_type'] = Single(configs.params)
configs.params['num_objs'] = 1
configs.params['objs'] = [min]
configs.params['obj_names'] = ['Threats']
configs.params['enc_name'] = 'Permutation'
configs.params['enc_type'] = Perm(configs.params)
configs.params['gene_size'] = 64
configs.params['enc_type'].params['min_value'] = 0
configs.params['enc_type'].params['max_value'] = 63
configs.params['eval_type'] = eight_queens
configs.params['cust_vis'] = chess_board
elif pre_prob == 'sudoku':
configs.params['prob_type'] = Single(configs.params)
configs.params['num_objs'] = 1
configs.params['objs'] = [min]
configs.params['obj_names'] = ['Conflicts']
configs.params['enc_name'] = 'Permutation'
configs.params['enc_type'] = Perm(configs.params)
configs.params['gene_size'] = 100
configs.params['enc_type'].params['min_value'] = 0
configs.params['enc_type'].params['max_value'] = 99
configs.params['eval_type'] = sudoku
configs.params['cust_vis'] = sudoku_board
return 'Saved'
def set_patient(self, bundle, prefix=None):
"""Generates and appends the Patient entry to the transaction"""
if GENERATION_MAP["patient"]:
patient = Patient.mpi[self.pid]
if prefix:
patient.pid = prefix + "-" + patient.pid
# look up patient photos
if self.pid in Document.documents:
for d in [
doc for doc in Document.documents[self.pid]
if doc.type == 'photograph'
]:
data = fetch_document(self.pid, d.file_name)
binary_id = uid(None, "%s-photo" % d.id, prefix)
self.appendEntry(
bundle,
Binary({
"mime_type": d.mime_type,
"content": data['base64_content'],
"id": binary_id
}))
patient.photo_title = d.title
patient.photo_code = d.mime_type
patient.photo_binary_id = binary_id
patient.photo_hash = data["hash"]
patient.photo_size = data["size"]
patientJSON = patient.toJSON(prefix)
bundle = self.set_documents(bundle, prefix)
self.appendEntry(bundle, Entry(patientJSON))
if patient.gestage:
self.appendEntry(
bundle,
Entry(
Observation(
{
"id": uid(None, "%s-gestage" % self.pid,
prefix),
"pid": self.pid,
"date": patient.dob,
"code": "18185-9",
"name": "Gestational age at birth",
"scale": "Qn",
"value": patient.gestage,
"units": "weeks",
"unitsCode": "wk",
"categoryCode": "exam",
"categoryDisplay": "Exam"
}, prefix)))
return bundle
def save(n, prob_type, num_objs, obj_1_name, obj_2_name, obj_3_name,
obj_1_goal, obj_2_goal, obj_3_goal, enc_type, gene_size,
min_value, max_value):
if None in (n, prob_type, num_objs, min_value, max_value):
raise PreventUpdate
elif num_objs == 1 and None in (obj_1_name, obj_1_goal):
raise PreventUpdate
elif num_objs == 2 and None in (obj_1_name, obj_1_goal, obj_2_name,
obj_2_goal):
raise PreventUpdate
elif num_objs == 3 and None in (obj_1_name, obj_1_goal, obj_2_name,
obj_2_goal, obj_3_name, obj_3_goal):
raise PreventUpdate
elif gene_size is not None and gene_size < 0:
raise PreventUpdate
elif min_value is not None and min_value >= max_value:
raise PreventUpdate
elif max_value is not None and max_value <= min_value:
raise PreventUpdate
if prob_type == 'sing-obj':
configs.params['prob_type'] = Single(configs.params)
else:
configs.params['prob_type'] = Multi(configs.params)
configs.params['num_objs'] = num_objs
configs.params['objs'], configs.params['obj_names'] = [], []
dirs, names = [obj_1_goal, obj_2_goal,
obj_3_goal], [obj_1_name, obj_2_name, obj_3_name]
for i in range(configs.params['num_objs']):
if dirs[i] == 'min':
configs.params['objs'].append(min)
else:
configs.params['objs'].append(max)
configs.params['obj_names'].append(names[i])
if enc_type == 'perm':
configs.params['enc_name'] = 'Permutation'
configs.params['enc_type'] = Perm(configs.params)
elif enc_type == 'binary':
configs.params['enc_name'] = 'Binary String'
configs.params['enc_type'] = Binary(configs.params)
elif enc_type == 'int':
configs.params['enc_name'] = 'Integer String'
configs.params['enc_type'] = Integer(configs.params)
elif enc_type == 'real':
configs.params['enc_name'] = 'Real-Valued String'
configs.params['enc_type'] = Real(configs.params)
configs.params['gene_size'] = gene_size
configs.params['enc_type'].params['min_value'] = min_value
configs.params['enc_type'].params['max_value'] = max_value
return 'Saved'
def _get_binary(data):
(length, data) = _get_int(data)
subtype = ord(data[0])
data = data[1:]
if subtype == 2:
(length2, data) = _get_int(data)
if length2 != length - 4:
raise InvalidBSON("invalid binary (st 2) - lengths don't match!")
length = length2
if subtype == 3 and _use_uuid:
return (uuid.UUID(bytes=data[:length]), data[length:])
return (Binary(data[:length], subtype), data[length:])
def where(self, si):
C = self.contain_loc(si)
if C:
si_filtered = self.PP_location_deletion(si, C)
binary_q = Binary.main(si_filtered)
binary_q = binary_q[0].lower() + binary_q[1:]
si_binary = binary_q
if si_binary:
return "where " + si_binary
else:
return None
return None
def set_clinical_notes(self, bundle, prefix=None):
"""Generates and appends a ClinicalNote entry to the transaction"""
if GENERATION_MAP[
"ClinicalNotes"] and self.pid in ClinicalNote.clinicalNotes:
for d in ClinicalNote.clinicalNotes[self.pid]:
if d.mime_type == 'text/plain':
data = fetch_document(self.pid, d.file_name)
# d.content = data['base64_content']
# b = d
# id = uid("Binary", "%s-note" % d.id, prefix)
# d.binary_id = id
binary_id = uid(None, "%s-note" % d.id, prefix)
note = Binary({
"mime_type": d.mime_type,
"content": data['base64_content'],
"id": binary_id
})
bundle["entry"].append(note)
# if GENERATION_MAP["Documents"]:
docRef = Document({
'ID':
uid(None, "%s-note-ref" % d.id, prefix),
'PID':
self.pid,
'DATE':
datetime.now().strftime("%Y-%m-%dT%H:%M:%S+" +
"05:00"), #.isoformat(),
'TITLE':
"Note",
'MIME_TYPE':
d.mime_type,
'FILE_NAME':
d.file_name,
'TYPE':
"Note",
'mime_type':
d.mime_type
})
bundle["entry"].append(
docRef.toJSON(data, binary_id, prefix))
# id = uid("DocumentReference", "%s-note" % d.id, prefix)
# d.system = "http://loinc.org"
# d.code = '34109-9'
# d.display = 'Note'
# template = template_env.get_template('document.xml')
# print >>pfile, template.render(dict(globals(), **locals()))
return bundle
def handle_uploaded_dataset_file( filename, datatypes_registry, ext = 'auto', is_multi_byte = False ):
is_valid, ext = handle_compressed_file( filename, datatypes_registry, ext = ext )
if not is_valid:
raise InappropriateDatasetContentError, 'The compressed uploaded file contains inappropriate content.'
if ext in AUTO_DETECT_EXTENSIONS:
ext = guess_ext( filename, sniff_order = datatypes_registry.sniff_order, is_multi_byte=is_multi_byte )
if check_binary( filename ):
if not Binary.is_ext_unsniffable(ext) and not datatypes_registry.get_datatype_by_extension( ext ).sniff( filename ):
raise InappropriateDatasetContentError, 'The binary uploaded file contains inappropriate content.'
elif check_html( filename ):
raise InappropriateDatasetContentError, 'The uploaded file contains inappropriate HTML content.'
return ext
def main(self, k):
print("\n\n\n\n")
print("Check Point 0")
T = Tokenize.main(k, "einstein.txt")
sentences_top_k = T[0]
NE = T[1]
questions = []
for si in sentences_top_k:
print(" *** org : ", si)
bin_attempt = Binary.main(si)
wh_attempt = WH.main(bin_attempt, si, NE)
if bin_attempt:
print(" *** bin : ", bin_attempt)
questions.append(bin_attempt)
if wh_attempt:
questions.append(wh_attempt)
def set_documents(self, bundle, prefix=None):
"""Generates and appends a SmokingStatus entry to the transaction"""
if self.pid in Document.documents and GENERATION_MAP["Documents"]:
for d in [
doc for doc in Document.documents[self.pid]
if doc.type != 'photograph'
]:
data = fetch_document(self.pid, d.file_name)
# d.content = data['base64_content']
# d.size = data['size']
# d.hash = data['hash']
# b = d
# id = uid("Binary", "%s-document" % d.id, prefix)
# d.binary_id = id
# template = template_env.get_template('binary.xml')
# print >>pfile, template.render(dict(globals(), **locals()))
doc = Binary({
"mime_type": d.mime_type,
"content": data['base64_content'],
"id": uid(None, "%s-document" % d.id, prefix)
})
# bundle["entry"].append(doc)
patientFile = open(
os.path.join(
self.path, "patient-%s-document-%s.fhir-bundle.json" %
(self.pid, d.id)), "w")
print >> patientFile, pp_json({
"resourceType": "Bundle",
"type": "transaction",
"entry": [doc]
})
# id = uid("DocumentReference", "%s-document" % d.id, prefix)
# d.system = 'http://smarthealthit.org/terms/codes/DocumentType#'
# d.code = d.type
# d.display = d.type
# template = template_env.get_template('document.xml')
# print >>pfile, template.render(dict(globals(), **locals()))
# if GENERATION_MAP["Documents"]:
# if self.pid in SocialHistory.socialHistories:
# bundle["entry"].append(
# SocialHistory.socialHistories[self.pid].toJSON(prefix)
# )
return bundle
def do_binary(self, s, silent=False):
# Do not split the filename with spaces since it might contain
# whitespaces
if len(s) == 0:
if not silent:
return self.help_binary()
return False
binary = s
self.__options.binary = binary
self.__binary = Binary(self.__options)
if self.__checksBeforeManipulations() == False:
return False
if not silent:
print "[+] Binary loaded"
def main():
args = get_args()
config = Config()
config.MODE = config.TRAIN
config.BINARY_PATH = args.binary
config.BINARY_NAME = args.binary
config.BAP_FILE_PATH = args.bap
config.DEBUG_INFO_PATH = args.debug_info
config.N2P_SERVER_URL = args.n2p_url
config.STAT_PATH = args.stat
config.TWO_PASS = args.two_pass
config.FP_MODEL_PATH = args.fp_model
if config.TWO_PASS:
reg_dict = open(os.path.join(config.FP_MODEL_PATH, 'reg.dict'), 'rb')
reg_model = open(os.path.join(config.FP_MODEL_PATH, 'reg.model'), 'rb')
reg_support = open(os.path.join(config.FP_MODEL_PATH, 'reg.support'), 'rb')
config.REG_DICT = pickle.load(reg_dict, encoding='latin1')
config.REG_SUPPORT = pickle.load(reg_support, encoding='latin1')
config.REG_DICT.restrict(config.REG_SUPPORT.get_support())
config.REG_MODEL = pickle.load(reg_model, encoding='latin1')
config.REG_MODEL.n_jobs = 1
off_dict = open(os.path.join(config.FP_MODEL_PATH, 'off.dict'), 'rb')
off_model = open(os.path.join(config.FP_MODEL_PATH, 'off.model'), 'rb')
off_support = open(os.path.join(config.FP_MODEL_PATH, 'off.support'), 'rb')
config.OFF_DICT = pickle.load(off_dict, encoding='latin1')
config.OFF_SUPPORT = pickle.load(off_support, encoding='latin1')
config.OFF_DICT.restrict(config.OFF_SUPPORT.get_support())
config.OFF_MODEL = pickle.load(off_model, encoding='latin1')
config.OFF_MODEL.n_jobs = 1
with open(config.BINARY_PATH, 'rb') as elffile, open(config.DEBUG_INFO_PATH, 'rb') as debug_elffile:
TIMER.start_scope('0ALL')
b = Binary(config, elffile, debug_elffile)
TIMER.start_scope('2CRF')
b.set_test_result_from_server(True)
TIMER.end_scope()
TIMER.end_scope()
b.dump_stat()
if config.TWO_PASS:
reg_dict.close()
reg_support.close()
reg_model.close()
off_dict.close()
off_support.close()
off_model.close()
def main():
args = get_args()
config = Config()
config.MODE = config.TEST
config.BINARY_PATH = args.binary
config.BINARY_NAME = args.binary
config.OUTPUT_BINARY_PATH = args.output
config.BAP_FILE_PATH = args.bap
config.MODIFY_ELF_LIB_PATH = args.elf_modifier
config.TWO_PASS = args.two_pass
config.FP_MODEL_PATH = args.fp_model
if config.TWO_PASS:
reg_dict = open(os.path.join(config.FP_MODEL_PATH, 'reg.dict'), 'rb')
reg_model = open(os.path.join(config.FP_MODEL_PATH, 'reg.model'), 'rb')
reg_support = open(os.path.join(config.FP_MODEL_PATH, 'reg.support'),
'rb')
config.REG_DICT = pickle.load(reg_dict, encoding='latin1')
config.REG_SUPPORT = pickle.load(reg_support, encoding='latin1')
config.REG_DICT.restrict(config.REG_SUPPORT.get_support())
config.REG_MODEL = pickle.load(reg_model, encoding='latin1')
config.REG_MODEL.n_jobs = 1
off_dict = open(os.path.join(config.FP_MODEL_PATH, 'off.dict'), 'rb')
off_model = open(os.path.join(config.FP_MODEL_PATH, 'off.model'), 'rb')
off_support = open(os.path.join(config.FP_MODEL_PATH, 'off.support'),
'rb')
config.OFF_DICT = pickle.load(off_dict, encoding='latin1')
config.OFF_SUPPORT = pickle.load(off_support, encoding='latin1')
config.OFF_DICT.restrict(config.OFF_SUPPORT.get_support())
config.OFF_MODEL = pickle.load(off_model, encoding='latin1')
config.OFF_MODEL.n_jobs = 1
config.N2P_SERVER_URL = args.n2p_url
with open(config.BINARY_PATH, 'rb') as elffile:
b = Binary(config, elffile)
b.set_test_result_from_server()
b.modify_elf()
if config.TWO_PASS:
reg_dict.close()
reg_support.close()
reg_model.close()
off_dict.close()
off_support.close()
off_model.close()
def get_raw_answer(self, question, answer): q_tree = sNLP.parse(question) q_tree = Tree.fromstring(str(q_tree)) a_tree = sNLP.parse(Binary.main(answer)) a_tree = Tree.fromstring(str(a_tree)) res = True (q_top_level_structure, q_parse_by_structure) = self.get_top_level_structure(q_tree) (a_top_level_structure, a_parse_by_structure) = self.get_top_level_structure(a_tree) for i in range(0, len(q_top_level_structure)): q_label = q_top_level_structure[i] if q_label in a_top_level_structure: a_index = a_top_level_structure.index(q_label) else: print "label not found" return False # print "Result:!!!!!", self.partial_matching(q_parse_by_structure[i], a_parse_by_structure[a_index]) if not self.partial_matching(q_parse_by_structure[i], a_parse_by_structure[a_index]): # print "struct:", q_parse_by_structure[i], a_parse_by_structure[a_index] return False return True
def main(self, k):
print("\n\n\n\n")
binary_questions = []
print "Check Point 0"
T = Tokenize.main(k)
sentences_top_k = T[0]
NE = T[1]
# print(sentences_top_k)
questions = []
for si in sentences_top_k:
print(" *** org : ", si)
# print(" *** original sentence: ", si)
bin_attempt = Binary.main(si)
wh_attempt = WH.main(bin_attempt, si, NE)
if bin_attempt:
print(" *** bin : ", bin_attempt)
questions.append(bin_attempt)
if wh_attempt:
questions.append(wh_attempt)
print("\n")
print("\n\n\n\n")
def handle_uploaded_dataset_file(filename,
datatypes_registry,
ext='auto',
is_multi_byte=False):
is_valid, ext = handle_compressed_file(filename,
datatypes_registry,
ext=ext)
if not is_valid:
raise InappropriateDatasetContentError, 'The compressed uploaded file contains inappropriate content.'
if ext in AUTO_DETECT_EXTENSIONS:
ext = guess_ext(filename,
sniff_order=datatypes_registry.sniff_order,
is_multi_byte=is_multi_byte)
if check_binary(filename):
if not Binary.is_ext_unsniffable(
ext) and not datatypes_registry.get_datatype_by_extension(
ext).sniff(filename):
raise InappropriateDatasetContentError, 'The binary uploaded file contains inappropriate content.'
elif check_html(filename):
raise InappropriateDatasetContentError, 'The uploaded file contains inappropriate HTML content.'
return ext
class Patcher:
def __init__(self, binary, verbose=False, cflags=None):
self.bin = Binary(binary)
self.bin.verbose = verbose
self.bin.linker.cflags = cflags
self.patches = []
self.patchfiles = []
self.verbose = verbose
self.cflags = cflags
def add(self, path):
if path.endswith('.py'):
self.patchfiles.append((path, os.path.basename(path)))
else:
base = os.path.basename(path.rstrip(os.path.sep))
for name in glob.glob(path + '/*.py'):
if os.path.basename(name).startswith('_'):
continue
self.patchfiles.append((name, os.path.join(base, os.path.basename(name))))
def patch(self):
cwd = os.getcwd()
try:
for path, pathname in self.patchfiles:
sys.path.insert(0, os.path.dirname(path))
print '[*]', pathname
patchfile = os.path.basename(path).rsplit('.', 1)[0]
patch = __import__(patchfile)
sys.path.pop(0)
# preserve function order
try:
source, _ = inspect.getsourcelines(patch)
order = []
for line in source:
if line.startswith('def'):
name = line.split(' ', 1)[1].split('(', 1)[0]
try:
order.append(getattr(patch, name))
except AttributeError:
pass
except Exception:
print 'Warning: could not preserve patch function order'
traceback.print_exc()
order = vars(patch).values()
for func in order:
if func.__name__.startswith('_'):
# skip "private" functions
continue
if hasattr(func, '__call__'):
print ' [+] %s()' % func.__name__
with self.bin.collect() as patchset:
try:
func(patchset)
except Exception as e:
print 'Exception thrown by patch:', path, func.__name__
traceback.print_exc()
print 'Memory maps:'
for prog in self.bin.elf.progs:
if elfutil.is_load(prog):
print '0x%x-0x%x' % (prog.vaddr, prog.vaddr + prog.vsize)
sys.exit(1)
print
finally:
os.chdir(cwd)
def save(self, path):
self.bin.save(path)
def test_binary_illegal_index():
with pytest.raises(IndexError):
Binary('01101010')[7]
def test_binary_negative_index():
assert Binary('0101110001')[-1] == '1'
assert Binary('0101110001')[-2] == '0'
def test_binary_get_bit():
binary = Binary('0101110001')
assert binary[0] == '1'
assert binary[5] == '1'
def test_binary_division_rem_binary():
assert Binary(21) / Binary(5) == Binary(4)
def __init__(self, binary, verbose=False):
self.bin = Binary(binary)
self.bin.verbose = verbose
self.patches = []
self.patchfiles = []
self.verbose = verbose
def __init__(self, auto_login=True):
log.info('Main init started')
self.binary = Binary(auto_login)
self.rl = RL()
self.loadQ()
log.info('Main init ended')
def _element_to_bson(key, value, check_keys):
if check_keys:
if key.startswith("$"):
raise InvalidName("key %r must not start with '$'" % key)
if "." in key:
raise InvalidName("key %r must not contain '.'" % key)
name = _make_c_string(key)
if isinstance(value, float):
return "\x01" + name + struct.pack("<d", value)
# Use Binary w/ subtype 3 for UUID instances
try:
import uuid
if isinstance(value, uuid.UUID):
value = Binary(value.bytes, subtype=3)
except ImportError:
pass
if isinstance(value, Binary):
subtype = value.subtype
if subtype == 2:
value = struct.pack("<i", len(value)) + value
return "\x05%s%s%s%s" % (name, struct.pack("<i", len(value)), chr(subtype), value)
if isinstance(value, Code):
cstring = _make_c_string(value)
scope = _dict_to_bson(value.scope, False)
full_length = struct.pack("<i", 8 + len(cstring) + len(scope))
length = struct.pack("<i", len(cstring))
return "\x0F" + name + full_length + length + cstring + scope
if isinstance(value, str):
cstring = _make_c_string(value)
length = struct.pack("<i", len(cstring))
return "\x02" + name + length + cstring
if isinstance(value, unicode):
cstring = _make_c_string(value)
length = struct.pack("<i", len(cstring))
return "\x02" + name + length + cstring
if isinstance(value, dict):
return "\x03" + name + _dict_to_bson(value, check_keys)
if isinstance(value, (list, tuple)):
as_dict = SON(zip([str(i) for i in range(len(value))], value))
return "\x04" + name + _dict_to_bson(as_dict, check_keys)
if isinstance(value, ObjectId):
return "\x07" + name + value.binary
if value is True:
return "\x08" + name + "\x01"
if value is False:
return "\x08" + name + "\x00"
if isinstance(value, (int, long)):
# TODO this is a really ugly way to check for this...
if value > 2 ** 64 / 2 - 1 or value < -2 ** 64 / 2:
raise OverflowError("MongoDB can only handle up to 8-byte ints")
if value > 2 ** 32 / 2 - 1 or value < -2 ** 32 / 2:
return "\x12" + name + struct.pack("<q", value)
return "\x10" + name + struct.pack("<i", value)
if isinstance(value, datetime.datetime):
millis = int(calendar.timegm(value.timetuple()) * 1000 + value.microsecond / 1000)
return "\x09" + name + struct.pack("<q", millis)
if value is None:
return "\x0A" + name
if isinstance(value, _RE_TYPE):
pattern = value.pattern
flags = ""
if value.flags & re.IGNORECASE:
flags += "i"
if value.flags & re.LOCALE:
flags += "l"
if value.flags & re.MULTILINE:
flags += "m"
if value.flags & re.DOTALL:
flags += "s"
if value.flags & re.UNICODE:
flags += "u"
if value.flags & re.VERBOSE:
flags += "x"
return "\x0B" + name + _make_c_string(pattern) + _make_c_string(flags)
if isinstance(value, DBRef):
return _element_to_bson(key, SON([("$ref", value.collection), ("$id", value.id)]), False)
raise InvalidDocument("cannot convert value of type %s to bson" % type(value))
def decode(cls, dat):
t, obj = Binary.decode(dat)
obj = loads(obj)
return obj
def encode(cls, t, dat):
dat = dumps(dat, HIGHEST_PROTOCOL)
return Binary.encode(t, dat)
class Core(cmd.Cmd):
def __init__(self, options):
cmd.Cmd.__init__(self)
self.__options = options
self.__binary = None
self.__gadgets = []
self.__offset = 0
self.prompt = '(ROPgadget)> '
def __checksBeforeManipulations(self):
if self.__binary == None or self.__binary.getBinary() == None or self.__binary.getArch() == None or self.__binary.getArchMode() == None:
return False
return True
def __getAllgadgets(self):
if self.__checksBeforeManipulations() == False:
return False
G = Gadgets(self.__binary, self.__options, self.__offset)
execSections = self.__binary.getExecSections()
# Find ROP/JOP/SYS gadgets
self.__gadgets = []
for section in execSections:
if not self.__options.norop: self.__gadgets += G.addROPGadgets(section)
if not self.__options.nojop: self.__gadgets += G.addJOPGadgets(section)
if not self.__options.nosys: self.__gadgets += G.addSYSGadgets(section)
# Pass clean single instruction and unknown instructions
self.__gadgets = G.passClean(self.__gadgets, self.__options.multibr)
# Delete duplicate gadgets
if not self.__options.all:
self.__gadgets = rgutils.deleteDuplicateGadgets(self.__gadgets)
# Applicate some Options
self.__gadgets = Options(self.__options, self.__binary, self.__gadgets).getGadgets()
# Sorted alphabetically
self.__gadgets = rgutils.alphaSortgadgets(self.__gadgets)
return True
def __lookingForGadgets(self):
if self.__checksBeforeManipulations() == False:
return False
arch = self.__binary.getArchMode()
print "Gadgets information\n============================================================"
for gadget in self.__gadgets:
vaddr = gadget["vaddr"]
insts = gadget["gadget"]
print ("0x%08x" %(vaddr) if arch == CS_MODE_32 else "0x%016x" %(vaddr)) + " : %s" %(insts)
print "\nUnique gadgets found: %d" %(len(self.__gadgets))
return True
def __lookingForAString(self, string):
if self.__checksBeforeManipulations() == False:
return False
dataSections = self.__binary.getDataSections()
arch = self.__binary.getArchMode()
print "Strings information\n============================================================"
for section in dataSections:
allRef = [m.start() for m in re.finditer(string, section["opcodes"])]
for ref in allRef:
vaddr = self.__offset + section["vaddr"] + ref
string = section["opcodes"][ref:ref+len(string)]
rangeS = int(self.__options.range.split('-')[0], 16)
rangeE = int(self.__options.range.split('-')[1], 16)
if (rangeS == 0 and rangeE == 0) or (vaddr >= rangeS and vaddr <= rangeE):
print ("0x%08x" %(vaddr) if arch == CS_MODE_32 else "0x%016x" %(vaddr)) + " : %s" %(string)
return True
def __lookingForOpcodes(self, opcodes):
if self.__checksBeforeManipulations() == False:
return False
execSections = self.__binary.getExecSections()
arch = self.__binary.getArchMode()
print "Opcodes information\n============================================================"
for section in execSections:
allRef = [m.start() for m in re.finditer(opcodes.decode("hex"), section["opcodes"])]
for ref in allRef:
vaddr = self.__offset + section["vaddr"] + ref
rangeS = int(self.__options.range.split('-')[0], 16)
rangeE = int(self.__options.range.split('-')[1], 16)
if (rangeS == 0 and rangeE == 0) or (vaddr >= rangeS and vaddr <= rangeE):
print ("0x%08x" %(vaddr) if arch == CS_MODE_32 else "0x%016x" %(vaddr)) + " : %s" %(opcodes)
return True
def __lookingForMemStr(self, memstr):
if self.__checksBeforeManipulations() == False:
return False
sections = self.__binary.getExecSections()
sections += self.__binary.getDataSections()
arch = self.__binary.getArchMode()
print "Memory bytes information\n======================================================="
chars = list(memstr)
for char in chars:
try:
for section in sections:
allRef = [m.start() for m in re.finditer(char, section["opcodes"])]
for ref in allRef:
vaddr = self.__offset + section["vaddr"] + ref
rangeS = int(self.__options.range.split('-')[0], 16)
rangeE = int(self.__options.range.split('-')[1], 16)
if (rangeS == 0 and rangeE == 0) or (vaddr >= rangeS and vaddr <= rangeE):
print ("0x%08x" %(vaddr) if arch == CS_MODE_32 else "0x%016x" %(vaddr)) + " : '%c'" %(char)
raise
except:
pass
return True
def analyze(self):
try:
self.__offset = int(self.__options.offset, 16) if self.__options.offset else 0
except ValueError:
print "[Error] The offset must be in hexadecimal"
return False
if self.__options.console:
if self.__options.binary:
self.__binary = Binary(self.__options)
if self.__checksBeforeManipulations() == False:
return False
self.cmdloop()
return True
self.__binary = Binary(self.__options)
if self.__checksBeforeManipulations() == False:
return False
if self.__options.string: return self.__lookingForAString(self.__options.string)
elif self.__options.opcode: return self.__lookingForOpcodes(self.__options.opcode)
elif self.__options.memstr: return self.__lookingForMemStr(self.__options.memstr)
else:
self.__getAllgadgets()
self.__lookingForGadgets()
if self.__options.ropchain:
ROPMaker(self.__binary, self.__gadgets, self.__offset)
return True
def gadgets(self):
return self.__gadgets
# Console methods ============================================
def do_binary(self, s, silent=False):
# Do not split the filename with spaces since it might contain
# whitespaces
if len(s) == 0:
if not silent:
return self.help_binary()
return False
binary = s
self.__options.binary = binary
self.__binary = Binary(self.__options)
if self.__checksBeforeManipulations() == False:
return False
if not silent:
print "[+] Binary loaded"
def help_binary(self):
print "Syntax: binary <file> -- Load a binary"
return False
def do_EOF(self, s, silent=False):
return self.do_quit(s, silent)
def do_quit(self, s, silent=False):
return True
def help_quit(self):
print "Syntax: quit -- Terminates the application"
return False
def do_load(self, s, silent=False):
if self.__binary == None:
if not silent:
print "[-] No binary loaded."
return False
if not silent:
print "[+] Loading gadgets, please wait..."
self.__getAllgadgets()
if not silent:
print "[+] Gadgets loaded !"
def help_load(self):
print "Syntax: load -- Load all gadgets"
return False
def do_display(self, s, silent=False):
self.__lookingForGadgets()
def help_display(self):
print "Syntax: display -- Display all gadgets loaded"
return False
def do_depth(self, s, silent=False):
try:
depth = int(s.split()[0])
except:
if not silent:
return self.help_depth()
return False
if depth <= 0:
if not silent:
print "[-] The depth value must be > 0"
return False
self.__options.depth = int(depth)
if not silent:
print "[+] Depth updated. You have to reload gadgets"
def help_depth(self):
print "Syntax: depth <value> -- Set the depth search engine"
return False
def do_badbytes(self, s, silent=False):
try:
bb = s.split()[0]
except:
if not silent:
return self.help_badbytes()
else:
return False
self.__options.badbytes = bb
if not silent:
print "[+] Bad bytes updated. You have to reload gadgets"
def help_badbytes(self):
print "Syntax: badbytes <badbyte1|badbyte2...> -- "
return False
def __withK(self, listK, gadget):
if len(listK) == 0:
return True
for a in listK:
if a not in gadget:
return False
return True
def __withoutK(self, listK, gadget):
for a in listK:
if a in gadget:
return False
return True
def do_search(self, s, silent=False):
args = s.split()
if not len(args):
return self.help_search()
withK, withoutK = [], []
for a in args:
if a[0:1] == "!":
withoutK += [a[1:]]
else:
withK += [a]
if self.__checksBeforeManipulations() == False:
if not silent:
print "[-] You have to load a binary"
return False
arch = self.__binary.getArchMode()
for gadget in self.__gadgets:
vaddr = gadget["vaddr"]
insts = gadget["gadget"]
if self.__withK(withK, insts) and self.__withoutK(withoutK, insts):
# What to do if silent = True?
print ("0x%08x" %(vaddr) if arch == CS_MODE_32 else "0x%016x" %(vaddr)) + " : %s" %(insts)
def help_search(self):
print "Syntax: search <keyword1 keyword2 keyword3...> -- Filter with or without keywords"
print "keyword = with"
print "!keyword = witout"
return False
def count(self):
return len(self.__gadgets)
def do_count(self, s, silent=False):
if not silent:
print "[+] %d loaded gadgets." % self.count()
def help_count(self):
print "Shows the number of loaded gadgets."
return False
def do_filter(self, s, silent=False):
try:
self.__options.filter = s.split()[0]
except:
if not silent:
return self.help_filter()
return False
if not silent:
print "[+] Filter setted. You have to reload gadgets"
def help_filter(self):
print "Syntax: filter <filter1|filter2|...> - Suppress specific instructions"
return False
def do_only(self, s, silent=False):
try:
self.__options.only = s.split()[0]
except:
if not silent:
return self.help_only()
return False
if not silent:
print "[+] Only setted. You have to reload gadgets"
def help_only(self):
print "Syntax: only <only1|only2|...> - Only show specific instructions"
return False
def do_range(self, s, silent=False):
try:
rangeS = int(s.split('-')[0], 16)
rangeE = int(s.split('-')[1], 16)
self.__options.range = s.split()[0]
except:
if not silent:
return self.help_range()
return False
if rangeS > rangeE:
if not silent:
print "[-] The start value must be greater than the end value"
return False
if not silent:
print "[+] Range setted. You have to reload gadgets"
def help_range(self):
print "Syntax: range <start-and> - Search between two addresses (0x...-0x...)"
return False
def do_settings(self, s, silent=False):
print "All: %s" %(self.__options.all)
print "Badbytes: %s" %(self.__options.badbytes)
print "Binary: %s" %(self.__options.binary)
print "Depth: %s" %(self.__options.depth)
print "Filter: %s" %(self.__options.filter)
print "Memstr: %s" %(self.__options.memstr)
print "MultiBr: %s" %(self.__options.multibr)
print "NoJOP: %s" %(self.__options.nojop)
print "NoROP: %s" %(self.__options.norop)
print "NoSYS: %s" %(self.__options.nosys)
print "Offset: %s" %(self.__options.offset)
print "Only: %s" %(self.__options.only)
print "Opcode: %s" %(self.__options.opcode)
print "ROPchain: %s" %(self.__options.ropchain)
print "Range: %s" %(self.__options.range)
print "RawArch: %s" %(self.__options.rawArch)
print "RawMode: %s" %(self.__options.rawMode)
print "String: %s" %(self.__options.string)
print "Thumb: %s" %(self.__options.thumb)
def help_settings(self):
print "Display setting's environment"
return False
def do_nojop(self, s, silent=False):
try:
arg = s.split()[0]
except:
return self.help_nojop()
if arg == "enable":
self.__options.nojop = True
if not silent:
print "[+] NoJOP enable. You have to reload gadgets"
elif arg == "disable":
self.__options.nojop = False
if not silent:
print "[+] NoJOP disable. You have to reload gadgets"
else:
if not silent:
return self.help_nojop()
return False
def help_nojop(self):
print "Syntax: nojop <enable|disable> - Disable JOP search engin"
return False
def do_norop(self, s, silent=False):
try:
arg = s.split()[0]
except:
return self.help_norop()
if arg == "enable":
self.__options.norop = True
if not silent:
print "[+] NoROP enable. You have to reload gadgets"
elif arg == "disable":
self.__options.norop = False
if not silent:
print "[+] NoROP disable. You have to reload gadgets"
else:
if not silent:
return self.help_norop()
return False
def help_norop(self):
print "Syntax: norop <enable|disable> - Disable ROP search engin"
return False
def do_nosys(self, s, silent=False):
try:
arg = s.split()[0]
except:
return self.help_nosys()
if arg == "enable":
self.__options.nosys = True
if not silent:
print "[+] NoSYS enable. You have to reload gadgets"
elif arg == "disable":
self.__options.nosys = False
if not silent:
print "[+] NoSYS disable. You have to reload gadgets"
else:
if not silent:
return self.help_nosys()
return False
def help_nosys(self):
print "Syntax: nosys <enable|disable> - Disable SYS search engin"
return False
def do_thumb(self, s, silent=False):
try:
arg = s.split()[0]
except:
return self.help_thumb()
if arg == "enable":
self.__options.thumb = True
if not silent:
print "[+] Thumb enable. You have to reload gadgets"
elif arg == "disable":
self.__options.thumb = False
if not silent:
print "[+] Thumb disable. You have to reload gadgets"
else:
if not silent:
return self.help_thumb()
return False
def help_thumb(self):
print "Syntax: thumb <enable|disable> - Use the thumb mode for the search engine (ARM only)"
return False
def do_all(self, s, silent=False):
if s == "enable":
self.__options.all = True
if not silent:
print "[+] Showing all gadgets enabled. You have to reload gadgets"
elif s == "disable":
self.__options.all = False
if not silent:
print "[+] Showing all gadgets disabled. You have to reload gadgets"
else:
if not silent:
return self.help_all()
return False
def help_multibr(self):
print "Syntax: multibr <enable|disable> - Enable/Disable multiple branch gadgets"
return False
def do_multibr(self, s, silent=False):
if s == "enable":
self.__options.multibr = True
if not silent:
print "[+] Multiple branch gadgets enabled. You have to reload gadgets"
elif s == "disable":
self.__options.multibr = False
if not silent:
print "[+] Multiple branch gadgets disabled. You have to reload gadgets"
else:
if not silent:
return self.help_all()
return False
def help_all(self):
print "Syntax: all <enable|disable - Show all gadgets (disable removing duplice gadgets)"
return False
class Main():
def __init__(self, auto_login=True):
log.info('Main init started')
self.binary = Binary(auto_login)
self.rl = RL()
self.loadQ()
log.info('Main init ended')
def loadQ(self):
log.info('Q loading...')
q_key = ndb.Key(Q, 'main')
self.q = q_key.get()
if not self.q:
self.q = Q(key=q_key)
# validate properties
if not self.q.data:
self.q.data = {}
if not self.q.visits:
self.q.visits = {}
runs = Run.query().fetch()
for run in runs:
if run.getState() not in self.q.visits:
self.q.visits[run.getState()] = 0
self.q.visits[run.getState()] += 1
log.info('Q loaded {0}'.format(len(self.q.data)))
def saveQ(self):
log.info('Q saving...')
self.q.put()
log.info('Q saved {0}'.format(len(self.q.data)))
def new(self):
'''Create new iteration'''
log.info('Main new started')
currency, time_frame, trade_base, trade_aim = self.rl.selectNew(self.q)
run_key = ndb.Key('Run', str(dt.datetime.utcnow()))
run = Run(
key=run_key,
currency=currency,
time_frame=time_frame,
trade_base=trade_base,
trade_aim=trade_aim,
step=1,
payout=1.,
ended_at=dt.datetime.utcnow() + dt.timedelta(minutes=int(time_frame)),
)
if self.binary.createNew(run):
run.put()
log.info('New run: {0}'.format(run))
log.info('Main new ended')
def existing(self):
'''Go through all existing iterations'''
log.info('Main existing started')
runs = Run.query(Run.is_finished == False).fetch()
log.info('{0} runs found'.format(len(runs)))
if len(runs):
profit_table = self.binary.getProfitTable()
# continue every run
for run in runs:
log.info('Run: finding profit for {0}'.format(run.binary_ref))
# time frame ending?
if run.ended_at > dt.datetime.utcnow() + dt.timedelta(seconds=60):
log.info('Run: skipping till {0}'.format(run.ended_at))
continue
# wait for result
profit_table_update_delay = dt.timedelta(seconds=15)
to_sleep = max(0, int((run.ended_at - dt.datetime.utcnow() + profit_table_update_delay).total_seconds()))
if to_sleep > 0:
log.info('Run: waiting for {0} seconds'.format(to_sleep))
time.sleep(to_sleep + 5)
log.info('Run: refreshing profit table...')
profit_table = self.binary.getProfitTable()
# get result
if run.binary_ref in profit_table:
parent_stake = run.stake_parent if run.stake_parent else 0.
run.profit = profit_table[run.binary_ref]
run.profit_net = run.profit + run.profit_parent
run.stake_net = run.stake + parent_stake
run.is_win = True if run.profit > 0 else False
run.is_finished = True
run.put()
log.info('Run: finished with profit {0:.2f}'.format(run.profit))
# continue to cancel loss?
if not run.is_win:
run_child = self.martingale(run)
else:
# update q
self.q = self.rl.updateQ(self.q, run)
else:
log.error('{0} has no profit/loss in table'.format(run.binary_ref))
log.info('Main existing ended')
def martingale(self, run_parent):
log.info('Martingale: loss for {0}'.format(run_parent.binary_ref))
# a child is born
run_child_key = ndb.Key('Run', str(dt.datetime.utcnow()))
run_child = Run(
key=run_child_key,
currency=run_parent.currency,
time_frame=run_parent.time_frame,
trade_base=run_parent.trade_base,
trade_aim=run_parent.trade_aim,
parent_run=run_parent.key,
profit_parent=run_parent.profit_net,
stake_parent=run_parent.stake,
step=run_parent.step + 1,
payout=run_parent.payout * 2,
ended_at=dt.datetime.utcnow() + dt.timedelta(minutes=int(run_parent.time_frame)),
)
# a child is registered
if self.binary.createNew(run_child):
run_child.put()
log.info('New martingale run: {0}'.format(run_child))
log.info('Martingale: created new run')
return run_child
def notifyMe(self):
log.info('Main Notifying me')
log.info('Main waiting for trading to finish')
time.sleep(60)
log.info('Main waiting finished')
started_at = dt.datetime.utcnow() + dt.timedelta(hours=-1)
ended_at = dt.datetime.utcnow()
runs = Run.query(Run.ended_at >= started_at, Run.ended_at <= ended_at, Run.is_finished == True, Run.is_win == True).fetch()
log.info('Fetched {0} runs from {1} till {2}'.format(len(runs), started_at, ended_at))
# exit if nothing
if not runs:
log.warn('Exiting as there is no runs!')
return
net_profit = 0.
stakes = 0.
runs_size = len(runs) + 0.
fields = ['binary_ref', 'time_frame', 'trade_base', 'trade_aim', 'step', 'profit_parent', 'stake_parent', 'stake', 'probability', 'payout', 'profit_net']
# table header
table = '<table width=100%" border="1"><thead><tr>'
for field in fields:
table += '<th>{0}</th>'.format(field)
table += '</tr></thead>'
# table body
table += '<tbody>'
for run in runs:
# update results
net_profit += run.profit_net
stakes += run.stake
row = '<tr>'
for field in fields:
row += '<td>{0}</td>'.format(getattr(run, field))
row += '</tr>'
log.info('Row: {0}'.format(row))
table += row
while run.step > 1:
run = run.parent_run.get()
stakes += run.stake
row = '<tr><td> </td>'
for field in fields[1:-1]:
row += '<td>{0}</td>'.format(getattr(run, field))
row += '<td> </td></tr>'
log.info('Row: {0}'.format(row))
table += row
table += '</tbody></table>'
# pprint(table)
subject = '[{0:.2f}] {1} runs totalling {2:.2f} profit with {3:.2f} stakes'.format(
net_profit / stakes,
runs_size,
net_profit,
stakes,
)
log.info('Subject: {0}'.format(subject))
msg = mail.EmailMessage(
sender='*****@*****.**',
subject=subject,
to='*****@*****.**',
)
msg.body = table
msg.html = '<html><body>' + table + '</body></html>'
msg.send()
log.info('Main Me notified')
