def gen_input(): dir = 'C:/Workspace/Bills' files = [ 'Doc-Respicio-Test-001', 'Doc-Respicio-Test-002', 'Doc-Respicio-Test-003' ] vocab_name = 'Respicio-pp' ext = 'txt' inputs = list() #load vocabulary set vocab = list() vocab_file_name = dir + '/' + vocab_name + '.' + ext if path.isfile(vocab_file_name): temp = open(vocab_file_name).read() vocab = re.split('\s', temp) for f in files: label = np.zeros(len(files), dtype=int) features = np.zeros(len(vocab), dtype=int) filepath = dir + '/' + f + '.' + ext if path.isfile(filepath) == True: words = re.split('\s', open(filepath).read()) for word in words: idx = vocab.index(word) if idx != -1: features[idx] = 1 label[files.index(f)] = 1 inputs.append((features, label)) pp = pprint.PrettyPrinter(indent=4) pp.pprint(inputs)
def getCompilationFiles (): # Get the files to translate from the mandatory 'config' file from os import path from string import split configFile = None if opts.config: configFile = opts.config else: configFile = 'config' if not path.isfile(configFile): debug.exitMessage("Unable to find configuration file '%s' with the files to translate." % (configFile)) filesToCompile = [] for line in open(configFile, 'r'): line = line.strip() words = line.split('=') if line.startswith('files'): files = words[1].strip().split(' ') for f in files: f = f.strip() filesToCompile.append(f) if not path.isfile(f): debug.exitMessage("File '" + f + "' does not exist.") if not filesToCompile: debug.exitMessage("You did not specify which files need to compiled. Use files=<list/of/files> in the configuration file.") return filesToCompile
def install(): if path.isfile("/etc/apt/sources.list"): distro = "debian" command = "sudo apt-get install unrar-free unzip wget" py = "python3" elif path.isfile("/etc/pacman.conf"): distro = "arch" command = "sudo pacman -S unrar unzip wget" py = "python" elif path.isfile("/etc/yum.conf"): distro = "fedora" command = "sudo yum install unrar unzip wget" py = "python3" try: print(" [+] Creating symlink") with open("organon", "w") as symlink: symlink.write(\ """ #!/bin/bash cd /usr/share/organon %s organon.py $@""" % py) check_call("sudo mv organon /usr/bin", shell=True) print(green + " [+] Changing permission") check_call("sudo chmod +x /usr/bin/organon", shell=True) print(" [+] creating configuration file") with open("organon.conf", "w") as conf: conf.write("distro = %s" % distro) conf.write("\narch = %s" % machine()) check_call( "sudo mkdir /etc/organon && sudo mv organon.conf /etc/organon && sudo touch /etc/organon/installed.db", shell=True) check_call("sudo cp etc/mirrors /etc/organon", shell=True) check_call("sudo chmod 777 /etc/organon/installed.db", shell=True) print(" [+] Installing MAN page") check_call("sudo install -Dm644 doc/organon.8 /usr/share/man/man8/", \ shell = True) print(" [+] Creating organon\'s cache") check_call("sudo mkdir /var/cache/organon", shell=True) print(" [+] Moving organon to /usr/share") check_call("sudo mv ../organon /usr/share", shell=True) print(" [+] Installing dependencies" + default) check_call(command, shell=True) except (CalledProcessError, KeyboardInterrupt) as e: print(red + " [!] ainn. Something went wrong") print(str(e) + str(default)) exit()
def install(): if path.isfile("/etc/apt/sources.list"): distro = "debian" command = "sudo apt-get install unrar-free unzip wget" py = "python3" elif path.isfile("/etc/pacman.conf"): distro = "arch" command = "sudo pacman -S unrar unzip wget" py = "python" elif path.isfile("/etc/yum.conf"): distro = "fedora" command = "sudo yum install unrar unzip wget" py = "python3" try: print(" [+] Creating symlink") with open("organon", "w") as symlink: symlink.write(\ """ #!/bin/bash cd /usr/share/organon %s organon.py $@""" % py) check_call("sudo mv organon /usr/bin", shell = True) print(green + " [+] Changing permission") check_call("sudo chmod +x /usr/bin/organon", shell = True) print(" [+] creating configuration file") with open("organon.conf", "w") as conf: conf.write("distro = %s" % distro) conf.write("\narch = %s" % machine()) check_call("sudo mkdir /etc/organon && sudo mv organon.conf /etc/organon && sudo touch /etc/organon/installed.db", shell = True) check_call("sudo cp etc/mirrors /etc/organon", shell = True) check_call("sudo chmod 777 /etc/organon/installed.db", shell = True) print(" [+] Installing MAN page") check_call("sudo install -Dm644 doc/organon.8 /usr/share/man/man8/", \ shell = True) print(" [+] Creating organon\'s cache") check_call("sudo mkdir /var/cache/organon", shell = True) print(" [+] Moving organon to /usr/share") check_call("sudo mv ../organon /usr/share", shell = True) print(" [+] Installing dependencies" + default) check_call(command, shell = True) except (CalledProcessError, KeyboardInterrupt) as e: print(red + " [!] ainn. Something went wrong") print(str(e) + str(default)) exit()
def prepImage(file): ''' get .png image, if svg file specified then convert image to ong and apply skeletonization ''' from os import path img = np.zeros((HEIGHT, WIDTH)) if not path.isfile(file.split(".")[0] + ".png"): # given svg file, get a png file print("INFO : TARGET PNG IMAGE NOT FOUND, CREATING ...") from svglib.svglib import svg2rlg drawing = svg2rlg(file) from reportlab.graphics import renderPM renderPM.drawToFile(drawing, file.split(".")[0] + ".png", fmt="PNG") img = cv.imread(file.split(".")[0] + ".png", 0) # get GRAYSCALE image img = cv.resize(img, (100, 100), cv.INTER_CUBIC) # TODO : apply blur ex : medianBlur(img, 5) # load generated image and apply image transformations thresh = cv.THRESH_BINARY_INV _, img = cv.threshold( img, 0, 255, thresh + cv.THRESH_OTSU) # use otsu to find appropriate threshold img[np.where( img > 0)] = 1 # convert to image with 0's and 1's ex : binary image img = skeletonize( img, method="lee") # convert width of stroke to one pixel wide return img
def get_config(file): if not path.isfile(file): return None with open(file, "r") as f: raw_config = f.read() return json2obj(raw_config)
def get_added_counties_jsons_names(self, dirname): files = [] for f in listdir(dirname): if path.isfile(Path(dirname, f)): files.append(f) self.logger.log( logging.INFO, "Got all the added counties jsons from the file {}".format( dirname)) return files
def process_inputs(): dir = 'C:/Workspace/Bills/input' ext = '*-raw.txt' inputs = list() dirs = ['train', 'test'] for d1 in dirs: files = glob(join(dir, d1, ext)) for f in files: d = set([]) if path.isfile(f) == True: txtfile = open(f).readlines() raw = list() for line in txtfile: emails = regex_email.findall(line) if len(emails): for email in emails: raw.append(email) else: flag, txt = process_txt(line) if flag: if len(txt) > 2: raw.append(txt) if len(raw) > 0: for sentence in raw: emails = regex_email.findall(sentence) if len(emails): words = emails else: words = nltk.word_tokenize(sentence) d = d | set(words) sd = set(sorted(d)) vocab = set([]) porter = nltk.PorterStemmer() for stemming in sd: if not is_key_word(stemming): stemmed_word = porter.stem(stemming) else: stemmed_word = stemming vocab.add(stemmed_word) head, tail = path.split(f) find_idx = tail.rfind('.txt') if find_idx != -1: s = tail.replace('-raw.txt', '-input.txt') txtfilepath = path.join(head, s) txtf = open(txtfilepath, 'w+') for item in vocab: txtf.write(item) txtf.write('\n') txtf.close()
def stidy(structure, ang, d1, d2, d3): PLATON = find_executable('platon') if not PLATON: PLATON = '../bin/platon' with NamedTemporaryFile(suffix='.cif') as temp_file: # write temporary cif file CifWriter(structure).write_file(temp_file.name) temp_file.flush() # run ADDSYM_SHX to make PLATON recognize symmetries addsym_shx_process = Popen(['platon', '-o', temp_file.name], stdout=PIPE, stderr=STDOUT, stdin=PIPE) try: addsym_shx_process.communicate( input=b'ADDSYM_SHX {} {} {} {}'.format(ang, d1, d2, d3)) except TimeoutExpired as t: return ExitCode(408, 'ADDSYM_SHX timed out: {}'.format(t)) except Exception as e: return ExitCode(500, 'ADDSYM_SHX crashed: {}'.format(e)) # call STIDY on the ADDSYM_SHX output temp_file_dirname, temp_file_basename = path.split(temp_file.name) temp_file_basename_extless, _ = path.splitext(temp_file_basename) temp_file_basename_spf = temp_file_basename_extless + '_pl.spf' temp_file_spf = path.join(temp_file_dirname, temp_file_basename_spf) if not os.path.isfile(temp_file_spf): return ExitCode(500, 'ADDSYM_SHX failed to write *_pl.spf file') stidy_process = Popen(['platon', '-o', temp_file_spf], stdout=PIPE, stderr=STDOUT, stdin=PIPE) try: stidy_data = stidy_process.communicate(input=b'STIDY') except TimeoutExpired as t: return ExitCode(408, 'STIDY timed out: {}'.format(t)) except Exception as e: return Exitcode(500, 'STIDY crashed: {}'.format(e)) stidy_output = stidy_data[0].decode('utf-8') # clean up files if path.isfile('check.def'): remove('check.def') return stidy_output
def process_img(self, path_img, path_raw): log.info('AI::VERA - %s.process_img path=%s', type(self).__name__, path_img) if path.isfile(path_img) == True: img = Image.open(path_img) log.info('AI::VERA - Running OCR on %s', path_img) txt = pytesseract.image_to_string(img) log.info('AI::VERA - OCR result len(txt)=%s', len(txt)) flag,raw = self.process_txt(txt) if flag: raw_f = open(path_raw, 'w+') for i in raw: raw_f.write(i) raw_f.write('\n') raw_f.close() log.info('AI::VERA - raw text file %s has been created', path_raw) else: log.error('AI::VERA - failed to save raw text file %s', path_raw)
def prepImage(file, flag): ''' get target image for testing only : flag : 1 -> from svg : write image from svg data 0 -> from png : apply threshold and skeletonise image ''' from os import path img = np.zeros((HEIGHT, WIDTH)) if not flag: # TODO: FIX skeletonize # get target image from png print("INFO : TARGET IMAGE FROM PNG") if not path.isfile(file.split(".")[0] + ".png"): # get a png file from svg file print("INFO : TARGET PNG IMAGE NOT FOUND, CREATING ...") from svglib.svglib import svg2rlg drawing = svg2rlg(file) from reportlab.graphics import renderPM renderPM.drawToFile(drawing, file.split(".")[0] + ".png", fmt="PNG") img = cv.imread(file.split(".")[0] + ".png") img = cv.resize(img, (100, 100), cv.INTER_CUBIC) # load generated image and apply image transformations thresh = cv.THRESH_BINARY_INV # img = cv.imread('./res/japanese.png') # 2 - RBG -> GRAYSCALE img = cv.cvtColor(img, cv.COLOR_BGR2GRAY) img = cv.medianBlur(img, 5) _, img = cv.threshold(img, thresh_val, 255, thresh) img[np.where( img > 0 )] = 1 # convert to image with 0's and 1's ex : binary image img = skeletonize( img, method="lee") # convert width of stroke to one pixel wide else: # get target image from svg print("INFO : TARGET IMAGE FROM SVG") X_target, m_indices = getStrokesIndices(open(file, 'r').read()) img = drawStroke(X_target) return img
def run(): dir = 'C:/Workspace/Bills' files = [ 'Doc-Respicio-001', 'Doc-Respicio-002', 'Doc-Respicio-003', 'Doc-Respicio-004' ] ext = 'jpg' for f in files: filepath = dir + '/' + f + '.' + ext if path.isfile(filepath) == True: img = Image.open(filepath) txt = pytesseract.image_to_string(img) txtfilepath = dir + '/' + f + '.txt' flag, raw = process_txt(txt) if flag: txtf = open(txtfilepath, 'w') for i in raw: txtf.write(i) txtf.write('\n') txtf.close() err = 0
def getFreeVariablesModuleName (): # Get the name of the Fortran module containing free variables referenced in the kernel functions from string import split from os import path configFile = 'config' if not path.isfile(configFile): debug.exitMessage("Unable to find configuration file '%s' with the files to translate." % (configFile)) filename = None f = open(configFile, 'r') for line in f: line = line.strip() words = line.split('=') if line.startswith('freeVariables'): filename = words[1].strip().split(' ')[0] f.close () if filename is None: debug.exitMessage("You did not specify which file contains the free variables. Use freeVariables=<modulename> in the configuration file.") return filename
def process_raw(self, ai_path, vera_path): d = set([]) if path.isfile(ai_path) == True: ai_file = open(ai_path).readlines() raw = list() for line in ai_file: keywords, the_rest = self.get_keywords(line) if len(keywords): for keyword in keywords: raw.append(keyword) flag, txt = self.process_txt(the_rest) if flag: if len(txt) > 2: raw.append(txt) if len(raw) > 0: for sentence in raw: words = [] keywords, the_rest = self.get_keywords(sentence) if len(keywords): words.extend(keywords) if len(the_rest) > 0: words.extend(nltk.word_tokenize(the_rest)) d = d | set(words) sd = set(sorted(d)) vocab = set([]) porter = nltk.PorterStemmer() for stemming in sd: if not self.is_key_word(stemming): stemmed_word = porter.stem(stemming) else: stemmed_word = stemming vocab.add(stemmed_word) vera_f = open(vera_path, 'w+') for item in vocab: vera_f.write(item) vera_f.write('\n') vera_f.close()
def process_vocab(): dir = 'C:/Workspace/Bills/input' ext = '*-input.txt' d = set([]) dirs = ['train', 'test'] for d1 in dirs: vocab = set([]) files = glob(join(dir, d1, ext)) for f in files: d = set([]) if path.isfile(f) == True: txtfile = open(f).readlines() raw = list() for line in txtfile: raw.append(line) if len(raw) > 0: d = d | set(raw) vocab = vocab | d head, tail = path.split(f) txtfilepath = join(head, 'Respicio-pp.txt') txtf = open(txtfilepath, 'w+') for item in vocab: txtf.write(item) txtf.close()
def run(): ext = '*.jpg' dir = 'C:/Workspace/Bills' ext_txt = 'txt' files = glob(join(dir, 'image', ext)) txtdir = path.join(dir, 'input') for f in files: if path.isfile(f) == True: img = Image.open(f) txt = pytesseract.image_to_string(img) head, tail = path.split(f) find_idx = tail.rfind('.jpg') new_tail = tail if find_idx != -1: new_tail = tail.replace('.jpg', '-raw.txt') txtfilepath = path.join(txtdir, new_tail) flag, raw = process_txt(txt) if flag: txtf = open(txtfilepath, 'w+') for i in raw: txtf.write(i) txtf.write('\n') txtf.close() err = 0
shell=True) if __name__ == "__main__": if version[0] != "3": print("%s [!] Please execute Organon with Python 3.x %s"\ %(red, default)) print(""" Arch/Manjaro %s>>%s pacman -S python Debian/Ubuntu %s>>%s apt-get install python3 Fedora/CentOS %s>>%s yum install python3 """ % (yellow, default, yellow, default, yellow, default)) if len(argv) == 1 or argv[1] == "--help": print("Usage: python setup.py [install || uninstall]") exit() else: if getuid() == 0: print(getuid) print(" [!] Are you root? Please do NOT run this script as root") exit() else: if argv[1] == "install": if path.isfile("/usr/share/organon/organon.py"): print("Remove organon before continue") print("python setup.py uninstall") else: install() elif argv[1] == "uninstall": uninstall()
check_call("sudo rm -rf /usr/share/organon /var/cache/organon /etc/organon\ /usr/bin/organon", shell = True) if __name__ == "__main__": if version[0] != "3": print("%s [!] Please execute Organon with Python 3.x %s"\ %(red, default)) print(""" Arch/Manjaro %s>>%s pacman -S python Debian/Ubuntu %s>>%s apt-get install python3 Fedora/CentOS %s>>%s yum install python3 """ %(yellow, default,yellow, default,yellow, default)) if len(argv) == 1 or argv[1] == "--help": print("Usage: python setup.py [install || uninstall]") exit() else: if getuid() == 0: print(getuid) print(" [!] Are you root? Please do NOT run this script as root") exit() else: if argv[1] == "install": if path.isfile("/usr/share/organon/organon.py"): print("Remove organon before continue") print("python setup.py uninstall") else: install() elif argv[1] == "uninstall": uninstall()
'stroke_opacity': 1.0, 'stroke_weight': 3, 'fill_color': map_colors[district.id], 'fill_opacity': .5, 'path': [], 'name': district.name } coord = ast.literal_eval(district.coordinates) for c in coord: pol['path'].append([c[1], c[0]]) polygons.append(pol) return polygons, legend if __name__ == '__main__': from os import path import os extra_dirs = ['static', 'templates'] extra_files = extra_dirs[:] for extra_dir in extra_dirs: for dirname, dirs, files in os.walk(extra_dir): for filename in files: filename = path.join(dirname, filename) if path.isfile(filename): extra_files.append(filename) app.run(host='0.0.0.0', extra_files=extra_files, debug=True, port=int(os.environ.get('PORT', 5000)))
def run(config): """ :param config: GlobalParameters option instance :type config: `GlobalParameters` :raises: TypeError """ if not isinstance(config, GlobalParameters): raise TypeError("Expected GlobalParameters, got '%s' instead" % type(config)) # -------------------------------------------------------------------------- # CHECK ROOT USER # -------------------------------------------------------------------------- if geteuid(): six.print_(colored("[!] ERROR - Please run NetPyntest as root.", "red")) exit() # -------------------------------------------------------------------------- # CHECK CONFIG FILE # -------------------------------------------------------------------------- if not path.isfile("control_file"): six.print_("Creating config_file") control_file = open("control_file", "w") data = {'mac_flooding_pid': 0, 'port_stealing_pid': 0} control_file.write(str(data)) control_file.close() # -------------------------------------------------------------------------- # SELECT & LAUNCH ATTACK # -------------------------------------------------------------------------- attack = config.attack[0] action = config.action[0] if config.interface != None: iface = config.interface[0] #TODO valid interface and introduce interface in calls else: iface = "eth0" ################ MAC FLOODING ############## if attack == "mac_flooding": from .libs.plugins.mac_flooding import start from .libs.plugins.mac_flooding import stop from .libs.plugins.mac_flooding import generate_pcap if action == "start":#TODO This is not working for Python 2 from sys import version_info if version_info[0] >=3: if config.file != None: file = config.file[0] if path.isfile(file): six.print_("[*] Starting MAC Flooding with file '{}'...".format(file)) from scapy.error import Scapy_Exception try: start(file, iface) except Scapy_Exception: six.print_(colored("[!] ERROR - File '{}' is not a valid PCAP file".format(file), "red")) else: six.print_(colored("[!] ERROR - File '{}' doesn't exist.".format(file), "red")) else: six.print_(colored("[!] ERROR - You must specify a PCAP file. You can generate one with 'sudo python netpyntest.py mac_flooding generate_pcap'", "red")) else: six.print_(colored("[!] ERROR - Sorry, currently this feature is only supported in Python 3 or higher", "red")) elif action == "stop": stop() elif action == "generate_pcap": if config.size == None: six.print_("[*] Generating PCAP file with default size of 10000 packets") generate_pcap(10000) else: size = config.size[0] six.print_("[*] Generating PCAP file with size of {} packets".format(size)) generate_pcap(size) six.print_(colored("[*] PCAP file generated", "green")) else: six.print_(colored("[!] ERROR - Action {} doesn't exist for MAC Flooding attack".format(action), "red")) ################ PORT STEALING ############## elif attack == "port_stealing": if action == "start": if config.target != None: target = config.target[0] if validate_ip(target): if config.output != None: output = config.output[0] from .libs.plugins.port_stealing import start six.print_("[*] Starting Port Stealing...") start(target, output, iface) else: six.print_(colored("[!] ERROR - No output file specified (-o)", "red")) else: six.print_(colored("[!] ERROR - IP isn't valid. Enter valid IPv4 address (-t)", "red")) else: six.print_(colored("[!] ERROR - You must specify a target (-t)", "red")) elif action == "stop": from .libs.plugins.port_stealing import stop six.print_("[*] Stopping Port Stealing...") stop() else: six.print_(colored("[!] ERROR - Action {} doesn't exist for Port Stealing attack".format(action), "red")) ################ SNMP ############## elif attack == "snmp": if action == "sniff": from .libs.plugins.snmp import sniff_snmp six.print_("[*] Starting SNMP sniffing...") sniff_snmp(iface) elif action == "get": if config.com != None: com = config.com[0] else: com = "public" if config.target != None: target = config.target[0] if validate_ip(target): if config.oid != None: oid = config.oid[0] from .libs.plugins.snmp import snmp_get six.print_("[*] Performing SNMP GET request against host {} and OID {}...".format(target, oid)) snmp_get(target, oid, iface, com) else: six.print_(colored("[!] ERROR - No OID specified (-oid)", "red")) else: six.print_(colored("[!] ERROR - IP isn't valid. Enter valid IPv4 address.", "red")) else: six.print_(colored("[!] ERROR - You must specify a target (-t)", "red")) elif action =="set": if config.com != None: com = config.com[0] else: com = "private" if config.target != None: target = config.target[0] if validate_ip(target): if config.oid != None: oid = config.oid[0] if config.value != None: val = config.value[0] from .libs.plugins.snmp import snmp_set six.print_("[*] Performing SNMP SET request against host {}. Trying to set value {} in object {}...".format(target, val, oid)) snmp_set(target, oid, iface, com, val) else: six.print_(colored("[!] ERROR - No value specified (-v)", "red")) else: six.print_(colored("[!] ERROR - No OID specified (-oid)", "red")) else: six.print_(colored("[!] ERROR - IP isn't valid. Enter valid IPv4 address (-t)", "red")) else: six.print_(colored("[!] ERROR - You must specify a target (-t)", "red")) elif action == "dictionary_attack": if config.target != None: target = config.target[0] if validate_ip(target): if config.dict != None: dict = config.dict[0] if path.isfile(dict): from .libs.plugins.snmp import dictionary_attack six.print_("[*] Starting SNMP dictionary attack...") dictionary_attack(dict, target, iface) else: six.print_(colored("[!] ERROR - File '{}' doesn't exist.".format(dict), "red")) else: six.print_(colored("[!] ERROR - You must specify a dictionary file (-d)", "red")) else: six.print_(colored("[!] ERROR - IP isn't valid. Enter valid IPv4 address (-t)", "red")) else: six.print_(colored("[!] ERROR - You must specify a target (-t, --target)", "red")) elif action == "dos": if config.com != None: com = config.com[0] else: com = "private" if config.target != None: target = config.target[0] if validate_ip(target): from .libs.plugins.snmp import snmp_DoS six.print_("[*] Starting DoS attack to host {} with RW community {}...".format(target, com)) snmp_DoS(target, iface, com) else: six.print_(colored("[!] ERROR - IP isn't valid. Enter valid IPv4 address (-t, --target)", "red")) else: six.print_(colored("[!] ERROR - You must specify a target (-t)", "red")) else: six.print_(colored("[!] ERROR - Action {} doesn't exist for SNMP".format(action), "red"))