#! python3 import shutil, os, re datePattern = re.compile( r"""^(.*?) ((0|1?\d)- ((0|1|2|3)?\d)- ((19|20)\d\d) (.*?)$ """, re.VERBOSE) for amerFilename in os.lsitdir('.'): mo = datePattern.search(amerFilename) if mo == None: continue beforePart = mo.group(1) monthPart = mo.group(2) dayPart = mo.group(4) yearPart = mo.group(6) afterPart = mo.group(8) euroFilename = beforePart + dayPart + '-' + monthPart + '-' + yearPart + afterPart absWorkingDir = os.path.abspath('.') amerFilename = os.path.join(absWorkingDir, amerFilename) euroFilename = os.path.join(absWorkingDir, euroFilename) print(f'Renaming "{amerFilename}" to "{euroFilename}"...')
def train(model, config, experiment, training_directory=None, validation_directory=None, file_identifier=None): if training_directory is None: training_directory = config['dataset']['training_directory'] if validation_directory is None: validation_directory = config['dataset']['validation_directory'] # get file path weights_file = f'{file_identifier}_{config["experiment"]["name"]}_weights.h5' model_file = f'{file_identifier}_{config["experiment"]["name"]}_model.h5' # get plot file names accuracy_plot_file = f'{file_identifier}_accuracy_plot.png' loss_plot_file = f'{file_identifier}_loss_plot.png' # plot file paths accuracy_plot_path = os.path.join('../tmp', accuracy_plot_file) loss_plot_path = os.path.join('../tmp', loss_plot_file) # weights/model file paths model_path = os.path.join('../tmp', model_file) weights_path = os.path.join('../tmp', weights_file) # set patience (number of epochs where val_acc does not improve before stopping) patience = config['hyper_parameters'].get('patience', 0) baseline = config['hyper_parameters'].get('baseline', 0) monitor = config['hyper_parameters'].get('monitor', 'val_acc') # callbacks to be called after every epoch callbacks = [ ModelCheckpoint(weights_path, monitor=monitor, verbose=1, save_best_only=True), TensorBoard(log_dir=os.path.join('../tmp', 'logs'), batch_size=8) ] # number of splits in dataset split = config['dataset']['split'] # number of samples in dataset samples = config['dataset']['samples'] # metrics to measure under training metrics = ['accuracy'] # set number of training and validation samples training_samples = samples - (samples // split ) # number of training samples in dataset validation_samples = samples // split # number of validation samples in dataset # set number of epochs epochs = config['hyper_parameters']['epochs'] # set batch size batch_size = config['hyper_parameters']['batch_size'] # set number of layers to freeze (not train) freeze_layers = config['hyper_parameters']['freeze_layers'] # number of classes in dataset number_of_classes = config['dataset']['number_of_classes'] # set image dimensions image_width = config['image_processing'][ 'image_width'] # change based on the shape/structure of your images image_height = config['image_processing'][ 'image_height'] # change based on the shape/structure of your images image_channels = config['image_processing'][ 'image_channels'] # number of image channels # set training steps based on training sampels and batch size training_steps = training_samples // batch_size # number of training batches in one epoch validation_steps = validation_samples // batch_size # number of validation batches in one epoch # build optimizer optimizer = build_optimizer(config['optimizer']) training_generator_file = config['image_processing'][ 'training_data_generator'] validation_generator_file = config['image_processing'][ 'validation_data_generator'] # build data generators training_data_generator = importlib.import_module( f'generators.{training_generator_file}').train_data_generator validation_data_generator = importlib.import_module( f'generators.{validation_generator_file}').validation_data_generator # freeze layers based on freeze_layers parameter for layer in model.layers[:freeze_layers]: layer.trainable = False for layer in model.layers[freeze_layers:]: layer.trainable = True # initialize training generator training_generator = training_data_generator.flow_from_directory( training_directory, target_size=(image_width, image_height), batch_size=batch_size, class_mode='categorical', follow_links=True) # initialize validation generator validation_generator = validation_data_generator.flow_from_directory( validation_directory, target_size=(image_width, image_height), batch_size=batch_size, class_mode='categorical', follow_links=True) # only set early stopping if patience is more than 0 if patience > 0: # append early stopping to callbacks callbacks.append( EarlyStopping(monitor=monitor, patience=patience, verbose=0, baseline=baseline)) # print out the class indicies for sanity print('train indicies: ', training_generator.class_indices) print('validation indicies: ', validation_generator.class_indices) # compile model model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=metrics) # difference in class counts class_dispersion = {} class_list = os.listdir(training_directory) class_list.sort() class_weigths = None if config['hyper_parameters']['class_weights']: for class_index, class_name in enumerate(class_list): class_dispersion[class_index] = len( os.lsitdir(os.path.join(training_directory, class_name))) class_weigths = create_class_weight(class_dispersion) for index, weight in class_weigths.items(): print(f'{class_list[index]}: ', weight) # train model and get training metrics history = model.fit_generator(training_generator, steps_per_epoch=training_steps, epochs=epochs, validation_data=validation_generator, validation_steps=validation_steps, callbacks=callbacks, workers=0, use_multiprocessing=False, class_weight=class_weigths) # plot loss plot_loss(history, loss_plot_path) # plot accuracy plot_accuracy(history, accuracy_plot_path) # add plots to experiment experiment.add_artifact(accuracy_plot_path) experiment.add_artifact(loss_plot_path) # load best weights if os.path.isfile(weights_path): model.load_weights(weights_path) experiment.add_artifact(weights_path) # save keras model file model.save(model_path) # add keras model file to experiment experiment.add_artifact(model_path) # return training history metrics return model
self.defdict["AUTHOR"].append(self.article.author) self.defdict["DATELINE"].append(self.article.dateline) self.defdict["EXCHANGES"].append(self.article.exchanges) self.defdict["COMPANIES"].append(self.article.companies) self.defdict["ORGS"].append(self.article.orgs) self.defdict["MKNOTE"].append(self.article.mknote) #create new Article object self.article = Article() self.in_d = False # main function: directory = 'reuters21578/sgml_files' files = os.lsitdir(directory) # set up handler and parser parser = xml.sax.make_parser() handler = MyHandler() parser.setContentHandler(handler) # loop through 22 files for f in files: cur_file = directory + f #only open sgm files if f.endswith('.sgm') and f.startswith('reut'): parser.parse(cur_file) # csv, with doc id and "LEWISSPLIT" train/test information
if a == "Y": while True: add_user_name = input("Username:"******"Password:"******"limit(MB):") add_user_limit *= 1048576 adduser(add_user_name, add_user_passwd, add_user_limit) a = input("Do you want to go back?\n(c:Continue)[Y/n/c]:") if a == "Y": db.close() exit() elif a == "c": break # Add users from subscription. os.chdir("subscription") a = input("Do you want to add all users from subscritpion?[Y/n]:") if a == "Y": add_user_limit = input("Default user limit(MB):") filename = os.lsitdir() for i in filename: add_user_name = radom.randint(0, 10000) add_user_passwd = i add_user_limit *= 1048576 adduser(add_user_name, add_user_passwd, add_user_limit) print("All done!") db.close() a = input("Press Enter to go back.") exit() if a == "Y" else exit()
if len(args) < 1: print "[!] Not enough Arguments, Need at least file path" parser.print_help() sys.exit() # Check for file or dir is_file = os.path.isfile(args[0]) is_dir = os.path.isdir(args[0]) if options.recursive: if not is_dir: print "[!] Recursive requires a directory not a file" sys.exit() # Read all the things for filename in os.lsitdir(args[0]): file_data = open(os.path.join(args[0], filename), 'rb').read() print "[+] Reading {0}".format(filename) config_data = run(file_data) else: if not is_file: print "[!] You did not provide a valid file." sys.exit() # Read in the file. file_data = open(os.path.join(args[0], args[0]), 'rb').read() print "[+] Reading {0}".format(args[0]) config_data = run(file_data) print_output(config_data, options.output)
import zipfile local_zip = './horse-or-human.zip' zip_ref = zipfile.Zipfile(local_zip, 'r') zip_ref.extractall('./horse-or-human') zip_ref.close() # Define Directories train_horse_dir = os.path.join('./horse-or-human/horses') train_human_dir = os.path.join('./horse-or-human/humans') train_horse_names = os.listdir(train_horse_dir) train_human_names = os.listdir(train_human_dir) print('total training horse images: ', len(os.listdir(train_horse_dir))) print('total training human images: ', len(os.lsitdir(train_human_dir))) # Draw Image import matplotlib.pyplot as plt import matplotlib.image as mpimg ## parameters for the graph nrows = 4 ncols = 4 ## index iterating over images pic_index = 0 ## set up matplotlib fig, and size it to fit 4*4 pics fig = plt.gcf() fig.set_size_inches(ncols * 4, nrows * 4)
if len(args) < 1: print "[!] Not enough Arguments, Need at least file path" parser.print_help() sys.exit() # Check for file or dir is_file = os.path.isfile(args[0]) is_dir = os.path.isdir(args[0]) if options.recursive: if not is_dir: print "[!] Recursive requires a directory not a file" sys.exit() # Read all the things for filename in os.lsitdir(args[0]): file_data = open(os.path.join(args[0], filename), 'rb').read() print "[+] Reading {0}".format(filename) config_data = run(file_data) else: if not is_file: print "[!] You did not provide a valid file." sys.exit() # Read in the file. file_data = open(os.path.join(args[0], args[0]), 'rb').read() print "[+] Reading {0}".format(args[0]) config_data = run(file_data) print_output(config_data, options.output)
import os import sqlite3 def printTables(iphoneDB): try: conn = sqlite3.connect(iphoneDB) c = conn.cursor() c.execute("SELECT tbl_name FROM sqlite_master \ WHERE type=='table';") print("\n[*] Database: " + iphoneDB) for row in c: print("[-] Table: " + str(row)) except: pass conn.close() dirList = os.lsitdir(os.getcwd()) for fileName in dirList: printTables(fileName)