def load_model_and_generate(self, model_name='model7_laf', epochs=10): dt = datetime.datetime.now().strftime('_date_%Y-%m-%d_%H-%M-%S') dir_name = './generated_results/pdfs/' + model_name + dt + 'epochs_' + str( epochs) + '/' if not os.path.exists(dir_name): os.makedirs(dir_name) model = load_model( './model_checkpoint/best_models/' 'model7_laf_date_2018-06-19_12-23-39_epoch_30_val_loss_0.8395.h5', compile=False) optimizer = Adam(lr=0.0001) # Reduce from 0.001 to 0.0001 for model_10 model.compile( optimizer=optimizer, loss='categorical_crossentropy', # metrics=['accuracy'] metrics=['accuracy']) seq = self.generate_and_fuzz_new_samples(model=model, model_name=model_name, epochs=epochs, current_epoch=10, dir_name=dir_name) list_of_obj = preprocess.get_list_of_object(seq=seq, is_sort=False) return list_of_obj
def __init__(self, host_id=None, object_file_path=iu_config['baseline_object_path'], stream_directory_path=iu_config['stream_directory_path']): """ :param host_id: Name of host file without postfix, e.g. host1_max, host2_min or host3_avg :param object_file_path: See iu_config, new_objects_path :param stream_directory_path: See iu_config, stream_directory_path """ self.host_id = host_id self.object_file_path = object_file_path self.obj_list = preprocess.get_list_of_object(seq=preprocess.load_from_file(self.object_file_path), is_sort=False) self.stream_directory_path = '../' + stream_directory_path self.stream_filename_list = os.listdir(self.stream_directory_path) # Creating new directory foreach time that program run and we want to generate new test data dt = datetime.datetime.now().strftime(self.host_id + '_date_%Y-%m-%d_%H-%M-%S') self.storage_dir_name = iu_config['new_pdfs_directory'] + self.host_id + '/' + dt + '/' if not os.path.exists(self.storage_dir_name): os.makedirs(self.storage_dir_name) print('new storage directory build.') self.obj_getter = self.obj_generator(self.obj_list)
def get_one_object(): """ provide one pdf data object whether an existing object in corpus or an online new generated object from learnt model this function is not complete yet! """ object_file_path = '../trainset/pdf_object_trainset_100_to_500_percent33.txt' seq = poc.load_from_file(object_file_path) obj_list = poc.get_list_of_object(seq, is_sort=False) random_object_index = random.randint(50, len(obj_list) - 1) obj = obj_list[random_object_index] return obj
def __get_objects_sequence(self): seq = '' for filename in os.listdir(iu_config['new_objects_path']): try: seq += preprocess.load_from_file( iu_config['new_objects_path'] + filename) except Exception as e: print('Extracting failed from %s:' % filename, file=sys.stderr) print(str(e), file=sys.stderr) # finally: obj_list = preprocess.get_list_of_object(seq=seq, is_sort=False) print('obj_list len', len(obj_list)) print(obj_list) # for o in obj_list: # print(o, '\n', '#'*50) # input() return obj_list
def attach_new_object(): """ incremental update pdf file """ host_names = ['host1', 'host2', 'host3'] with open(host_directory + host_names[0] + '.pdf', 'br') as f: data = f.read() print(len(data)) # find last trailer in a pdf file trailer_index = 0 while data.find(b'trailer', trailer_index + 7) != -1: trailer_index = data.find(b'trailer', trailer_index + 7) print('trailer_index', trailer_index) trailer_index_dic_endof = data.find(b'>>', trailer_index) print('trailer_index_dic_endof', trailer_index_dic_endof) trailer_content = data[trailer_index:trailer_index_dic_endof + 2] print('trailer_content', trailer_content) # find last startxref offset in a pdf file startxref_index = trailer_index while data.find(b'startxref', startxref_index + 9) != -1: startxref_index = data.find(b'startxref', startxref_index + 9) # print('index ===', index_startxref) index_eof = data.find(b'%%EOF', startxref_index) # print('index 2===', index_eof) if data[startxref_index + 9] == b'\n' or b'\r': # print('yes', data[index_startxref+9]) startxref_index += 10 if data[index_eof - 1] == b'\n' or b'\r': index_eof -= 1 startxref_offset = int(data[startxref_index:index_eof]) print('startxref_offset', startxref_offset) # print(type(trailer_content)) trailer_content_new = trailer_content[:-2] + b' /Prev ' \ + bytes(str(startxref_offset), 'ascii') + b' \n>>' print('trailer_content_new', trailer_content_new) # print(bytes(str(startxref_offset), 'ascii')) # load the pdf object form file seq = poc.load_from_file( host_directory + 'gen_objs_20180221_142612_epochs10_div1.5_step1.txt') obj_list = poc.get_list_of_object(seq) random_object_index = random.randint(0, len(obj_list) - 1) obj = obj_list[random_object_index] last_object_id = str(get_last_object_id(host_names[0])) random_rewrite_object = str(random.randint(1, int(last_object_id))) print('len object', len(obj)) startxref_offset_new = len(data) + 1 + len( random_rewrite_object) + 3 + len(obj) # if we attach just one obj print('startxref_offset_new', startxref_offset_new) attach_content = bytes(str(random_rewrite_object + ' 0 ' + obj + '\nxref\n0 1\n0000000000 65535 f\n' +\ random_rewrite_object + ' 1\n' + str(len(data)).zfill(10) + ' 00000 n\n'), 'ascii') +\ trailer_content_new + b'\nstartxref\n' + \ bytes(str(startxref_offset_new), 'ascii') + b'\n%%EOF\n' print('attach_content\n', attach_content) new_pdf_file = data + attach_content with open(host_directory + host_names[0] + 'iu_auto7.pdf', 'bw') as f: f.write(new_pdf_file)
def generate_and_fuzz_new_samples(self, model=None, model_name='model_1', epochs=1, current_epoch=1, dir_name=None): """ sampling the model and generate new object :param model: The model which is training. :param model_name: Name of model (base on hyperparameters config in deep_model.py file) e.g. [model_1, model_2, ...] :param epochs: Number of total epochs of training, e.g. 10,20,30,40,50 or 60 :param current_epoch: Number of current epoch :param dir_name: root directory for this running. :return: Nothing """ # End time of current epoch dt = datetime.datetime.now().strftime('_date_%Y-%m-%d_%H-%M-%S') dir_name = dir_name + 'epoch_' + str(current_epoch) + dt + '/' if not os.path.exists(dir_name): os.makedirs(dir_name) # Fuzzing hyper-parameters diversities = [i * 0.10 for i in range(1, 20, 2)] # diversities = [0.2, 0.5, 1.0, 1.2, 1.5, 1.8] # diversities = [0.5, 1.0, 1.5] # for sou and for mou diversities = [1.0] generated_obj_total = 1100 # [5, 10, 100, 1000, 3000] {1000-1100 for sou and 3000-3100 for muo} generated_obj_with_same_prefix = 1 # [1, 5, 10, 20, 40] {10 for sou and 20 for mou} generated_obj_max_allowed_len = 400 # Choose max allowed len for object randomly exclude_from_fuzzing_set = {'s', 't', 'r', 'e', 'a', 'm'} # set(['s', 't', 'r', 'e', 'a', 'm']) # Learn and fuzz paper hyper-parameters t_fuzz = 0.9 # For comparision with p_fuzz where p_fuzz is a random number (if p_fuzz > t_fuzz) p_t = 0.9 # 0.9 and more for format fuzzing; 0.5 and less than 0.5 for data fuzzing. Now format fuzzing. # End of fuzzing hyper-parameters testset_objects_list = preprocess.get_list_of_object(self.text_test) testset_object_gt_maxlen_list = [] for obj in testset_objects_list: if len(obj) > self.maxlen + len(' endobj'): testset_object_gt_maxlen_list.append(obj) print('len filtered test-set: ', len(testset_object_gt_maxlen_list)) generated_total = '' for diversity in diversities: generated_total = '' for q in range( round(generated_obj_total / generated_obj_with_same_prefix)): obj_index = random.randint( 0, len(testset_object_gt_maxlen_list) - 1) # obj_index = 0 generated_obj_counter = 0 generated_obj_len = 0 generated = '' stop_condition = False endobj_attach_manually = False # print() print('-- Diversity:', diversity) obj_prefix = str( testset_object_gt_maxlen_list[obj_index])[0:self.maxlen] generated += obj_prefix # prob_vals = '1 ' * self.maxlen # learnt_grammar = obj_prefix # print('--- Generating ts_text with seed:\n "' + obj_prefix + '"') # sys.stdout.write(generated) if generated.endswith('endobj'): generated_obj_counter += 1 if generated_obj_counter > generated_obj_with_same_prefix: stop_condition = True while not stop_condition: x_pred = np.zeros((1, self.maxlen, len(self.chars))) for t, char in enumerate(obj_prefix): x_pred[0, t, self.char_indices[char]] = 1. preds = model.predict(x_pred, verbose=0)[0] next_index, prob, preds2 = self.sample(preds, diversity) next_char = self.indices_char[next_index] next_char_for_prefix = next_char ###### Fuzzing section we don't need it yet! # if next_char not in exclude_from_fuzzing_set: # p_fuzz = random.random() # if p_fuzz > t_fuzz and preds2[next_index] > p_t: # next_index = np.argmin(preds2) # print('((Fuzz!))') # next_char = self.indices_char[next_index] ###### End of fuzzing section # print() # print(preds2) # print(np.argmax(preds)) # print(preds[np.argmax(preds)]) # print(prob) # print(np.argmax(prob)) # print('====>',next_index) # print(prob[0, next_index]) # prob_vals += str(preds2[next_index]) + '\n' # if preds2[next_index] > 0.9980: # learnt_grammar += next_char # else: # learnt_grammar += '.' # input() obj_prefix = obj_prefix[1:] + next_char_for_prefix generated += next_char_for_prefix # next_char generated_obj_len += 1 if generated.endswith('endobj'): generated_obj_counter += 1 generated_obj_len = 0 elif (generated.endswith('endobj') is False) and \ (generated_obj_len > generated_obj_max_allowed_len): # Attach '\nendobj\n' manually, and reset obj_prefix generated += '\nendobj\n' generated_obj_counter += 1 generated_obj_len = 0 endobj_attach_manually = True if generated_obj_counter >= generated_obj_with_same_prefix: # Fix: Change > to >= (13970315) stop_condition = True elif endobj_attach_manually: # Reset prefix: # Here we need to modify obj_prefix because we manually change the generated_obj! # Below we add this new repair: # obj_prefix = obj_prefix[len('\nendobj\n'):] + '\nendobj\n' # Instead of modify obj_prefix we can reset prefix if we found that 'endobj' dose not generate # automatically. It seems to be better option, so we do this: # obj_index = random.randint(0, len(testset_object_gt_maxlen_list) - 1) obj_index = 0 obj_prefix = str( testset_object_gt_maxlen_list[obj_index])[0:self. maxlen] generated += obj_prefix endobj_attach_manually = False # sys.stdout.write(next_char) # sys.stdout.flush() # print() generated_total += generated + '\n' # save generated_result to file inside program file_name = model_name \ + '_diversity_' + repr(diversity) \ + '_epochs_' + repr(epochs) \ + '_step_' + repr(self.step) \ + '.txt' preprocess.save_to_file(dir_name + file_name, generated_total) # preprocess.save_to_file(dir_name + file_name + 'probabilities.txt', prob_vals) # preprocess.save_to_file(dir_name + file_name + 'learntgrammar.txt',learnt_grammar) print('Diversity %s save to file successfully.' % diversity) print('End of generation method.') print('Starting new epoch ...') return generated_total
def train(): trainset_path = './trainset/pdfobjs.txt' trainset_path = './trainset/pdf_object_trainset_100_to_500_percent01.txt' text = poc.load_from_file(trainset_path) print('corpus length:', len(text)) chars = sorted(list(set(text))) print('Total chars:', len(chars)) # print(chars) # Vectorization print('Building dictionary index ...') char_indices = dict((c, i) for i, c in enumerate(chars)) # print(char_indices) indices_char = dict((i, c) for i, c in enumerate(chars)) # print(indices_char) # cut the text in semi-redundant sequences of maxlen characters maxlen = 50 # Good idea: use ave_object_len to determine this hyper-parameter step = 1 # should set to 1 for best result epochs = 10 # number of epochs for training sentences = [] # list of all sentence as input next_chars = [] # list of all next chars as labels for i in range(0, len(text) - maxlen, step): # arg2 why this? sentences.append(text[i:i + maxlen]) # print(sentences) next_chars.append(text[i + maxlen]) # print(next_chars) print('semi sequences:', len(sentences)) print('One-Hot vectorization...') x = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool) # input x y = np.zeros((len(sentences), len(chars)), dtype=np.bool) # output label y for i, sentence in enumerate(sentences): for t, char in enumerate(sentence): x[i, t, char_indices[char]] = 1 y[i, char_indices[next_chars[i]]] = 1 # build the model: a single LSTM layer # we need to deep it print('Build model...') model = Sequential() # model.add(LSTM(128, input_shape=(maxlen, len(chars)))) model.add( LSTM(128, input_shape=(maxlen, len(chars)), return_sequences=True)) # model.add(LSTM(128, input_shape=(maxlen, len(chars)), # activation='relu', return_sequences=True, dropout=0.2)) model.add(LSTM(128, input_shape=(maxlen, len(chars)))) # model.add(LSTM(128, activation='relu', dropout=0.2)) model.add(Dense(len(chars))) model.add(Activation('softmax')) optimizer = RMSprop(lr=0.01) model.compile(loss='categorical_crossentropy', optimizer=optimizer) input() # sys.exit() model.fit(x, y, batch_size=128, epochs=epochs, validation_split=0.2) # why epochs=? save(model, epochs) # del model # model = load_model('./modelh5/lstm_text_generation_pdf_objs_1_20180214_235713_epochs10.h5') """ sampling the model and generate new object """ diversities = [0.2, 0.5, 1.0, 1.2, 1.5, 1.8] # diversities = [0.1, 0.2, 0.3, 0.5, 0.7, 1, 1.2, 1.5, 1.7, 2] generated_obj_max_number = 5 generated_obj_max_allowed_len = 500 t_fuzz = 0.9 p_t = 0.9 # 0.9 for format fuzzing and 0.5 or letter for data fuzzing. Now format fuzzing list_of_objects = poc.get_list_of_object(text) list_of_objects_with_maxlen = [] for o in list_of_objects: if len(o) > maxlen: list_of_objects_with_maxlen.append(o) for diversity in diversities: obj_index = random.randint(0, len(list_of_objects_with_maxlen) - 1) generated_obj_counter = 0 generated_obj_len_index = 0 stop_condition = False print() print('-- Diversity:', diversity) # generated = '' obj_prefix = str(list_of_objects_with_maxlen[obj_index])[ 0:maxlen] # len(sentence) equals 100 here generated = obj_prefix prob_vals = '100\n' * maxlen learnt_grammar = obj_prefix print('--- Generating text with seed:\n "' + obj_prefix + '"') sys.stdout.write(generated) if generated.endswith('endobj'): generated_obj_counter += 1 if generated_obj_counter > generated_obj_max_number: stop_condition = True while not stop_condition: x_pred = np.zeros((1, maxlen, len(chars))) for t, char in enumerate(obj_prefix): x_pred[0, t, char_indices[char]] = 1. preds = model.predict(x_pred, verbose=0)[0] next_index, prob, preds2 = sample(preds, diversity) p_fuzz = random.random() if p_fuzz > t_fuzz and preds2[next_index] > p_t: next_index = np.argmin(preds2) print('FUZZ DONE!') next_char = indices_char[next_index] # print() # print(preds2) # print(np.argmax(preds)) # print(preds[np.argmax(preds)]) # print(prob) # print(np.argmax(prob)) # print('====>',next_index) # print(prob[0, next_index]) # prob_vals += str(preds2[next_index]) + '\n' # if preds2[next_index] > 0.9980: # learnt_grammar += next_char # else: # learnt_grammar += '.' # input() obj_prefix = obj_prefix[1:] + next_char generated += next_char generated_obj_len_index += 1 if generated.endswith('endobj'): generated_obj_counter += 1 generated_obj_len_index = 0 elif generated_obj_len_index > generated_obj_max_allowed_len: generated += '\nendobj\n' generated_obj_counter += 1 generated_obj_len_index = 0 if generated_obj_counter > generated_obj_max_number: stop_condition = True sys.stdout.write(next_char) sys.stdout.flush() # save generated text to file inside program dt = datetime.datetime.now().strftime('_%Y%m%d_%H%M%S_') dir_name = './generated_results/pdfobjs_new/' file_name = 'gen_objs' + dt + 'epochs' + repr(epochs) + '_div' \ + repr(diversity) + '_step' + repr(step) + '.txt' poc.save_to_file(dir_name + file_name, generated)