def process_files(volume_number, production_prefix, start_bates_number, num_digits, confidentiality, files_to_convert, dirs, files): # get dirs and files prod_home, prod_data, prod_img, prod_nat, prod_txt, prod_img001, prod_nat001, prod_txt001, completed_dir = dirs opt_file, dat_file = files # integerize digit number num_digits = int(num_digits) # initialize bates number current_bates_number = int(start_bates_number) # print progress bar l = len(files_to_convert) progress_bar.printProgressBar(0, l, prefix='Progress:', suffix='Complete', length=50) # iterate over directory of input files for i, file in enumerate(files_to_convert): current_bates_number = process_file(file, volume_number, production_prefix, start_bates_number, num_digits, confidentiality, current_bates_number, dirs, files) progress_bar.printProgressBar(i + 1, l, prefix='Progress:', suffix='Complete', length=50)
def get_counts_in_gene_bodies(regions_filename, gene_bodies_output_filename): with open(regions_filename, 'r') as file: with open(gene_bodies_output_filename, 'w') as output_file: tsv_writer = csv.writer(output_file, delimiter='\t', lineterminator='\n') num_of_lines = sum(1 for _ in file) file.seek(0) print("Getting gene body counts.") printProgressBar(0, num_of_lines, prefix='Progress:', suffix='Complete', length=50) for curr_line_num, line in enumerate(file): chromosome, left, right, gene_name, _, strand = line.split() curr_total = 0 # Loop through each base in the region for i in range(int(left), int(right) + 1): # Get the height and add it to the curr_total if chromosome + '_' + str( i) + '_' + strand in three_prime_counts_dict: curr_total += three_prime_counts_dict[chromosome + '_' + str(i) + '_' + strand] tsv_writer.writerow([gene_name, curr_total]) printProgressBar(curr_line_num + 1, num_of_lines, prefix='Progress:', suffix='Complete', length=50)
def blacklist_regions(blacklisting_file): with open(blacklisting_file, 'r') as file: num_of_lines = sum(1 for _ in file) file.seek(0) print("Blacklisting.") printProgressBar(0, num_of_lines, prefix='Progress:', suffix='Complete', length=50) for curr_line_num, line in enumerate(file): chromosome, left, right, name, _, strand = line.split() # Loop through each base in the blacklist region and change the number of counts in the 3' dictionary to zero for i in range(int(left), int(right)): if chromosome + '_' + str( i) + '_' + strand in three_prime_counts_dict: # If that base has a 3' end mapped already, add one to the value three_prime_counts_dict[chromosome + '_' + str(i) + '_' + strand] = 0 if curr_line_num % 10_000 == 0 or curr_line_num == num_of_lines - 1: printProgressBar(curr_line_num + 1, num_of_lines, prefix='Progress:', suffix='Complete', length=50)
def make_5_prime_counts_dict(sequencing_filename): with open(sequencing_filename, 'r') as file: num_of_lines = sum(1 for _ in file) file.seek(0) print("Building the 5' counts dict.") printProgressBar(0, num_of_lines, prefix='Progress:', suffix='Complete', length=50) for i, line in enumerate(file): chromosome, left, right, _, _, strand = line.split() if strand == "+": five_prime_position = left else: five_prime_position = right if chromosome + '_' + five_prime_position + '_' + strand in five_prime_counts_dict: # If that base has a 5' end mapped already, add one to the value five_prime_counts_dict[chromosome + '_' + five_prime_position + '_' + strand] += 1 else: # If that base does not have a mapped 5' read yet, set the value to 1 five_prime_counts_dict[chromosome + '_' + five_prime_position + '_' + strand] = 1 if i % 100_000 == 0 or i == num_of_lines - 1: printProgressBar(i + 1, num_of_lines, prefix='Progress:', suffix='Complete', length=50)
def video_processing(filename, output_video=None): cap = cv2.VideoCapture(filename) #input video file object if(output_video): width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) # original file width height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # original file height fps = int(cap.get(cv2.CAP_PROP_FPS)) # original file FPS fourcc = cv2.VideoWriter_fourcc(*'DIVX') # video codec for processed video with points out = cv2.VideoWriter(output_video,fourcc,fps,(width,height)) # outpur video object coordinate_from_all_frames={} #dictionary where for each frame we will save coordinate of bacteria l = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) # total number of frames in video print("Frames in video: %d"%(l)) printProgressBar(0,l,prefix="Progress:",suffix="Complete",length=50) # begin of beautiful progress bar :3 for frame_number in range(0,l): retval,frame = cap.read() # reading one frame gray_frame = frame[:,:,0] # converting RGB -> Grayspace segmentation_map = recognize(gray_frame) # inference prediction for frame coordinates_from_one_frame=get_centre_of_shapes(segmentation_map) # coordinates of bacteria for frame if(output_video): # if you want to save video with marked bacterias for point in coordinates_from_one_frame: cv2.circle(frame, (point[0],point[1]), 3, (255, 0, 0), -1) # draw blue point on frame frame[:,:,1]=frame[:,:,1]+segmentation_map*200 # green out.write(frame) coordinate_from_all_frames[frame_number]=coordinates_from_one_frame printProgressBar(frame_number+1,l,prefix="Progress:",suffix="Complete",length=50) # update of beautiful progress bar :3 cap.release() if(output_video): out.release() return coordinate_from_all_frames
def main(books, path_download): """Recebendo todas os livros em um lista de dicionario contendo as informações do livro""" path_url = '/content/pdf/' url = urljoin(BASE_URL, path_url) l = len(books) print("\nExecutando Downloads dos Livros") printProgressBar(0, l, prefix="Progress", suffix="Complete", length=50) for i, book in enumerate(books): title = book['title'] path_book = book['url_book'] path_book = path_book[6:] data = quote_plus(path_book) + '.pdf' url_full = urljoin(url, data) file_name = title + '.pdf' file_name = file_name.replace('/', '-') if os.path.exists(os.path.join(path_download, file_name)): continue urlretrieve(url_full, os.path.join(path_download, file_name)) printProgressBar(i + 1, l, prefix='Progress:', suffix='Complete', length=50)
def map_tsrs(tsr_filename): with open(tsr_filename, 'r') as file: num_of_lines = sum(1 for _ in file) file.seek(0) print("Mapping TSRs.") printProgressBar(0, num_of_lines, prefix='Progress:', suffix='Complete', length=50) # Loop through the TSR file for i, line in enumerate(file): tsr_chromosome, tsr_left, tsr_right, _, tsr_counts, tsr_strand = line.split( ) # Set the correct chromosome to true. It is false when the chromosome in the TSR file is not present in the annotation file correct_chromosome = True try: regions_dict[tsr_chromosome] except KeyError: # This means the TSR file has a chromosome that is not present in the annotation file correct_chromosome = False if correct_chromosome: for region in regions_dict[tsr_chromosome]: chromosome, left, right, strand, gene_name = region # If there is a TSR as that base pair, add it to the gene_tsr dict if tsr_strand == strand and not ( int(tsr_right) < int(left) or int(tsr_left) > int(right)): if gene_name not in gene_tsr_dict: gene_tsr_dict[gene_name] = [line.split()] else: gene_tsr_dict[gene_name].append(line.split()) # If the left of the region is past the right side of the TSR, we don't need to search the rest of the file if int(left) > int(tsr_right): break if i % 300 == 0 or i == num_of_lines - 1: printProgressBar(i + 1, num_of_lines, prefix='Progress:', suffix='Complete', length=50)
def predict(self, X, conv_s=1, pool_f=2, pool_s=2, read_from_file=False): ret = [] for i, image in enumerate(X): printProgressBar(self, iteration=i + 1, total=X.shape[0], prefix='Progress:', suffix=' | Example #: ') if read_from_file: #print(pickle.load(open(self.save_path, 'rb'))) self.params = pickle.load(open(self.save_path, 'rb')) [f1, f2, w3, w4, b1, b2, b3, b4] = self.params #image-= np.mean(image) #image/= np.std(image) image = image.reshape(1, 28, 28) ################################################ ############## Forward Operation ############### ################################################ conv1 = convolution(image, f1, b1, conv_s) # convolution operation conv1[conv1 <= 0] = 0 # pass through ReLU non-linearity conv2 = convolution(conv1, f2, b2, conv_s) # second convolution operation conv2[conv2 <= 0] = 0 # pass through ReLU non-linearity pooled = maxpool(conv2, pool_f, pool_s) # maxpooling operation (nf2, dim2, _) = pooled.shape fc = pooled.reshape((nf2 * dim2 * dim2, 1)) # flatten pooled layer z = w3.dot(fc) + b3 # first dense layer z[z <= 0] = 0 # pass through ReLU non-linearity out = w4.dot(z) + b4 # second dense layer probs = softmax( out) # predict class probabilities with the softmax activation #print(probs) prediction = np.argmax(probs) ret.append(prediction) return ret
def fit(self, X, y, maxEpochs=500): n, d = X.shape if y.ndim == 1: y = y[:, None] self.layer_sizes = [X.shape[1] ] + self.hidden_layer_sizes + [y.shape[1]] self.classification = y.shape[ 1] > 1 # assume it's classification iff y has more than 1 column # random init scale = 0.01 weights = list() for i in range(len(self.layer_sizes) - 1): W = scale * np.random.randn(self.layer_sizes[i + 1], self.layer_sizes[i]) b = scale * np.random.randn(1, self.layer_sizes[i + 1]) weights.append((W, b)) weights_flat = flatten_weights(weights) for e in range(maxEpochs - 1): printProgressBar(self, iteration=e + 1, total=maxEpochs, prefix='Progress:', suffix=' | Epoch #: ') random_indices = np.random.randint(n, size=2000) #Take a random sample of corresponding X and y X_rand = X[random_indices] y_rand = y[random_indices] weights_flat_new, f = findMin(self.funObj, weights_flat, self.max_iter, X_rand, y_rand, verbose=True, alpha=0.0001) weights_flat = weights_flat_new self.weights = unflatten_weights(weights_flat_new, self.layer_sizes)
def train(data, cnp, epochs, num_test_maximum, alpha, optimizer): cnp.train() for epoch in range(epochs): total_loss = 0 iteration = 0 length = len(data) printProgressBar(0, length, prefix='Epoch {} Progress:'.format(epoch + 1), suffix='Complete', length=100) for i, function in enumerate(data): optimizer.zero_grad() num_points = function[0].size()[0] perm = torch.randperm(num_points) num_context = np.random.randint(num_points - num_test_maximum, num_points) context_x = function[0][perm][0:num_context] context_y = function[1][perm][0:num_context] test_x = function[0][perm][num_context:num_points] test_y = function[1][perm][num_context:num_points] mu, sigma, log_p, en_dist, t_en_dist, MSE = cnp( context_x, context_y, test_x, test_y) loss = lossf( log_p, torch.distributions.Normal(torch.zeros(cnp.encoded_size), 1), en_dist, t_en_dist, alpha, MSE) loss.backward() optimizer.step() total_loss += loss / len(data) iteration += 1 printProgressBar( i + 1, length, prefix='Epoch {} Progress:'.format(epoch + 1), suffix='Complete. Iteration = {}. Average loss = {}.'.format( iteration, total_loss), length=50) print('EPOCH LOSS {}'.format(total_loss))
def predict(self, Xtest): #empty vector of appropriate size to store predictions y_pred = np.empty(len(Xtest)) #iterate over Xtest, generating prediction for each for i in range(len(Xtest)): printProgressBar(self, iteration=i + 1, total=len(Xtest), prefix='Progress:', suffix=' | Example #: ') #array to store tuples of training examples' distance and class sim_class = [] #iterate over each training example for j in range(len(self.X)): #calculate distance to Xtest example in question #dist = np.linalg.norm(Xtest[i] - self.X[j]) #add a new distance, label entry to dist_class sim = cosineSim(Xtest[i], self.X[j]) sim_class.append([sim, self.y[j]]) #print(sim_class) #key method to sort sim_class by similarity def take_first(arr): return arr[0] #sort sim_class by similarity sim_class.sort(key=take_first) #take the k closest examples k_nearest = sim_class[-self.k:] print(k_nearest) #extract the labels k_nearest_labels = [x[1] for x in k_nearest] #find the mode of those labels mode_label = stats.mode(k_nearest_labels) #enter the calculated mode as the predicted y value y_pred[i] = mode_label[0] return y_pred
def fit(self, X, Y, maxEpochs=15000): # Add a bias to the X data matrix Z = np.concatenate((np.ones((X.shape[0], 1)), X), axis=1) n, d = Z.shape #set learning rate: lr = .001 #Control the degree of regularization: C = 0.4 self.W = np.zeros((10, d)) for e in range(maxEpochs): printProgressBar(self, iteration=e + 1, total=maxEpochs, prefix='Progress:', suffix=' | Epoch #: ') #Choose a sandom set of 50 indices random_indices = np.random.randint(n, size=50) #Take a random sample of corresponding X and y X_rand = Z[random_indices] Y_rand = Y[random_indices] # Loop through all tuples, (x,y), from random selection for (x, y) in zip(X_rand, Y_rand): #loop through y (10 values, one for each digit) for i in range(len(y)): #set yi to -1 if y[i] is 0, 1 otherwise (ie. if the current digit --i-- is not represented by x) if y[i] == 0: yi = -1 else: yi = 1 #Apply the appropriate gradient if there is error (direction determined by yi) if yi * np.dot(x, self.W[i].T) <= 1: self.W[i] = np.add(self.W[i], lr * C * yi * x) return self.W
def iphostname(ip_list): # pull hostnames for list of IPs r = 1 for i, ip in enumerate(ip_list): try: nmScan = nmap.PortScanner() results = nmScan.scan(str(ip), arguments='-A -T5') hostname = results['scan'][str(ip)]['hostnames'][0]['name'] OS = results['scan'][str(ip)]['osmatch'][0]['name'] if OS == None: ips_hostnames.add_row([ip, hostname, '']) else: ips_hostnames.add_row([ip, hostname, OS]) a('clear') except Exception, e: print e continue progress_bar.printProgressBar( i + 1, len(ip_list), prefix='Scanning %d of %d...\nProgress:' % (r + 1, len(ip_list)), suffix='Complete', length=50)
def train_on_memory(self, batch_size, epochs): """ Trains the agent on experiences from its experience replay memory. :param batch_size: Batch size for training :param epochs: Number of times the mem should be fully browsed """ print("Training on ", epochs, " epochs from the replay memory..") # Get all data from the replay memory states, actions, next_states, rewards = self.mem.all() # Shuffling the batches lines_shuffle = torch.randperm(states.size()[0]) states = states[lines_shuffle] actions = actions[lines_shuffle] rewards = rewards[lines_shuffle] next_states = next_states[lines_shuffle] # Split them into batches states_batches = torch.split(states, batch_size) actions_batches = torch.split(actions, batch_size) next_states_batches = torch.split(next_states, batch_size) rewards_batches = torch.split(rewards, batch_size) # Number of batches nb_batches = len(states_batches) # Train for ep in range(epochs): batches_completed = 0 for states, actions, next_states, rewards \ in zip(states_batches, actions_batches, next_states_batches, rewards_batches): self.train_on_batch(states, actions, next_states, rewards) batches_completed += 1 printProgressBar(batches_completed, nb_batches, "Epoch " + str(ep + 1) + "/" + str(epochs), length=90)
def synthesize_images(set_name, op_list): """Synthesize data from original images""" op_todo = [([op_list[0]]), ([op_list[1]]), ([op_list[2]]), ([op_list[0], op_list[2]]), ([op_list[1], op_list[2]])] print op_todo # for ind in range(len(op_list)): # for item in itertools.combinations(op_list, ind+1): # op_todo.append(item) # img_path = "data_sets/%s/data/" % (set_name) img_path = [ "data_sets/%s/left/" % (set_name), "data_sets/%s/right/" % (set_name) ] csv_file = "model_data/%s_log.csv" % (set_name) with open(csv_file, 'r') as in_csv: for line in in_csv: if re.search(r"(flip|autocont|equalize|darken|brighten)", line): printProgressBar(1, 1) return print "Processing images..." with open(csv_file, 'a+') as io_csv: io_csv.seek(0) reader = csv.reader(io_csv, delimiter=',') attribute = next(reader, None) entries = list(reader) cnt_total = len(entries) cnt_iter = 0 printProgressBar(cnt_iter, cnt_total) for entry in entries: cnt_iter += 1 printProgressBar(cnt_iter, cnt_total) # try: new_entries = process_image(img_path, [entry[0], entry[1]], int(entry[-1]), op_todo) writer = csv.writer(io_csv, delimiter=',') for new_entry in new_entries: writer.writerow(new_entry) # except: # print "CSV entry error" time.sleep(0.1)
all_df = pd.merge(df, picks_df) season_totals = all_df.groupby(['season', 'player'])['points'].sum() return season_totals.groupby('season').std().mean() first = list(range(1, CONFIG.PLAYERS + 1)) # first round goes in order rest = (x + 1 for y in range(CONFIG.PICKS - 1) for x in range(CONFIG.PLAYERS)) perms = itertools.permutations(rest) best_combo = None best_result = 10000 total_runs = math.factorial(CONFIG.PLAYERS * (CONFIG.PICKS - 1)) print("There will be {0} total runs".format(total_runs)) run_count = 1 for rest in perms: picks = first + list(rest) result = calculate_std(picks) printProgressBar(run_count, total_runs) run_count += 1 if result < best_result: best_combo = picks best_result = result print("Best result was: {0}".format(best_result)) print("Best combo was: {0}".format(best_combo))
import time from get_dir_files import run_fast_scandir from essentia_bpm import get_file_bpm from progress_bar import printProgressBar directory = r"/Volumes/Carl'S/Music" subf, files = run_fast_scandir(directory, [".flac", ".mp3"]) files_length = len(files) files_count = 0 print("Processing %d files" % files_length) time1 = time.time() printProgressBar(0, files_length, prefix='Progress:', suffix='Complete', length=50) for file in files: files_count = files_count + 1 try: song = mutagen.File(file, easy=True) if (not song.get("bpm", False)): song["bpm"] = str(int(get_file_bpm(file))) song.save() print("Processed:" + file, file=open("output.txt", "a")) except: print("Failed:" + file, file=open("output.txt", "a")) printProgressBar(files_count, files_length, prefix='Progress:',
def build_dict(filename): """ pass this method training data and it will create dictionaries, one for the ammount of each negative token, one for the amount of each positve token and on probability dictionary which states the probability of each token beeing positive( positive = over 0.5) Posiive means that the ticket/token has been solved on the first level of support in this cenario Parameters: filename - Requiered : name of the csv-file with the training data (string) """ dictionary = {} regex_noword = re.compile(r"\W", re.IGNORECASE) #everything that is not a word regex_digit = re.compile(r"\d", re.IGNORECASE) #all numbers #start processing training data csv_file = open(filename, 'r', encoding='utf-8', errors='ignore') ticketreader = csv.reader(csv_file, delimiter=',', quotechar='|') tickets = list(ticketreader) #progress l = len(tickets) i = 0 progress_bar.printProgressBar(0, l, prefix='Progress:', suffix='Complete', length=50) for i, ticket in enumerate(tickets): progress_bar.printProgressBar(iteration=i + 1, total=l, prefix='Progress:', suffix='Complete', length=50) more_tokens = [] # deleting all the unwanted characters line = str(ticket) line = line.replace( '\\ufeff', ' ' ) #this is a so called byte order mark (BOM, not relevant anymore) line = line.replace(';', ' ') line = line.replace(',', ' ') line = line.replace('_', ' ') line = line.replace('\\n', ' ') line = line.replace('\\xa0', ' ') line = line.replace('\\t', ' ') line = line.replace('\\r', ' ') line = line.replace('\\', ' ') line = re.sub(regex_noword, ' ', line) line = re.sub(regex_digit, ' ', line) tokens = line.split() tokens = list(filter(None, tokens)) # check for chinese/japanese tokens tokens = cj.check_for_cj_chars(tokens) for token in tokens: token = token.lower() if token in dictionary: dictionary[token] = dictionary[token] + 1 else: dictionary[token] = 1 csv_file.close() return dictionary
def merge_files(path_to_folder_with_files, merged_file_full_name): merged_file = open(merged_file_full_name, 'w', encoding='utf-8', newline="") writer = csv.writer(merged_file, delimiter='\t') writer.writerow([ "ANO_BO", "NUM_BO", "BO_INICIADO", "BO_EMITIDO", "DATAOCORRENCIA", "PERIDOOCORRENCIA", "FLAGRANTE", "LATITUDE", "LONGITUDE", "DESCRICAOLOCAL", "DELEGACIA_NOME", "DELEGACIA_CIRCUNSCRICAO", "RUBRICA", "VITIMAFATAL", "SEXO", "IDADE", "CORCUTIS", "NATUREZAVINCULADA" ]) files_to_be_merged = [ join(path_to_folder_with_files, f) for f in listdir(path_to_folder_with_files) if isfile(join(path_to_folder_with_files, f)) and f.split('.')[-1] == 'csv' ] number_of_files_to_be_merged_str = str(len(files_to_be_merged)) print("Begining to merge " + number_of_files_to_be_merged_str + " files") total_row_count = 0 current_file_index = 1 for file_name in files_to_be_merged: print("File " + str(current_file_index) + '/' + number_of_files_to_be_merged_str + ':') print(file_name) print("Counting rows...") source_file = open(file_name, 'r', encoding='ansi') reader = csv.reader(source_file, delimiter='\t') next(reader) row_count = sum(1 for row in reader) print(str(row_count) + " rows") total_row_count += row_count source_file.close() current_file_index += 1 current_file_index = 1 current_row_index = 0 for file_name in files_to_be_merged: print("File " + str(current_file_index) + '/' + number_of_files_to_be_merged_str + ':') print(file_name) source_file = open(file_name, 'r', encoding='utf-8') reader = csv.reader(source_file, delimiter='\t') next(reader) for row in reader: writer.writerow(row) current_row_index += 1 if (current_row_index % 100 == 0 or current_row_index == total_row_count): printProgressBar(current_row_index, total_row_count, suffix='Completed', decimals=2) print() source_file.close() current_file_index += 1 merged_file.close()
def trainABit(execution_plan, previous_losses = []): with tf.Session() as sess: # Create model. print("Creating %d layers of %d units." % (FLAGS.num_layers, FLAGS.size)) model = create_model(sess, False) dev_set, train_set = execution_plan.getData(model.global_step.eval()) train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_buckets))] print("Train bucket sizes", train_bucket_sizes) train_total_size = float(sum(train_bucket_sizes)) # A bucket scale is a list of increasing numbers from 0 to 1 that we'll use # to select a bucket. Length of [scale[i], scale[i+1]] is proportional to # the size if i-th training bucket, as used later. train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in xrange(len(train_bucket_sizes))] # This is the training loop. step_time, loss = 0.0, 0.0 current_step = 0 while current_step <= 10 * FLAGS.steps_per_checkpoint: progress_bar.printProgressBar(current_step, 10 * FLAGS.steps_per_checkpoint) # Choose a bucket according to data distribution. We pick a random number # in [0, 1] and use the corresponding interval in train_buckets_scale. random_number_01 = np.random.random_sample() bucket_id = min([i for i in xrange(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01]) # Get a batch and make a step. start_time = time.time() encoder_inputs, decoder_inputs, target_weights = model.get_batch( train_set, bucket_id) _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, False) step_time += (time.time() - start_time) / FLAGS.steps_per_checkpoint loss += step_loss / FLAGS.steps_per_checkpoint current_step += 1 # Once in a while, we save checkpoint, print statistics, and run evals. if current_step % FLAGS.steps_per_checkpoint == 0: # Print statistics for the previous epoch. perplexity = math.exp(float(loss)) if loss < 300 else float("inf") print("global step %d learning rate %.4f step-time %.2f perplexity " "%.2f loss %.2f " % (model.global_step.eval(), model.learning_rate.eval(), step_time, perplexity, loss)) # Decrease learning rate if no improvement was seen over last 3 times. if len(previous_losses) > 2 and loss > max(previous_losses[-3:]): sess.run(model.learning_rate_decay_op) previous_losses.append(loss) # Save checkpoint and zero timer and loss. checkpoint_path = os.path.join(FLAGS.train_dir, "translate.ckpt") if current_step % (5 * FLAGS.steps_per_checkpoint) == 0: model.saver.save(sess, checkpoint_path, global_step=model.global_step) step_time, loss = 0.0, 0.0 # Run evals on development set and print their perplexity. for bucket_id in xrange(len(_buckets)): if len(dev_set[bucket_id]) == 0: print(" eval: empty bucket %d" % (bucket_id)) continue encoder_inputs, decoder_inputs, target_weights = model.get_batch(dev_set, bucket_id) _, eval_loss, outputs = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, True) eval_ppx = math.exp(float(eval_loss)) if eval_loss < 300 else float( "inf") print(" eval: bucket %d perplexity %.2f" % (bucket_id, eval_ppx)) sys.stdout.flush()
essay_score_graduated, interview_score_dont_graduated, essay_score_dont_graduated, titles, fig_title, xlabel, ylabel, savename) titles = [ 'Results of students which graduated', "Results of students which didn't graduated", '', "" ] fig_title = "Exams and scores result in " + str(years[i]) xlabel = ["GPA", "GPA", "LANGUAGE EXAM", "LANGUAGE EXAM"] ylabel = [ "TOTAL SCORE", "TOTAL SCORE", "SOCIAL ACTIVITY", "SOCIAL ACTIVITY" ] savename = "graduated_score_graph_" + str(years[i]) + "_part2" graduated_scores_graph.draw_graph(gpa_graduated, score_graduated, gpa_dont_graduated, score_dont_graduated, language_exam_graduated, social_activity_graduated, language_exam_dont_graduated, social_activity_dont_graduated, titles, fig_title, xlabel, ylabel, savename) printProgressBar(progress_bar_count, len(years), prefix='Getting data:', suffix='Complete', length=50) progress_bar_count = progress_bar_count + 1 # database disconnect database_disconnect.database_disconnect(db_connection, db_connection.cursor())
try: nmScan = nmap.PortScanner() nmScan.scan(str(ip), arguments='-n -sP -PE -T4') state = nmScan[str(ip)].state() if state == 'up': ip_list.append(str(ip)) else: pass except Exception, e: continue print "%d Hosts are up." % len(ip_list) progress_bar.printProgressBar(0, len(ip_list), prefix='Scanning %d of %d...\nProgress:' % (1, len(ip_list)), suffix='Complete', length=50) # initialize progress bar iphostname(ip_list) def iphostname(ip_list): # pull hostnames for list of IPs r = 1 for i, ip in enumerate(ip_list): try: nmScan = nmap.PortScanner() results = nmScan.scan(str(ip), arguments='-A -T5') hostname = results['scan'][str(ip)]['hostnames'][0]['name'] OS = results['scan'][str(ip)]['osmatch'][0]['name'] if OS == None:
if print_progress_bar: # Initial call to print 0% progress if progress_bar_varable < 6: print_data_name = gamepad_axis_dict[progress_bar_varable] elif progress_bar_varable > 5: print_data_name = gamepad_button_dict[progress_bar_varable - 6] elif progress_bar_varable > 17: print( "ERROR: Attempting to access value outside of preassigned range." ) print("ERROR: Exiting...") #return printProgressBar(0, max_time, prefix=print_data_name, suffix='', length=50) start = time.time() while not done: pygame.event.get() ##for event in pygame.event.get(): # User did something. ## if event.type == pygame.JOYBUTTONDOWN: ## #print("Joystick button pressed.") ## #print("") ##elif event.type == pygame.JOYBUTTONUP: #print("Joystick button released.") #print("")
def main(): x = [] y = [] #alegere poza if len(sys.argv) != 2: print( "Wrong number of arguments taken from console! (expected 1 argument but recieved:", len(sys.argv) - 1, ")") sys.exit(1) else: photo_name = sys.argv[1] #identificare extensie poza extension = photo_name.split('.')[-1] if extension != 'png' and extension != 'jpg': print("File must be .png or .jpg. Choose another file:") photo_name = input() extension = photo_name.split('.')[-1] #citire si afisare imagine originala OrgImage = Image.open(photo_name) OrgImage.show() image = cv2.imread(photo_name) img = rgb2gray(image) print("Blur level ( in range (1, 100) ):") nivel_blur = float(input()) while nivel_blur < 1: print("Blur must be over 1 :") nivel_blur = float(input()) #training if nivel_blur < 5: printProgressBar(0, 20, prefix='Progress:', suffix='Complete', length=50) for i in range(20): x.append(i) boxImage = OrgImage.filter(ImageFilter.BoxBlur(i)) boxImage.save(".\\auxiliar." + extension) blur = cv2.imread("auxiliar." + extension) bl = rgb2gray(blur) y.append(get_contrast(bl)) printProgressBar(i + 1, 20, prefix='Progress:', suffix='Complete', length=50) else: printProgressBar(0, math.ceil((nivel_blur + 10) / 2), prefix='Progress:', suffix='Complete', length=50) for i in range(math.ceil((nivel_blur + 10) / 2)): x.append(2 * i) boxImage = OrgImage.filter(ImageFilter.BoxBlur(2 * i)) boxImage.save(".\\auxiliar." + extension) blur = cv2.imread("auxiliar." + extension) bl = rgb2gray(blur) y.append(get_contrast(bl)) printProgressBar(i + 1, math.ceil((nivel_blur + 10) / 2), prefix='Progress:', suffix='Complete', length=50) # determinarea polinomului care aproximeaza functia coef = polyfit(x, y) y_contrast = [] x_contrast = [] for i in range(math.ceil((nivel_blur + 10) / 2)): x_contrast.append(2 * i) y_contrast.append(evalpoly(coef, 2 * i)) # aflare minim for i in range(len(coef)): coef[i] = (-1) * coef[i] x_min, y_min, val_min = gold(0, 20, coef, 2) if nivel_blur > 20: k = math.ceil((nivel_blur / 20) - 1) for i in range(1, k): x_aux, y_aux, val_aux = gold(20 * i, 20 * (i + 1), coef, 1) if x_min < x_aux: val_min = [val_aux[0]] + val_min else: val_min = [val_min[0]] + val_aux if y_min > y_aux: x_min = x_aux y_min = y_aux val_min = val_aux y_min = (-1) * y_min for i in range(len(coef)): coef[i] = (-1) * coef[i] # afisarea rezultatelor plt.figure() plt.subplot(121) plt.plot(x_contrast, y_contrast) plt.plot(nivel_blur, evalpoly(coef, nivel_blur), 'r*') plt.grid(True) plt.title('Grafic') boxImage = OrgImage.filter(ImageFilter.BoxBlur(nivel_blur)) boxImage.save(".\\auxiliar." + extension) blur = cv2.imread("auxiliar." + extension) plt.subplot(122) plt.imshow(cv2.cvtColor(blur, cv2.COLOR_BGR2RGB)) plt.title('Imagine') plt.xticks([]), plt.yticks([]) plt.savefig('results.png') print('Open result.png and press enter to continue') input() for i in val_min: plt.figure() plt.subplot(121) plt.plot(x_contrast, y_contrast) plt.plot(i, evalpoly(coef, i), 'r*') plt.grid(True) plt.title('Grafic') boxImage = OrgImage.filter(ImageFilter.BoxBlur(i)) boxImage.save(".\\auxiliar." + extension) blur = cv2.imread("auxiliar." + extension) plt.subplot(122) plt.imshow(cv2.cvtColor(blur, cv2.COLOR_BGR2RGB)) plt.title('Imagine') plt.xticks([]), plt.yticks([]) plt.savefig('results.png') input()
else: print('Parameters: N_dim N_data N_times') sys.exit() def alpha(theta, phi): return np.sin(theta) * np.exp(1j * phi) / 2 # %% zero_state = np.matrix([1, 0], dtype=np.complex128) times = np.linspace(1e-7, 10, N_times) # %% E = np.zeros((N_times, N_data)) theta_d = np.random.uniform(0, np.pi, (N_data, N_dim)) phi_d = np.random.uniform(0, 2 * np.pi, (N_data, N_dim)) # %% for i, dt in enumerate(times): for p in range(N_data): alp = alpha(theta_d[p, 0], phi_d[p, 0]) zero_state = np.matrix([np.conj(alp)/np.abs(alp), 1]) / np.sqrt(2) rho_0 = zero_state.H @ zero_state res = minimize(wrapper, np.full(2 * N_dim, 1e-3 * np.pi), args=(theta_d[p], phi_d[p], dt, rho_0, N_dim)) E[i, p] = res.fun if p == 0: print(np.linalg.norm(zero_state)) printProgressBar(p, N_data, suffix='Time {0}'.format(dt)) np.save('eigen_work_N_{0}_norm'.format(N_dim), E)
def test(agent: QNetwork, movements=100, nb_episodes=1000, step=0.01, show_plots=True): """ Tests the ability of the QNetwork to learn to reach the position (0.5, 0.5) while spawning at random coordinates in [0, 1]^2. :param agent: QNetwork to be tested. Needs to have state_dim == 2 and 5 possible actions. :param movements: Number of moves the agent is allowed to have :param step: Distance travelled at each move :param nb_episodes: Number of episodes on which the agent trains :param show_plots: if True, the agent will plot the results of the training :return: The agent's loss memory """ # A state is defined as its x and y coordinates state_dim = 2 # Calculation device device = torch.device("cpu") # net = Net1(state_dim, nb_actions) # QNetwork(net, state_dim, movements, lr=0.1, device=torch.device("cpu")) for ep in range(nb_episodes): # Play a single episode # Create arrays to store the successive states and taken actions states = torch.empty( (movements + 1, state_dim), device=device) # + 1 to make space for the last state actions = torch.empty(movements, dtype=torch.int32, device=device) # Start with a random position states[0] = torch.rand(2) for move in range(movements): # Take action actions[move] = agent.decide(states[move].view(1, -1)).item() # Get next state states[move + 1] = next_state(states[move], actions[move], step, device) # Get rewards rewards = get_rewards(states[:-1], actions, step, device) # Memorize the episode agent.memorize_exploration(states, actions, rewards, last_state_is_final=False) # Train after the episode agent.update() printProgressBar(ep + 1, nb_episodes, "Episodes completed: ", length=90) # print("Final position: ", states[-1], " | Initial: ", states[0]) if show_plots: plt.figure("Training summary") plt.subplot(111) plt.title("Agent Trajectories") agent.plot_trajectory(torch.rand((50, 2)), lambda s, a: next_state(s, a, step, device)) # plt.subplot(212) # plt.title("MSE Loss") # agent.show_training() plt.show() return agent.loss_mem return 0
def get_books_describe(base_url): """Faz a busca de todos os livros e retorna uma lista com as descrições de cada um""" headers = { 'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3' } BASE_URL = urllib.parse.urljoin(base_url, 'search/page/') #Quantidade de paginas para ser percorrido qtd_pages = 20 #lista de paginas para ser adicionado na url num_pages = [str(num) for num in range(1,qtd_pages+1)] values = { 'facet-language' : 'En', 'facet-content-type' : 'Book', 'package' : 'mat-covid19_textbooks', 'sortOrder' : 'newestFirst', 'showAll' : 'false' } data = urllib.parse.urlencode(values) #print(data) url_list = [] for i in range(0, qtd_pages): url_with_num = BASE_URL+num_pages[i]+'?' url = url_with_num+ data #print(url) url_list.append(url) books_list = [] l = len(url_list) print("Fazendo Busca de todos os livros") #Barra de progresso do prompt printProgressBar(0, l, prefix="Progress", suffix="Complete", length=50) for i, url in enumerate(url_list): req = Request(url=url, headers=headers) try: uclient = uReq(req) page_html = uclient.read() uclient.close() page_soup = soup(page_html, "html.parser") #print(page_soup) #Busca a lista de livros da pagina content_list = page_soup.find("ol", class_="content-item-list") books = content_list.findAll('li') for book in books: divs = book.findAll("div") title = divs[1].find("h2").text.strip() url_book = divs[1].find("a").get('href') p = divs[1].findAll("p") subtitle = p[0].text.strip() span = p[1].findAll("span") autor = span[0].text.strip() year = span[1].text.strip() #print(title) book_data = { 'title': title, 'url_book' : url_book, 'subtitle' : subtitle, 'autor' : autor, 'year' : year } books_list.append(book_data) except HTTPError as e: print(e.reason, " ", e.code) printProgressBar(i + 1, l, prefix='Progress:', suffix='Complete', length=50) return books_list
if '&' in meta: # for extra commands tacked on meta = meta.split('&')[0] title = meta.replace('+', ' ') + '.csv' else: # improper usage print('The entered youtube link is incompatible with the program') print('youtube link styles that work:') print('\thttps://www.youtube.com/') print('\thttps://www.youtube.com/results?search_query=valuetainment') print('\thttps://www.youtube.com/watch?v=x9dgZQsjR6s') print('\thttps://www.youtube.com/user/patrickbetdavid') print('\thttps://www.youtube.com/playlist?list=PLFa0bDwXvBlDGFtce9u__1sBj6fgi21BE') driver.close() sys.exit(-1) # scroll to the bottom of the youtube page for i in range(scroll_amnt): printProgressBar(i + 1, scroll_amnt, 'Navigating Youtube:', length=50) driver.find_element_by_tag_name('body').send_keys(Keys.END) sleep(3) html = driver.page_source soup = BeautifulSoup(html, 'html.parser') master_list = [] if 'list=' not in yt_link and not single_video: # channel, homepage, and search videos = soup.find_all('div', {'id': 'dismissable'}) for video in videos: if 'ytd-shelf-renderer' in video['class'] or 'ytd-compact-promoted-item-renderer' in video['class'] or 'ytd-rich-shelf-renderer' in video['class']: continue # remove the outer most nested 'video' data_dict = {} # get title data_dict['title'] = video.find('a', {'id': title_id}).text.replace('\n', '') # get video url + use id for youtube API
def process_file(reader, writer, writer_for_rows_without_location_with_address, current_row_index=0, total_row_count=1000): count_rows_without_coordinates_location_found = 0 count_rows_without_coordinates_location_not_found = 0 count_rows_without_coordinates_error_when_geocoding = 0 count_rows_without_coordinates_and_address = 0 count_rows_without_period = 0 count_rows_without_year = 0 count_rows_without_bo_number = 0 count_rows_without_police_station_name = 0 #current_row_index = 0 for row in reader: current_row_index += 1 printProgressBar(current_row_index, total_row_count, suffix="Complete", decimals=3) row_is_incomplete = False # row[6] -> PERIODOOCORRENCIA if (row[6] == ""): count_rows_without_period += 1 row_is_incomplete = True # row[0] -> ANO_BO # row[1] -> NUM_BO # row[22] -> DELEGACIA_NOME if (row[0] == ""): count_rows_without_year += 1 row_is_incomplete = True if (row[1] == ""): count_rows_without_bo_number += 1 row_is_incomplete = True if (row[22] == ""): count_rows_without_police_station_name += 1 row_is_incomplete = True if (row_is_incomplete): continue # row[17] -> LATITUDE # row[18] -> LONGITUDE if (row[17] == "" or row[18] == ""): if (row[12] != "" and row[13] != "" and row[15] != ""): writer_for_rows_without_location_with_address.writerow([ row[0], row[1], convert_date(row[3]), convert_date(row[4]), convert_date(row[5]), convert_period(row[6]), convert_yes_no(row[10]), row[12], row[13], row[14], row[15], row[17], row[18], row[19], row[22], row[23], row[25], (convert_yes_no(row[30]) if row[30] != '' else ''), (row[35][0] if row[35] != '' else ''), (row[37] if (row[37] != '' and row[37] != '0') else ''), (convert_skin_color(row[41]) if row[41] != '' else ''), (row[42] if row[42] != '' else '') ]) count_rows_without_coordinates_location_not_found += 1 continue """ try: time.sleep(2) location = geocoder.google(row[12] + ", " + row[13] + " - " + ((row[14] + ", ") if row[14] != "" else "") + row[15] + " - SP", key="AIzaSyDn0gFwQ9wWVjFofAuxZVKh8-Pfqg_Y5yM") if (location.ok): row[17] = location.latlng[0] row[18] = location.latlng[1] count_rows_without_coordinates_location_found += 1 else: count_rows_without_coordinates_location_not_found += 1 row_is_incomplete = True except requests.exceptions.RequestException: count_rows_without_coordinates_error_when_geocoding += 1 """ else: count_rows_without_coordinates_and_address += 1 row_is_incomplete = True if (row_is_incomplete): continue if ((type(row[17]) is not str and type(row[17]) is not float) or (type(row[18]) is not str and type(row[18]) is not float)): raise Exception() writer.writerow([ row[0], row[1], convert_date(row[3]), convert_date(row[4]), convert_date(row[5]), convert_period(row[6]), convert_yes_no(row[10]), (row[17].replace(',', '.') if type(row[17]) is str else row[17]), (row[18].replace(',', '.') if type(row[18]) is str else row[18]), row[19], row[22], row[23], row[25], (convert_yes_no(row[30]) if row[30] != '' else ''), (row[35][0] if row[35] != '' else ''), (row[37] if (row[37] != '' and row[37] != '0') else ''), (convert_skin_color(row[41]) if row[41] != '' else ''), (row[42] if row[42] != '' else '') ]) return [ count_rows_without_coordinates_location_found, count_rows_without_coordinates_location_not_found, count_rows_without_coordinates_error_when_geocoding, count_rows_without_coordinates_and_address, count_rows_without_period, count_rows_without_year, count_rows_without_bo_number, count_rows_without_police_station_name, current_row_index ]
# building_positive dictionary print('building positive dictionary...') print('') positive_dict = build_dict(positive_path) print('') #calculate porbabilities print("calulating probabilities...") print("iterating through negative_tokens...") #progress l = len(negative_dict) i = 0 progress_bar.printProgressBar(0, l, prefix='Progress:', suffix='Complete', length=50) for i, k in enumerate(negative_dict): progress_bar.printProgressBar(iteration=i + 1, total=l, prefix='Progress:', suffix='Complete', length=50) num_in_negatives = negative_dict[k] if k in positive_dict: num_in_positives = positive_dict[k] else: num_in_positives = 0