def __init__(self, root: str = folder, train_subset: bool = True, suffix: str = '.png', min_num_cls: int = 5, max_num_cls: int = 20, k_shot: int = 20, expand_dim: bool = False, load_images: bool = True) -> None: """Initialize a data loader for Omniglot data set or a two-level dataset with structure similar to Omniglot: alphabet -> character -> image Args: root (str): path to the folder of Omniglot data set train_subset (bool): if True, this will load data from the ``images_background`` folder (or, training set). If False, it loads data from ``images_evaluation``` (or, validation set) suffix (str): the suffix of images min_num_cls (int): minimum number of classes within a generated episode max_num_cls (int): maximum number of classes within a generated episode expand_dim (bool): if True, repeat the channel dimension from 1 to 3 load_images (bool): if True, this will place all image data (PIL) on RAM. This option is optimal for small data set since it would speed up the data loading process. If False, it will load images whenever called. This option is suitable for large data set. Returns: an OmniglotLoader instance """ self.root = os.path.join( root, 'images_background' if train_subset else 'images_evaluation') self.suffix = suffix self.min_num_cls = min_num_cls self.max_num_cls = max_num_cls self.k_shot = k_shot self.expand_dim = expand_dim self.load_images = load_images # create a nested dictionary to store data self.data = dict.fromkeys(list_dir(root=self.root)) for alphabet in self.data: self.data[alphabet] = dict.fromkeys( list_dir(root=os.path.join(self.root, alphabet))) # loop through each alphabet for character in self.data[alphabet]: self.data[alphabet][character] = [] # loop through all images in an alphabet character for img_name in list_files(root=os.path.join( self.root, alphabet, character), suffix=suffix): if self.load_images: # load images img = _load_image(img_url=os.path.join( self.root, alphabet, character, img_name), expand_dim=self.expand_dim) else: img = img_name self.data[alphabet][character].append(img)
def wait_for_rar(folder, sab_nzo_id, some_rar): isCanceled = False is_rar_found = False # If some_rar exist we skip dialogs for file, bytes in utils.sorted_rar_file_list(utils.list_dir(folder)): if file == some_rar: is_rar_found = True break if not is_rar_found: seconds = 0 progressDialog = xbmcgui.DialogProgress() progressDialog.create('NZBS', 'Request to SABnzbd succeeded, waiting for ', some_rar) while not is_rar_found: seconds += 1 time.sleep(1) dirList = utils.sorted_rar_file_list(utils.list_dir(folder)) for file, bytes in dirList: if file == some_rar: path = os.path.join(folder,file) # Wait until the file is written to disk before proceeding size_now = int(bytes) size_later = 0 while (size_now != size_later) or (size_now == 0) or (size_later == 0): size_now = os.stat(path).st_size if size_now != size_later: time.sleep(0.5) size_later = os.stat(path).st_size is_rar_found = True break label = str(seconds) + " seconds" # TODO # Shorten some_rar if to long for the dialog window progressDialog.update(0, 'Request to SABnzbd succeeded, waiting for', some_rar, label) if progressDialog.iscanceled(): progressDialog.close() dialog = xbmcgui.Dialog() ret = dialog.select('What do you want to do?', ['Delete job', 'Just download']) if ret == 0: pause = SABNZBD.pause('',sab_nzo_id) time.sleep(3) delete_ = SABNZBD.delete_queue('',sab_nzo_id) if not "ok" in delete_: xbmc.log(delete_) xbmc.executebuiltin('Notification("NZBS","Deleting failed")') else: xbmc.executebuiltin('Notification("NZBS","Deleting succeeded")') iscanceled = True return iscanceled if ret == 1: iscanceled = True xbmc.executebuiltin('Notification("NZBS","Downloading")') return iscanceled progressDialog.close() return isCanceled
def import_call(self, e): if setting_fftool.has_query: utils.showinfo("有任务正在执行,请稍后") return tup = tuple([]) ft = self.file_types ft_tup = self.file_types_tup if e.widget == self.import_btn: tup = filedialog.askopenfilenames( filetypes=ft, title='导入文件', initialdir=setting_fftool.last_folder) elif e.widget == self.import_list_btn: if os.path.exists(setting_fftool.list_file): arr = utils.read_txt(setting_fftool.list_file) new_arr = [] for f in arr: if os.path.exists(f): new_arr.append(f) if not len(new_arr): utils.showinfo('txt中的地址都不正确' + setting_fftool.list_file) return tup = tuple(new_arr) elif e.widget == self.import_dir_btn: folder = filedialog.askdirectory( title='选择目录', initialdir=setting_fftool.last_folder) if folder: folder = utils.pathlib_path(folder) setting_fftool.last_folder = folder arr = [] new_arr = [] # 获得目录下所有文件 utils.list_dir(folder, arr) # 过滤出指定格式的文件 for f in arr: suffix = str(Path(f).suffix) for f_type in ft_tup: if suffix == f_type: new_arr.append(f) break tup = tuple(new_arr) if len(tup): tup = utils.pathlib_path_tup(tup, True) self.tree.set_list(list(tup)) # self.start.set_state(True) # self.clear_query() setting_fftool.last_folder = utils.pathlib_parent(tup[0])
def loadNorlmalRace(self, detect_dir): # .../detect => .../summary summary_dir = detect_dir.replace('detect', 'summary') for f in list_dir(summary_dir, ext='NORMAL.tot-race'): with open(f, 'r') as fd: for line in fd: self.normal_race_.add(InstRace.parse(line))
def upload(self, container, path, verbose=True): if path[-1] == '/': path = path[:-1] files = utils.list_dir(path) for filename in files: (fh, content_type, content_length) = utils.get_file_infos(filename) url = "{}/{}/{}".format(self.get_storage_url(), container, filename) data = fh.read() headers = { 'Content-Type': content_type, 'X-Storage-Token': self.get_token(), 'Content-Length': content_length } request = urllib2.Request(url, data, headers) request.get_method = lambda: 'PUT' response = urllib2.urlopen(request) if response.code == 201: msg = '{} - OK'.format(filename) else: msg = '{} - FAIL (error {})'.format(filename, response.code) if verbose: print(msg)
def main(conf): """ Read js-test run log (data/log/), check whether error occurs, fill a result dict (ret_dict: {0: [paths], 1:[paths]}); Pick those without errors (ret_dict[0]), write to seed files (data/seed). Remove lines containing 'load("' or "load('" -- they reference to other files (how to deal with them??) :param conf: Config object :return: """ func = get_func(conf.eng_name) ret_dict = {} for log_path in list_dir(conf.log_dir): ret = func(log_path) if ret not in ret_dict: ret_dict[ret] = [] ret_dict[ret] += [log_path] new_seed_dir = os.path.join(conf.data_dir, 'seed') make_dir(new_seed_dir) for log_path in ret_dict[0]: file_name = os.path.basename(log_path) + '.js' js_path = os.path.join(conf.seed_dir, file_name) new_js_path = os.path.join(new_seed_dir, file_name) with open(js_path, 'r') as fr, \ open(new_js_path, 'w') as fw: for line in fr: if ('load("' not in line and 'load(\'' not in line): fw.write(line)
def loadTotalRaceFile(self, detect_dir, sched): # if sched != 'NORMAL': # self.loadNorlmalRace(detect_dir) for f in list_dir(detect_dir, ext='.race', fkey=self.fnameTimestamp): if sched is not None and sched not in f: continue self.loadRaceFile(f)
def detectTotalRace(self, detect_dir, sched): for trace_file in list_dir(detect_dir, ext='.trace'): if sched is not None and sched not in trace_file: continue race_file = trace_file.replace('.trace', '.race') if not os.path.exists(race_file): print 'TO detect:', trace_file bin = os.path.join(PROJ_DIR, 'bin/detect') shell_exec('%s %s' % (bin, trace_file))
def parse(self, seed_dir, ast_dir): js_list = list_dir(seed_dir) num_js = len(js_list) msg = 'Start parsing %d JS files' % (num_js) print_msg(msg, 'INFO') cmd = ['node', 'utils/parse.js'] cmd += [seed_dir, ast_dir] parser = Popen(cmd, cwd='./', stdin=PIPE, stdout=PIPE, stderr=PIPE) parser.wait()
def __init__(self, root='omniglot', *args, **kwargs): ''' :param root: folder containing alphabets for background and evaluation set ''' self.root = root self.alphabets = list_dir(root) self._characters = {} for alphabet in self.alphabets: for character in list_dir(os.path.join(root, alphabet)): full_character = os.path.join(root, alphabet, character) character_idx = len(self._characters) self._characters[full_character] = [] for filename in list_files(full_character, '.png'): self._characters[full_character].append({ 'path': os.path.join(root, alphabet, character, filename), 'character_idx': character_idx }) characters_list = np.asarray(self._characters.items()) AbstractMetaOmniglot.__init__(self, characters_list, *args, **kwargs)
def import_zupc(import_='True'): require.files.directory('/tmp/zupc') with cd('/tmp/zupc/'): for f in list_dir(): if f == '*' or f.endswith('zip'): continue run('rm -f {}'.format(f)) if import_=='True': import_contours() base_dir = '' with cd(env.deploy_dir): for f in list_dir(): if files.is_dir(f) and 'deployment' in f and f > base_dir: base_dir = f api_dir = env.deploy_dir+'/' + base_dir with cd('/tmp/zupc'): wget(env.zupc_fichier) with python.virtualenv(base_dir + '/venvAPITaxi'), cd(base_dir+'/APITaxi-master'): with shell_env(APITAXI_CONFIG_FILE='prod_settings.py'): run('python manage.py load_zupc /tmp/zupc/zupc.geojson')
def removeEmptyFile(self, detect_dir): for f in list_dir(detect_dir): # Iterate files while deleting if not os.path.exists(f): continue if os.path.getsize(f) == 0: print 'rm', f rm_file(f) # To remove corresponding trace file if f.endswith('.race'): print 'rm', f.replace('.race', '.trace') rm_file(f.replace('.race', '.trace'))
def __init__(self, root, background=True, transform=None, target_transform=None, download=False): self.root = join(os.path.expanduser(root), self.folder) self.background = background self.transform = transform self.target_transform = target_transform if download: self.download() if not self._check_integrity(): raise RuntimeError('Dataset not found or corrupted.' + ' You can use download=True to download it') self.target_folder = join(self.root, self._get_target_folder()) self._alphabets = list_dir(self.target_folder) self._characters = sum([[join(a, c) for c in list_dir(join(self.target_folder, a))] for a in self._alphabets], []) self._character_images = [[(image, idx) for image in list_files(join(self.target_folder, character), '.png')] for idx, character in enumerate(self._characters)] self._flat_character_images = sum(self._character_images, [])
def build_dict(ast_dir): def_dict = {} ast_list = list_dir(ast_dir) num_ast = len(ast_list) for idx, ast_name in enumerate(ast_list): msg = '[%d/%d] %s' % (idx + 1, num_ast, ast_name) print_msg(msg, 'INFO') js_name, ast = load_ast(ast_name) js_name = trim_seed_name(js_name) if js_name not in def_dict: def_dict[js_name] = set() build_def_dict(ast, def_dict[js_name]) return def_dict
def run(self): # type: () -> None while not self._join: loginfo('ros_rest : ' + self.name + ' : looking for new records') try: # Check for new sorted records to send, if there is send it to the server using the shared session for bag in list_dir(self.__last): _bags.put(bag) self.__last = bag.bag.split('.')[0].split('_')[-1] except BaseException as err: logwarn('ros_rest : ' + self.name + ' : ' + str(err)) _event.wait(_err_sleep) _event.wait(self.sleep) _bags.join() self.join()
def main(pool, conf): make_dir(conf.log_dir) js_list = [] for js in list_dir(conf.seed_dir): if (js.endswith('.js') and os.path.getsize(js) < 30 * 1024): # Excludes JS over 3KB js_list += [js] num_js = len(js_list) msg = 'Start executing %d JS files' % (num_js) print_msg(msg, 'INFO') pool_map(pool, exec_eng, js_list, conf=conf)
def archives(self): files = list(list_dir(config.events_dir)) files.sort() events = defaultdict(dict) for f in files: for (tid, event) in self.load_events(f).items(): events[tid].update(event) tids = self.get_tids() history = [] for tid, e in events.items(): info = self.get_torrent_info(e['filename']) info and e.update(info.__dict__) if tid not in tids: history.append(e) return sorted(history, key = lambda e: self.event_date(e), reverse = True)
def parse(self, seed_dir, ast_dir): js_list = list_dir(seed_dir) num_js = len(js_list) msg = 'Start parsing %d JS files' % (num_js) print_msg(msg, 'INFO') cmd = ['node', 'utils/parse.js'] cmd += [seed_dir, ast_dir] print(cmd) parser = Popen(cmd, cwd='./', stdin=PIPE, stdout=PIPE, stderr=PIPE) ## Error would be silented out by default. Uncomment when having problems. #for line in parser.stderr.readlines(): # print(line) # sys.stderr.flush() parser.wait()
def __init__(self, root, loader, extensions, transform=None): self.root = root self.loader = loader self.extensions = extensions dirs = list_dir(self.root) if(dirs != []): self.classes, self.classes_to_labels = self._find_classes(self.root) self.classes_size = len(self.classes_to_labels.keys()) self.output_names = ['img','label'] self.classes_counts = make_classes_counts(self.data['label'],self.classes_size) else: self.classes_to_labels = None self.classes_size = 0 self.output_names = ['img'] self.data = make_img_dataset(self.root, self.extensions, self.classes_to_labels) self.transform = transform
def get_cls_img(root: str, suffix: str) -> dict: """Get folders from root, and images in each folder Args: root (str): the desired directory suffix (str): the suffix of file or image within each folder Returns: dictionary with keys are the folder names, and values are the lists of files within the corresponding folder """ cls_img = dict.fromkeys(list_dir(root=root)) for dir_ in cls_img: cls_img[dir_] = list_files(root=os.path.join(root, dir_), suffix=suffix) return cls_img
def build_dict(self): js_list = list_dir(self._seed_dir) for js_path in js_list: with open(js_path, 'r') as f: js_name = os.path.basename(js_path) for line in f: line = line.strip() if not self.is_load(line): continue harness = self.get_harness(line) if not harness.endswith('.js'): continue if js_name not in self._harness_dict: self._harness_dict[js_name] = [] self._harness_dict[js_name] += [harness]
def today(self): torrents = dict(self.get_torrents()) files = list(list_dir(config.events_dir)) files.sort() events = defaultdict(dict) for f in files: for (tid, event) in self.load_events(f).items(): events[tid].update(event) tids = torrents.keys() history = [] for tid, e in events.items(): info = self.get_torrent_info(e['filename']) info and e.update(info.__dict__) if tid in tids: torrents[tid].__dict__.update(e) elif e['last'].date() == date.today(): history.append(e) return (torrents.values(), sorted(history, key = lambda e: self.event_date(e), reverse = True))
def build_dict(ast_dir): ''' Gather all ast .json files, load them (a dict for each .json), and product a map for each file. :param ast_dir: :return: ''' def_dict = {} ast_list = list_dir(ast_dir) num_ast = len(ast_list) for idx, ast_name in enumerate(ast_list): msg = '[%d/%d] %s' % (idx + 1, num_ast, ast_name) print_msg(msg, 'INFO') js_name, ast = load_ast(ast_name) js_name = trim_seed_name(js_name) if js_name not in def_dict: def_dict[js_name] = set() build_def_dict(ast, def_dict[js_name]) return def_dict
def main(pool, conf): """ Read from js-test-suite, process with the engine, and write stdout/stderr to data/log/* :param pool: :param conf: :return: """ make_dir(conf.log_dir) js_list = [] for js in list_dir(conf.seed_dir): if (js.endswith('.js') and os.path.getsize(js) < 30 * 1024): # Excludes JS over 3KB js_list += [js] num_js = len(js_list) msg = 'Start executing %d JS files' % (num_js) print_msg(msg, 'INFO') pool_map(pool, exec_eng, js_list, conf=conf)
def main(conf): func = get_func(conf.eng_name) ret_dict = {} for log_path in list_dir(conf.log_dir): ret = func(log_path) if ret not in ret_dict: ret_dict[ret] = [] ret_dict[ret] += [log_path] new_seed_dir = os.path.join(conf.data_dir, 'seed') make_dir(new_seed_dir) for log_path in ret_dict[0]: file_name = os.path.basename(log_path) + '.js' js_path = os.path.join(conf.seed_dir, file_name) new_js_path = os.path.join(new_seed_dir, file_name) with open(js_path, 'r') as fr, \ open(new_js_path, 'w') as fw: for line in fr: if ('load("' not in line and 'load(\'' not in line): fw.write(line)
def main(pool, conf): ast_list = list_dir(conf.ast_dir) pool_map(pool, normalize, ast_list)
def pre_play(nzbname, mode=None): iscanceled = False folder = INCOMPLETE_FOLDER + nzbname sab_nzo_id = SABNZBD.nzo_id(nzbname) file_list = utils.list_dir(folder) sab_file_list = [] multi_arch_list = [] if sab_nzo_id is None: sab_nzo_id_history = SABNZBD.nzo_id_history(nzbname) else: sab_file_list = SABNZBD.file_list(sab_nzo_id) file_list.extend(sab_file_list) sab_nzo_id_history = None file_list = utils.sorted_rar_file_list(file_list) multi_arch_list = utils.sorted_multi_arch_list(file_list) # Loop though all multi archives and add file to the play_list = [] for arch_rar, byte in multi_arch_list: if sab_nzo_id is not None: t = Thread(target=to_bottom, args=( sab_nzo_id, sab_file_list, file_list, )) t.start() iscanceled = get_rar(folder, sab_nzo_id, arch_rar) if iscanceled: break else: if sab_nzo_id: set_streaming(sab_nzo_id) # TODO is this needed? time.sleep(1) # RAR ANALYSYS # in_rar_file_list = utils.rar_filenames(folder, arch_rar) movie_list = utils.sort_filename(in_rar_file_list) # Make sure we have a movie if not (len(movie_list) >= 1): xbmc.executebuiltin('Notification("NZBS","Not a movie!")') break # Who needs sample? movie_no_sample_list = utils.no_sample_list(movie_list) # If auto play is enabled we skip samples in the play_list if AUTO_PLAY and mode is not MODE_INCOMPLETE_LIST: for movie_file in movie_no_sample_list: play_list.append(arch_rar) play_list.append(movie_file) else: for movie_file in movie_list: play_list.append(arch_rar) play_list.append(movie_file) # If the movie is a .mkv we need the last rar if utils.is_movie_mkv(movie_list) and sab_nzo_id: # If we have a sample or other file, the second rar is also needed.. if len(in_rar_file_list) > 1: second_rar = utils.find_rar(file_list, 0) iscanceled = get_rar(folder, sab_nzo_id, second_rar) last_rar = utils.find_rar(file_list, -1) iscanceled = get_rar(folder, sab_nzo_id, last_rar) if iscanceled: break if iscanceled: return else: rar_file_list = [x[0] for x in file_list] if (len(rar_file_list) >= 1): if AUTO_PLAY and (mode is None or mode is MODE_JSONRPC): video_params = dict() if not mode: video_params['mode'] = MODE_AUTO_PLAY else: video_params['mode'] = MODE_JSONRPC video_params['play_list'] = urllib.quote_plus( ';'.join(play_list)) video_params['file_list'] = urllib.quote_plus( ';'.join(rar_file_list)) video_params['folder'] = urllib.quote_plus(folder) return play_video(video_params) else: return playlist_item(play_list, rar_file_list, folder, sab_nzo_id, sab_nzo_id_history) else: xbmc.executebuiltin( 'Notification("NZBS","No rar\'s in the NZB!!")') return
def wait_for_rar(folder, sab_nzo_id, some_rar): isCanceled = False is_rar_found = False # If some_rar exist we skip dialogs for file, bytes in utils.sorted_rar_file_list(utils.list_dir(folder)): if file == some_rar: is_rar_found = True break if not is_rar_found: seconds = 0 progressDialog = xbmcgui.DialogProgress() progressDialog.create('NZBS', 'Request to SABnzbd succeeded, waiting for ', utils.short_string(some_rar)) while not is_rar_found: seconds += 1 time.sleep(1) dirList = utils.sorted_rar_file_list(utils.list_dir(folder)) for file, bytes in dirList: if file == some_rar: path = os.path.join(folder, file) # Wait until the file is written to disk before proceeding size_now = int(bytes) size_later = 0 while (size_now != size_later) or (size_now == 0) or (size_later == 0): size_now = os.stat(path).st_size if size_now != size_later: time.sleep(0.5) size_later = os.stat(path).st_size is_rar_found = True break label = str(seconds) + " seconds" progressDialog.update(0, 'Request to SABnzbd succeeded, waiting for', utils.short_string(some_rar), label) if progressDialog.iscanceled(): progressDialog.close() dialog = xbmcgui.Dialog() ret = dialog.select('What do you want to do?', ['Delete job', 'Just download']) if ret == 0: pause = SABNZBD.pause('', sab_nzo_id) time.sleep(3) delete_ = SABNZBD.delete_queue('', sab_nzo_id) if not "ok" in delete_: xbmc.log(delete_) xbmc.executebuiltin( 'Notification("NZBS","Deleting failed")') else: xbmc.executebuiltin( 'Notification("NZBS","Deleting succeeded")') iscanceled = True return iscanceled if ret == 1: iscanceled = True xbmc.executebuiltin('Notification("NZBS","Downloading")') return iscanceled progressDialog.close() return isCanceled
def tile_to_csv(inputpath, coordspath, coordsfilename, patches_per_tile, patch_size, classes): """ save location of patches in csv-file. Locations of top-left corner-pixel of patches are saved. These pixels are chosen at random, however the percentual class division is respected. parameters ---------- inputpath: string path to folders with tiles. Each tile should be in separate folder coordspath: string path to outputfolder to save file with coordinates coordsfilename: string output filename, extention should be '.csv' patches_per_tile: int number of patches to extract per tile. Final number can be lower if the classes cover very few pixels patch_size: int size of patch to extract. Final extracted patches will include padding to be able to predict full image. classes: list list with classes to be predicted calls ----- sample_patches_of_class() output ------- outputfile: csv each row contains tile-name and row + column of top-left pixel of patch: tile,row,column saved at outputpath. """ # init dirs = list_dir(inputpath) patch_size = patch_size // 5 # because downsample from 20cmX20cm to 1mx1m patch_size_padded = int(patch_size * 3) if not os.path.isdir(coordspath): os.makedirs(coordspath) for i_lap, d in enumerate(dirs): # ground truth path_SHP = inputpath + d + '/tare.tif' gt = gdal.Open(path_SHP,gdal.GA_ReadOnly) # resample to 1m resolution gt = gdal.Warp('', [gt], format='MEM', width=gt.RasterXSize//5, height=gt.RasterYSize//5, resampleAlg=0) band = gt.GetRasterBand(1) gt = np.int16(band.ReadAsArray()) del band # take care of classes tara0_mask = gt==classes[0] tara20_mask = gt==classes[1] tara50_mask = gt==classes[2] woods_mask = np.logical_or(gt==classes[3],gt==656) no_coltivable_mask = np.logical_or(gt==classes[4],gt==780) gt[woods_mask]=classes[3] gt[no_coltivable_mask]=classes[4] classes_mask = np.logical_or(tara50_mask,np.logical_or(tara0_mask,tara20_mask)) classes_mask = np.logical_or(no_coltivable_mask,np.logical_or(classes_mask,woods_mask)) gt[np.logical_not(classes_mask)]=0 rc_tara0 = np.argwhere(tara0_mask[0:-patch_size_padded, 0:-patch_size_padded]) rc_tara20 = np.argwhere(tara20_mask[0:-patch_size_padded, 0:-patch_size_padded]) rc_tara50 = np.argwhere(tara50_mask[0:-patch_size_padded, 0:-patch_size_padded]) rc_woods = np.argwhere(woods_mask[0:-patch_size_padded, 0:-patch_size_padded]) rc_no_coltivable = np.argwhere(no_coltivable_mask[0:-patch_size_padded, 0:-patch_size_padded]) rc_UPAS = np.argwhere(gt[0:-patch_size_padded, 0:-patch_size_padded]!=0) if np.sum(tara0_mask)==0 and np.sum(tara20_mask)==0 and np.sum(tara50_mask)==0 and np.sum(woods_mask)==0 and np.sum(no_coltivable_mask)==0 : continue # sample patches and write coordinate of origin to output csv-file sample_patches_of_class(rc_tara0, rc_UPAS, patches_per_tile, classes[0], gt, patch_size_padded, coordspath+coordsfilename,d) sample_patches_of_class(rc_tara20, rc_UPAS, patches_per_tile, classes[1], gt, patch_size_padded, coordspath+coordsfilename,d) sample_patches_of_class(rc_tara50, rc_UPAS, patches_per_tile, classes[2], gt, patch_size_padded, coordspath+coordsfilename,d) sample_patches_of_class(rc_woods, rc_UPAS, patches_per_tile, classes[3], gt, patch_size_padded, coordspath+coordsfilename,d) sample_patches_of_class(rc_no_coltivable, rc_UPAS, patches_per_tile, classes[4], gt, patch_size_padded, coordspath+coordsfilename,d) del gt gc.collect() if i_lap+1 % 10 == 0: print('\r {}/{}'.format(i_lap+1, len(dirs)),end='')
def tile_to_csv_grid(inputpath, coordspath, coordsfilename, patch_size_padded, classes=[638, 659, 654, 650, 770], tile=None, fraction=1, final=False): """ save location of patches in csv-file. Locations of top-left corner-pixel of patches are saved in csv. These corner pixels are based on a grid overlaying the original tiles. parameters ---------- inputpath: string path to folders with tiles. Each tile should be in separate folder coordspath: string path to outputfolder to save file with coordinates coordsfilename: string output filename, extention should be '.csv' patch_size_padded: int size of patch to extract including padding to be able to predict full image. classes: list classes to be considered in the ground truth tile: string (optional) name of folder where tile is stored fraction: float (optional) float between 0 and 1, fraction of pixels that must be in 'classes' calls ----- find_patches_options() plot_patch_options() output ------- outputfile: csv each row contains row + column of top-left pixel of patch,tilename and number of rows/cols in original tile: row,column,tile,tile-rows, tile-cols saved at coordspath with name coordsfilename """ # init if tile != None: dirs = [tile] else: dirs = list_dir(inputpath) if not os.path.isdir(coordspath): os.makedirs(coordspath) for i_lap, d in enumerate(dirs): # get tile path_shp = inputpath + d + '/tare.tif' ds = gdal.Open(path_shp, gdal.GA_ReadOnly) gt = ds.GetRasterBand(1).ReadAsArray() gt = np.uint16(gt) ds = None # take care of classes gt[np.where(gt == 656)] = 650 gt[np.where(gt == 780)] = 770 gt[np.isin(gt, classes) == False] = 0 # get options if i_lap == 0: options = find_patches_options(gt, patch_size_padded, d, fraction, final) else: options = options.append( find_patches_options(gt, patch_size_padded, d, fraction, final)) if i_lap % 50 == 0: print('\r {}/{}'.format(i_lap, len(dirs)), end='') # save in csv options['row'] = options['row'].multiply(patch_size_padded) options['col'] = options['col'].multiply(patch_size_padded) options.to_csv(coordspath + coordsfilename, index=False)
def csv_to_patch(inputpath, dtmpath, patchespath, coordsfile, patch_size, classes, resolution, tile=None): """ extract the patches from the original images, normalize and save. Ground truth is converted to one-hot labels. Parameters ---------- inputpath: string path to folders with tiles. Each tile should be in separate folder dtmpath: string path to folder containing a dtm (will be resampled to same resolution) patchespath: string path to outputfolder to save the patches coordsfile: csv-file path to file where the coordinates are saved patch_size: int size of patch to extract. Final extracted patches will include padding to be able to predict full image. classes: list list with classes to be predicted resolution: int either 20 for 20cm or 1 for 1m tile: string name of folder where tile is stored Calls ----- read_patch() read_patch_1m() to_categorical_classes() Output ------ patches saved at patchespath in two folders: images and labels. """ imagespath = patchespath + 'images/' labelspath = patchespath + 'labels/' if not os.path.isdir(patchespath): os.makedirs(patchespath) if not os.path.isdir(imagespath): os.makedirs(imagespath) if not os.path.isdir(labelspath): os.makedirs(labelspath) if tile != None: dirs = [tile] else: dirs = list_dir(inputpath) coords = pd.read_csv(coordsfile, sep=',') patch_size_padded = int(patch_size * 3) if resolution == 20: # resample dtm to 20cmx20xm for d in dirs: if not os.path.isdir(dtmpath + d + '/'): os.makedirs(dtmpath + d + '/') input_file = inputpath + d + '/dtm135.tif' shadow_file = inputpath + d + '/' + d + '_NIR.tif' dtm_file = dtmpath + d + '/dtm135_20cm.tif' ds = gdal.Open(shadow_file) width = ds.RasterXSize height = ds.RasterYSize ds = gdal.Warp(dtm_file, input_file, format='GTiff', width=width, height=height, resampleAlg=1) ds = None # extract patches for idx in range(len(coords)): im, gt = read_patch(inputpath, dtmpath, coords, patch_size_padded, idx, classes) np.save(imagespath + str(idx) + '.npy', im) np.save(labelspath + str(idx) + '.npy', gt) if idx % 500 == 0: print('\r {}/{}'.format(idx, len(coords)), end='') elif resolution == 1: warpedtile = None for idx, d in enumerate(coords['tiles']): # resample rgb + nir to 1m (keep in memory (no space on disk)) if not d == warpedtile: dtm_file = inputpath + d + '/dtm135.tif' nir_file = inputpath + d + '/' + d + '_NIR.tif' rgb_file = inputpath + d + '/' + d + '_RGB.tif' gt_file = inputpath + d + '/tare.tif' ds = gdal.Open(dtm_file) width = ds.RasterXSize height = ds.RasterYSize nir_1m = gdal.Warp("", nir_file, format='mem', width=width, height=height, resampleAlg=1) rgb_1m = gdal.Warp("", rgb_file, format='mem', width=width, height=height, resampleAlg=1) gt_1m = gdal.Warp("", gt_file, format='mem', width=width, height=height, resampleAlg=0) warpedtile = d ds = None # extract patches im, gt = read_patch_1m(rgb_1m, nir_1m, dtm_file, gt_1m, coords, patch_size_padded, idx, classes) np.save(imagespath + str(idx) + '.npy', im) np.save(labelspath + str(idx) + '.npy', gt) if idx % 500 == 0: print('\r {}/{}'.format(idx, len(coords)), end='') else: print("Only resoltions of 20cmx20xm (20) and 1mx1m (1) are supported.")
def get_queue(self): for f in sorted(list_dir(config.torrent_queue_dir), key = lambda f: os.stat(f).st_ctime): yield self.get_torrent_info(os.path.basename(f))
def return_data(args): # train_dset_dir = args.train_dset_dir # test_dset_dir = args.test_dset_dir set_seed(args.seed) train_batch_size = args.train_batch_size test_batch_size = args.test_batch_size num_workers = args.num_workers image_size = args.image_size time_window = args.time_window darker_threshold = args.darker_threshold trivial_augmentation = bool(args.trivial_augmentation) sliding_augmentation = bool(args.sliding_augmentation) transform_list = [ transforms.Resize((image_size, image_size)), RetouchDarker(darker_threshold) ] if args.channel == 1: transform_list.append(transforms.Grayscale(num_output_channels=1)) if trivial_augmentation: trivial_transform_list = [ transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1), transforms.RandomResizedCrop(image_size, scale=(0.8, 1.0), ratio=(1, 1)), RandomNoise(mean=0, std=10), ] transform_list.append(transforms.RandomChoice(trivial_transform_list)) if sliding_augmentation: transform_list.append(RandomTimeWindow(time_window=time_window)) else: transform_list.append(TimeWindow(time_window=time_window)) transform_list.append(transforms.ToTensor()) if args.channel == 1: transform_list.append(transforms.Normalize([0.5], [0.5])) else: transform_list.append( transforms.Normalize([0.5] * args.channel, [0.5] * args.channel)) print(transform_list) transform = transforms.Compose(transform_list) # if args.channel == 1: # transform = transforms.Compose([ # transforms.Resize((image_size, image_size)), # transforms.Grayscale(num_output_channels=1), # TimeWindow(time_window=time_window), # transforms.ToTensor(), # transforms.Normalize([0.5], [0.5]), # ]) # else: # transform = transforms.Compose([ # transforms.Resize((image_size, image_size)), # TimeWindow(time_window=time_window), # transforms.ToTensor(), # transforms.Normalize([0.5] * args.channel, [0.5] * args.channel), # ]) """ train_root = Path(train_dset_dir) test_root = Path(test_dset_dir) train_kwargs = {'root': train_root, 'transform': transform} test_kwargs = {'root': test_root, 'transform': transform} dset = ImageFolder train_data = dset(**train_kwargs) test_data = dset(**test_kwargs) train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=True) test_loader = DataLoader(test_data, batch_size=test_batch_size, shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=True) data_loader = dict() data_loader['train'] = train_loader data_loader['test'] = test_loader """ def _init_fn(worker_id): np.random.seed(int(args.seed)) data_loader = {} if args.incremental: root = './data/per_acitivity/' else: root = './data/per_subject/' num_tasks = len(list_dir(root)) train_imagefolders = [] test_imagefolders = [] for i in range(num_tasks): data_loader['task{}'.format(i)] = {} target_subject = join(root, 'Subject{}'.format(i + 1)) train_data = ImageFolder(root=target_subject + '/train', transform=transform) test_data = ImageFolder(root=target_subject + '/test', transform=transform) train_imagefolders.append(train_data) test_imagefolders.append(test_data) if args.continual: for i in range(num_tasks): # data loader가 cnn model 학습 이전에 이미 생성완료되어 선언되므로 여기서 replay를 하는건 불가능. if args.task_upper_bound: train_dataset = ConcatDataset(train_imagefolders[:i + 1]) test_dataset = ConcatDataset(test_imagefolders[:i + 1]) else: train_dataset = train_imagefolders[i] test_dataset = test_imagefolders[i] train_loader = DataLoader(train_dataset, batch_size=train_batch_size, shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=True, worker_init_fn=_init_fn) test_loader = DataLoader(test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=True, worker_init_fn=_init_fn) data_loader['task{}'.format(i)]['train'] = train_loader data_loader['task{}'.format(i)]['test'] = test_loader else: num_tasks = 1 train_data_concat = ConcatDataset(train_imagefolders) test_data_concat = ConcatDataset(test_imagefolders) train_loader = DataLoader(train_data_concat, batch_size=train_batch_size, shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=True, worker_init_fn=_init_fn) test_loader = DataLoader(test_data_concat, batch_size=test_batch_size, shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=True, worker_init_fn=_init_fn) data_loader['train'] = train_loader data_loader['test'] = test_loader return data_loader, num_tasks, transform
def main(pool, conf): ast_list = list_dir(conf.ast_dir) ast_data = pool_map(pool, fragmentize, ast_list) return ast_data
def pre_play(nzbname, mode = None): iscanceled = False folder = INCOMPLETE_FOLDER + nzbname sab_nzo_id = SABNZBD.nzo_id(nzbname) file_list = utils.list_dir(folder) sab_file_list = [] multi_arch_list = [] if sab_nzo_id is None: sab_nzo_id_history = SABNZBD.nzo_id_history(nzbname) else: sab_file_list = SABNZBD.file_list(sab_nzo_id) file_list.extend(sab_file_list) sab_nzo_id_history = None file_list = utils.sorted_rar_file_list(file_list) multi_arch_list = utils.sorted_multi_arch_list(file_list) # Loop though all multi archives and add file to the play_list = [] for arch_rar, byte in multi_arch_list: if sab_nzo_id is not None: t = Thread(target=to_bottom, args=(sab_nzo_id, sab_file_list, file_list,)) t.start() iscanceled = get_rar(folder, sab_nzo_id, arch_rar) if iscanceled: break else: if sab_nzo_id: set_streaming(sab_nzo_id) # TODO is this needed? time.sleep(1) # RAR ANALYSYS # in_rar_file_list = utils.rar_filenames(folder, arch_rar) movie_list = utils.sort_filename(in_rar_file_list) # Make sure we have a movie if not (len(movie_list) >= 1): xbmc.executebuiltin('Notification("NZBS","Not a movie!")') break # Who needs sample? movie_no_sample_list = utils.no_sample_list(movie_list) # If auto play is enabled we skip samples in the play_list if AUTO_PLAY and mode is not MODE_INCOMPLETE_LIST: for movie_file in movie_no_sample_list: play_list.append(arch_rar) play_list.append(movie_file) else: for movie_file in movie_list: play_list.append(arch_rar) play_list.append(movie_file) # If the movie is a .mkv we need the last rar if utils.is_movie_mkv(movie_list) and sab_nzo_id: # If we have a sample or other file, the second rar is also needed.. if len(in_rar_file_list) > 1: second_rar = utils.find_rar(file_list, 0) iscanceled = get_rar(folder, sab_nzo_id, second_rar) last_rar = utils.find_rar(file_list, -1) iscanceled = get_rar(folder, sab_nzo_id, last_rar) if iscanceled: break if iscanceled: return else: rar_file_list = [x[0] for x in file_list] if (len(rar_file_list) >= 1): if AUTO_PLAY and ( mode is None or mode is MODE_JSONRPC): video_params = dict() if not mode: video_params['mode'] = MODE_AUTO_PLAY else: video_params['mode'] = MODE_JSONRPC video_params['play_list'] = urllib.quote_plus(';'.join(play_list)) video_params['file_list'] = urllib.quote_plus(';'.join(rar_file_list)) video_params['folder'] = urllib.quote_plus(folder) return play_video(video_params) else: return playlist_item(play_list, rar_file_list, folder, sab_nzo_id, sab_nzo_id_history) else: xbmc.executebuiltin('Notification("NZBS","No rar\'s in the NZB!!")') return
def main(pool, conf): ast_list = list_dir(conf.ast_dir) ast_list = pool_map(pool, strip, ast_list, conf=conf)