def capture_video(self, sky, millis_end, millis_start=0, quiet=False): """ Captures a list of images, each one separated from the other at a rate of 1. / fps seconds Parameters: sky: sky object with stars and cubesats millis_end: time at which the last image is taken millis_start: time at which the first image is taken quiet: if True, no progress bar is displayed Returns a list of frames and a list of timestamps """ video = [] timestamps = [] millis = millis_start base = Image(width=self.camera.cols, height=self.camera.rows) # Base image with stars only progress_bar = ProgressBar('Capturing video', millis_start, millis_end, quiet) sky.draw_stars_on_image(base) while millis <= millis_end: captured = base.clone() sky.draw_cubesats_on_image(captured, millis) self._add_noise(captured) video.append(captured) timestamps.append(millis) progress_bar.update(millis) millis += 1000. / self.fps progress_bar.end() return timestamps, video
def fit(self, train_df): self.ds = pd.Series(train_df.index) print("Fitting...") progress_bar = ProgressBar(len(train_df.columns)) for item in train_df.columns: target = train_df[item].interpolate().bfill() if self.use_boxcox: idx = target.index target, self.lmbda_boxcox[item] = boxcox(target) target = pd.Series(target, index=idx) target.index.name = "ds" target.name = "y" self.models[item] = pm.auto_arima( target, seasonal=False, exogenous=fourier( len(target), seasonality=self.seasonality, n_terms=self.n_fourier_terms), method="bfgs", suppress_warnings=True) progress_bar.update() progress_bar.finish() return self.models
def createDemand(self): def determineStartingInterval(): start_interval = P_constrain(np.random.normal(), -3.0, 3.0) start_interval = P_map(start_interval, -3.0, 3.0, 15 * 60, self.MAX_INTERVAL - self.MAX_DURATION - 1) start_interval = int(P_constrain(start_interval, 0, self.MAX_INTERVAL - self.MAX_DURATION - 1)) return start_interval def determineSkillAssignments(): SKILLS = ['LEVEL_1', 'LEVEL_2', 'LEVEL_3'] skill = random.choice(SKILLS) return skill def determineDuration(): return int(P_map(P_constrain(np.random.normal(), -3.0, 3.0), -3.0, 3.0, 5, self.MAX_DURATION)) print('Creating demand list...') toolbar_width = 40 pb = ProgressBar(toolbar_width=toolbar_width) for x_ in range(self.NUM_CALLS): pb.update(x_, self.NUM_CALLS) start_interval = determineStartingInterval() duration = determineDuration() skill = determineSkillAssignments() self.addDemand({'id': x_ + 10, 'interval': start_interval, 'duration': duration, 'skill': skill}) pb.update(1, 1) pb.clean()
class ProgressBarTestCase(unittest.TestCase): """Test the ProgressBar class.""" DEFAULT_VALUE = 0 DEFAULT_MAX_VALUE = 100 DEFAULT_PRINT_WIDTH = 50 DEFAULT_NEWLINE = "\r" def setUp(self): """Set up for test.""" self.bar = ProgressBar(quiet=True) def tearDown(self): """Clean up after test.""" self.bar = None def test_defaults(self): """Test default values are correct.""" test_data = ( (self.bar.value, self.DEFAULT_VALUE, "default value = {v}".format(v=self.DEFAULT_VALUE)), (self.bar.initial_value, self.DEFAULT_VALUE, "default initial value = {v}".format(v=self.DEFAULT_VALUE)), (self.bar.max_value, self.DEFAULT_MAX_VALUE, "default max value = {v}".format(v=self.DEFAULT_MAX_VALUE)), (self.bar.print_width, self.DEFAULT_PRINT_WIDTH, "default print width = {v}".format(v=self.DEFAULT_PRINT_WIDTH)), (self.bar.newline, self.DEFAULT_NEWLINE, "default newline = {v}".format(v=self.DEFAULT_NEWLINE)), ) for actual, expected, description in test_data: with self.subTest(msg=description): self.assertEqual(actual, expected) def test_set_and_reset(self): """Test setting and resetting the current value.""" self.bar.set(value=50) self.assertEqual(self.bar.value, 50) self.bar.reset() self.assertEqual(self.bar.value, 0) def test_update_ad_draw(self): """Test updating and drawing the progress bar.""" self.bar.reset() self.bar.quiet = False for i in range(self.bar.initial_value, self.bar.max_value + 1): percent = int(i * 100 / self.bar.max_value) dots = int(i * self.bar.print_width / self.bar.max_value) expected_bar = "{nl}[{c}{nc}] {p}% ".format( c="".join(["+"] * dots), nc="".join(["."] * (self.bar.print_width - dots)), p=percent, nl=self.bar.newline) with captured_output() as (out, _): self.bar.update(i) with self.subTest(msg="value = {v}".format(v=i)): self.assertEqual(self.bar.value, i) with self.subTest(msg="output = {v}".format(v=expected_bar)): self.assertEqual(out.getvalue(), expected_bar)
def _generate_AX(self): self.log('Creating features and adjacency matrices..') pr = ProgressBar(60, len(self.data)) data = [] smiles = [] data_S = [] data_A = [] data_X = [] data_D = [] data_F = [] data_Le = [] data_Lv = [] max_length = max(mol.GetNumAtoms() for mol in self.data) max_length_s = max(len(Chem.MolToSmiles(mol)) for mol in self.data) for i, mol in enumerate(self.data): A = self._genA(mol, connected=True, max_length=max_length) D = np.count_nonzero(A, -1) if A is not None: data.append(mol) smiles.append(Chem.MolToSmiles(mol)) data_S.append(self._genS(mol, max_length=max_length_s)) data_A.append(A) data_X.append(self._genX(mol, max_length=max_length)) data_D.append(D) data_F.append(self._genF(mol, max_length=max_length)) L = np.diag(D) - A Le, Lv = np.linalg.eigh(L) data_Le.append(Le) data_Lv.append(Lv) pr.update(i + 1) self.log(date=False) self.log( 'Created {} features and adjacency matrices out of {} molecules!'. format(len(data), len(self.data))) self.data = data self.smiles = smiles self.data_S = data_S self.data_A = data_A self.data_X = data_X self.data_D = data_D self.data_F = data_F self.data_Le = data_Le self.data_Lv = data_Lv self.__len = len(self.data)
def process_frame_segments(args, segments, width, height): """Post-process frame segments to set frame images, etc.""" fn = "process_frame_segments" globals.log.info("Processing frames...") frame_segments = [s for s in segments if isinstance(s, FrameSegment)] n = len(frame_segments) globals.log.debug("{fn}(): num frames = {n}".format(fn=fn, n=n)) progress = ProgressBar(max_value=n, quiet=args.quiet or args.debug or n == 0) progress.update(0) for i, f in enumerate(frame_segments): try: globals.log.debug( "{fn}(): frame (before) = {b}".format(fn=fn, b=f)) # Frame segments that use a frame from the previous segment. if (f.input_file == "^"): if (f.segment_number > 0): prev = segments[f.segment_number - 1] globals.log.debug( "{fn}(): prev = {p}".format(fn=fn, p=prev)) prev.generate_temp_file(args.output, width=width, height=height) f.use_frame( prev.generate_frame(f.frame_number, args.output, width=width, height=height)) else: globals.log.error( "frame segment {s} is attempting to use the last " "frame of a non-existent previous " "segment".format(s=f.segment_number)) sys.exit(1) # Frame segments whose frame comes from a PDF file. else: suffix = PurePath(f.input_file).suffix if (suffix.lower() == ".pdf"): f.use_frame(f.generate_temp_file(args.output, width=width, height=height)) else: globals.log.error( 'unexpected input file type "{s}" for frame segment ' "{f}".format(s=suffix, f=f.segment_number)) sys.exit(1) progress.update(i) globals.log.debug("{fn}(): frame (after) = ""{a}".format(fn=fn, a=f)) except SegmentError as e: progress.finish() globals.log.exception(e) sys.exit(1) else: progress.finish()
def runSimulation(self): abort = False toolbar_width = 40 pb = ProgressBar(toolbar_width=toolbar_width) for k, demand in enumerate(self.demandList): if k % round((self.NUM_CALLS / 40)) == 0: # print(round(k * 100 / self.NUM_CALLS)) pb.update(k, len(self.demandList)) if demand['skill'] in self.exhaustedSkills: continue assigned = self.findAvailableResource(demand) if assigned == 'FAIL' or assigned == 'FAIL_EXHAUSTED': if demand['skill'] not in self.exhaustedSkills and assigned == 'FAIL_EXHAUSTED': self.exhaustedSkills.append(demand['skill']) # print('No employees available beginning at interval: ' + str(demand['interval']) + ' for skill: ' + demand['skill']) # turned off due to max interval code interfering with this logic if len(self.exhaustedSkills) == self.NUM_SKILLS: pb.update(1, 1) print(f'\nAll skills exhausted. Simulated {k} out of {self.NUM_CALLS} demand events.') abort = True # switch if abort: pb.update(1, 1) break pb.update(k + 1, len(self.demandList)) pb.clean()
def _generate_AX(self): self.log('Creating features and adjacency matrices..') pr = ProgressBar(60, len(self.data)) data_ax = [] smiles = [] data_s = [] data_a = [] data_x = [] data_d = [] data_f = [] data_le = [] data_lv = [] max_length = max(mol.GetNumAtoms() for mol in self.data) max_length_s = max(len(Chem.MolToSmiles(mol)) for mol in self.data) for i, mol in enumerate(self.data): a = self._genA(mol, connected=True, max_length=max_length) d = np.count_nonzero(a, -1) if a is not None: data_ax.append(mol) smiles.append(Chem.MolToSmiles(mol)) data_s.append(self._genS(mol, max_length=max_length_s)) data_a.append(a) data_x.append(self._genX(mol, max_length=max_length)) data_d.append(d) data_f.append(self._genF(mol, max_length=max_length)) le, lv = np.linalg.eigh(d - a) data_le.append(le) data_lv.append(lv) pr.update(i + 1) self.log(date=False) self.log('Created {} features and adjacency matrices out of {} molecules!'.format(len(data_ax), len(self.data))) self.data = data_ax self.smiles = smiles self.data_S = data_s self.data_A = data_a self.data_X = data_x self.data_D = data_d self.data_F = data_f self.data_Le = data_le self.data_Lv = data_lv self.__len = len(self.data)
def predict(self, steps=365, freq="D"): print("Forecasting...") progress_bar = ProgressBar(len(self.models.items())) for item, model in self.models.items(): future = model.make_future_dataframe(steps, freq=freq) pred = model.predict(future).set_index("ds") pred = pred[["yhat", "yhat_lower", "yhat_upper"]] self.fcst[item] = pred if self.use_boxcox: self.fcst[item] = inv_boxcox(self.fcst[item], self.lmbda_boxcox[item]) progress_bar.update() progress_bar.finish() fcst_df = pd.concat(self.fcst, axis=1).sort_index(axis=1) return fcst_df
def process_video(self, video, quiet=False): """ Process a video taken from a Receiver Parameters: video quiet: if True, no progress bar is displayed """ processed = [] progress_bar = ProgressBar('Processing video', 0, len(video) - 1, quiet) for i, image in enumerate(video): processed.append(self.process_image(image)) progress_bar.update(i) progress_bar.end() return processed
def createResources(self): def determineStartingInterval(): start_interval = P_constrain(np.random.normal(), -3.0, 3.0) start_interval = P_map(start_interval, -3.0, 3.0, 0, 3.999) start_interval = int(start_interval * 60 * 60) return start_interval def determineSchedule(start_interval): # Build out the employee's schedule # Prefill 0's from the beginning of the day until the start of this employee's shift temp_sch = [0 for x in range(start_interval)] # Fill in 1's for the duration of this employee's shift temp_sch.extend([1 for x in range(self.SHIFT_LENGTH)]) # Fill in 0's from the end of this employee's shift until the end of the day temp_sch.extend([0 for x in range(self.NUM_INTERVALS - self.SHIFT_LENGTH - start_interval)]) return temp_sch def determineSkillAssignments(): SKILLS = ['LEVEL_1', 'LEVEL_2', 'LEVEL_3'] skill = [random.choice(SKILLS)] if random.randint(0, 100) <= 30: skill.append(random.choice([x_ for x_ in list(set(SKILLS) - set(skill))])) return skill print('Creating employee list...') toolbar_width = 40 pb = ProgressBar(toolbar_width=toolbar_width) for x_ in range(self.NUM_RESOURCES): pb.update(x_, self.NUM_RESOURCES) # Select the start time for this employee's shift start_interval = determineStartingInterval() # Build out the employee's schedule schedule = determineSchedule(start_interval) utilization = [0 for x in range(self.NUM_INTERVALS)] skillset = determineSkillAssignments() self.addResource({'id': x_ + 10, 'schedule': schedule, 'utilization': utilization, 'skills': skillset}) pb.update(1, 1) pb.clean()
def fit(self, train_df): print("Fitting...") progress_bar = ProgressBar(len(train_df.columns)) for item in train_df.columns: target = train_df[item].dropna() if self.use_boxcox: idx = target.index target, self.lmbda_boxcox[item] = boxcox(target) target = pd.Series(target, index=idx) target.index.name = "ds" target.name = "y" target = target.reset_index() self.models[item] = Prophet(**self.prophet_config) self.models[item].fit(target) progress_bar.update() progress_bar.finish() return self.models
def predict(self, steps=365): print("Forecasting...") progress_bar = ProgressBar(len(self.models.items())) for item, model in self.models.items(): pred = model.predict( exogenous=fourier( steps, seasonality=self.seasonality, n_terms=self.n_fourier_terms), n_periods=steps, return_conf_int=True, alpha=(1.0 - self.interval_width)) fcst = pd.DataFrame() fcst["yhat_lower"] = pred[1][:,0] fcst["yhat"] = pred[0] fcst["yhat_upper"] = pred[1][:,1] self.fcst[item] = fcst if self.use_boxcox: self.fcst[item] = inv_boxcox( self.fcst[item], self.lmbda_boxcox[item]) progress_bar.update() progress_bar.finish() return pd.concat(self.fcst, axis=1)
def run_deep_dream_simple(self, img, steps = 100, learning_rate = 1.0, update_frequency = 5): updates = [] progress_bar = ProgressBar(steps) progress_bar.start() steps_remaining = steps steps_done = 0 while steps_remaining: if steps_remaining>update_frequency: run_steps = tf.constant(update_frequency) else: run_steps = tf.constant(steps_remaining) steps_remaining -= run_steps steps_done += run_steps img = self(img, run_steps, tf.constant(learning_rate)) updates.append(img.numpy()) progress_bar.update(steps_done) return img, updates
def general_image_folder(opt): """Create lmdb for general image folders Users should define the keys, such as: '0321_s035' for DIV2K sub-images If all the images have the same resolution, it will only store one copy of resolution info. Otherwise, it will store every resolution info. """ #### configurations read_all_imgs = False # whether real all images to memory with multiprocessing # Set False for use limited memory BATCH = 5000 # After BATCH images, lmdb commits, if read_all_imgs = False n_thread = 40 ######################################################## img_folder = opt['img_folder'] lmdb_save_path = opt['lmdb_save_path'] meta_info = {'name': opt['name']} if not lmdb_save_path.endswith('.lmdb'): raise ValueError("lmdb_save_path must end with \'lmdb\'.") if osp.exists(lmdb_save_path): print('Folder [{:s}] already exists. Exit...'.format(lmdb_save_path)) sys.exit(1) #### read all the image paths to a list print('Reading image path list ...') all_img_list = sorted(glob.glob(osp.join(img_folder, '*'))) keys = [] for img_path in all_img_list: keys.append(osp.splitext(osp.basename(img_path))[0]) if read_all_imgs: #### read all images to memory (multiprocessing) dataset = { } # store all image data. list cannot keep the order, use dict print('Read images with multiprocessing, #thread: {} ...'.format( n_thread)) pbar = ProgressBar(len(all_img_list)) def mycallback(arg): '''get the image data and update pbar''' key = arg[0] dataset[key] = arg[1] pbar.update('Reading {}'.format(key)) pool = Pool(n_thread) for path, key in zip(all_img_list, keys): pool.apply_async(read_image_worker, args=(path, key), callback=mycallback) pool.close() pool.join() print('Finish reading {} images.\nWrite lmdb...'.format( len(all_img_list))) #### create lmdb environment data_size_per_img = cv2.imread(all_img_list[0], cv2.IMREAD_UNCHANGED).nbytes print('data size per image is: ', data_size_per_img) data_size = data_size_per_img * len(all_img_list) env = lmdb.open(lmdb_save_path, map_size=data_size * 10) #### write data to lmdb pbar = ProgressBar(len(all_img_list)) txn = env.begin(write=True) resolutions = [] for idx, (path, key) in enumerate(zip(all_img_list, keys)): pbar.update('Write {}'.format(key)) key_byte = key.encode('ascii') data = dataset[key] if read_all_imgs else cv2.imread( path, cv2.IMREAD_UNCHANGED) if data.ndim == 2: H, W = data.shape C = 1 else: H, W, C = data.shape txn.put(key_byte, data) resolutions.append('{:d}_{:d}_{:d}'.format(C, H, W)) if not read_all_imgs and idx % BATCH == 0: txn.commit() txn = env.begin(write=True) txn.commit() env.close() print('Finish writing lmdb.') #### create meta information # check whether all the images are the same size assert len(keys) == len(resolutions) if len(set(resolutions)) <= 1: meta_info['resolution'] = [resolutions[0]] meta_info['keys'] = keys print('All images have the same resolution. Simplify the meta info.') else: meta_info['resolution'] = resolutions meta_info['keys'] = keys print( 'Not all images have the same resolution. Save meta info for each image.' ) pickle.dump(meta_info, open(osp.join(lmdb_save_path, 'meta_info.pkl'), "wb")) print('Finish creating lmdb meta info.')
def REDS(mode): """Create lmdb for the REDS dataset, each image with a fixed size GT: [3, 720, 1280], key: 000_00000000 LR: [3, 180, 320], key: 000_00000000 key: 000_00000000 flow: downsampled flow: [3, 360, 320], keys: 000_00000005_[p2, p1, n1, n2] Each flow is calculated with the GT images by PWCNet and then downsampled by 1/4 Flow map is quantized by mmcv and saved in png format """ #### configurations read_all_imgs = False # whether real all images to memory with multiprocessing # Set False for use limited memory BATCH = 5000 # After BATCH images, lmdb commits, if read_all_imgs = False if mode == 'train_sharp': img_folder = '../REDS/train_sharp' lmdb_save_path = '../REDS/train_sharp_wval.lmdb' H_dst, W_dst = 720, 1280 elif mode == 'train_sharp_bicubic': img_folder = '../REDS/train_sharp_bicubic' lmdb_save_path = '../REDS/train_sharp_bicubic_wval.lmdb' H_dst, W_dst = 180, 320 elif mode == 'train_blur_bicubic': img_folder = '../REDS/train_blur_bicubic' lmdb_save_path = '../REDS/train_blur_bicubic_wval.lmdb' H_dst, W_dst = 180, 320 elif mode == 'train_blur': img_folder = '../REDS/train_blur' lmdb_save_path = '../REDS/train_blur_wval.lmdb' H_dst, W_dst = 720, 1280 elif mode == 'train_blur_comp': img_folder = '../REDS/train_blur_comp' lmdb_save_path = '../REDS/train_blur_comp_wval.lmdb' H_dst, W_dst = 720, 1280 elif mode == 'train_sharp_flowx4': img_folder = '../REDS/train_sharp_flowx4' lmdb_save_path = '../REDS/train_sharp_flowx4.lmdb' H_dst, W_dst = 360, 320 n_thread = 40 ######################################################## if not lmdb_save_path.endswith('.lmdb'): raise ValueError("lmdb_save_path must end with \'lmdb\'.") if osp.exists(lmdb_save_path): print('Folder [{:s}] already exists. Exit...'.format(lmdb_save_path)) sys.exit(1) #### read all the image paths to a list print('Reading image path list ...') all_img_list = data_util._get_paths_from_images(img_folder) keys = [] for img_path in all_img_list: split_rlt = img_path.split('/') folder = split_rlt[-2] img_name = split_rlt[-1].split('.png')[0] keys.append(folder + '_' + img_name) if read_all_imgs: #### read all images to memory (multiprocessing) dataset = { } # store all image data. list cannot keep the order, use dict print('Read images with multiprocessing, #thread: {} ...'.format( n_thread)) pbar = ProgressBar(len(all_img_list)) def mycallback(arg): '''get the image data and update pbar''' key = arg[0] dataset[key] = arg[1] pbar.update('Reading {}'.format(key)) pool = Pool(n_thread) for path, key in zip(all_img_list, keys): pool.apply_async(read_image_worker, args=(path, key), callback=mycallback) pool.close() pool.join() print('Finish reading {} images.\nWrite lmdb...'.format( len(all_img_list))) #### create lmdb environment data_size_per_img = cv2.imread(all_img_list[0], cv2.IMREAD_UNCHANGED).nbytes print('data size per image is: ', data_size_per_img) data_size = data_size_per_img * len(all_img_list) env = lmdb.open(lmdb_save_path, map_size=data_size * 10) #### write data to lmdb pbar = ProgressBar(len(all_img_list)) txn = env.begin(write=True) for idx, (path, key) in enumerate(zip(all_img_list, keys)): pbar.update('Write {}'.format(key)) key_byte = key.encode('ascii') data = dataset[key] if read_all_imgs else cv2.imread( path, cv2.IMREAD_UNCHANGED) if 'flow' in mode: H, W = data.shape assert H == H_dst and W == W_dst, 'different shape.' else: H, W, C = data.shape assert H == H_dst and W == W_dst and C == 3, 'different shape.' txn.put(key_byte, data) if not read_all_imgs and idx % BATCH == 0: txn.commit() txn = env.begin(write=True) txn.commit() env.close() print('Finish writing lmdb.') #### create meta information meta_info = {} meta_info['name'] = 'REDS_{}_wval'.format(mode) channel = 1 if 'flow' in mode else 3 meta_info['resolution'] = '{}_{}_{}'.format(channel, H_dst, W_dst) meta_info['keys'] = keys pickle.dump(meta_info, open(osp.join(lmdb_save_path, 'meta_info.pkl'), "wb")) print('Finish creating lmdb meta info.')
def vimeo90k(mode): """Create lmdb for the Vimeo90K dataset, each image with a fixed size GT: [3, 256, 448] Now only need the 4th frame, e.g., 00001_0001_4 LR: [3, 64, 112] 1st - 7th frames, e.g., 00001_0001_1, ..., 00001_0001_7 key: Use the folder and subfolder names, w/o the frame index, e.g., 00001_0001 flow: downsampled flow: [3, 360, 320], keys: 00001_0001_4_[p3, p2, p1, n1, n2, n3] Each flow is calculated with GT images by PWCNet and then downsampled by 1/4 Flow map is quantized by mmcv and saved in png format """ #### configurations read_all_imgs = False # whether real all images to memory with multiprocessing # Set False for use limited memory BATCH = 5000 # After BATCH images, lmdb commits, if read_all_imgs = False if mode == 'GT': img_folder = '../vimeo90k/vimeo_septuplet/sequences' lmdb_save_path = '../vimeo90k/vimeo90k_train_GT.lmdb' txt_file = '../vimeo90k/vimeo_septuplet/sep_trainlist.txt' H_dst, W_dst = 256, 448 elif mode == 'LR': img_folder = '../vimeo90k/vimeo_septuplet_matlabLRx4/sequences' lmdb_save_path = '../vimeo90k/vimeo90k_train_LR7frames.lmdb' txt_file = '../vimeo90k/vimeo_septuplet/sep_trainlist.txt' H_dst, W_dst = 64, 112 elif mode == 'flow': img_folder = '../vimeo90k/vimeo_septuplet/sequences_flowx4' lmdb_save_path = '../vimeo90k/vimeo90k_train_flowx4.lmdb' txt_file = '../vimeo90k/vimeo_septuplet/sep_trainlist.txt' H_dst, W_dst = 128, 112 else: raise ValueError('Wrong dataset mode: {}'.format(mode)) n_thread = 40 ######################################################## if not lmdb_save_path.endswith('.lmdb'): raise ValueError("lmdb_save_path must end with \'lmdb\'.") if osp.exists(lmdb_save_path): print('Folder [{:s}] already exists. Exit...'.format(lmdb_save_path)) sys.exit(1) #### read all the image paths to a list print('Reading image path list ...') with open(txt_file) as f: train_l = f.readlines() train_l = [v.strip() for v in train_l] all_img_list = [] keys = [] for line in train_l: folder = line.split('/')[0] sub_folder = line.split('/')[1] all_img_list.extend( glob.glob(osp.join(img_folder, folder, sub_folder, '*'))) if mode == 'flow': for j in range(1, 4): keys.append('{}_{}_4_n{}'.format(folder, sub_folder, j)) keys.append('{}_{}_4_p{}'.format(folder, sub_folder, j)) else: for j in range(7): keys.append('{}_{}_{}'.format(folder, sub_folder, j + 1)) all_img_list = sorted(all_img_list) keys = sorted(keys) if mode == 'GT': # only read the 4th frame for the GT mode print('Only keep the 4th frame.') all_img_list = [v for v in all_img_list if v.endswith('im4.png')] keys = [v for v in keys if v.endswith('_4')] if read_all_imgs: #### read all images to memory (multiprocessing) dataset = { } # store all image data. list cannot keep the order, use dict print('Read images with multiprocessing, #thread: {} ...'.format( n_thread)) pbar = ProgressBar(len(all_img_list)) def mycallback(arg): """get the image data and update pbar""" key = arg[0] dataset[key] = arg[1] pbar.update('Reading {}'.format(key)) pool = Pool(n_thread) for path, key in zip(all_img_list, keys): pool.apply_async(read_image_worker, args=(path, key), callback=mycallback) pool.close() pool.join() print('Finish reading {} images.\nWrite lmdb...'.format( len(all_img_list))) #### write data to lmdb data_size_per_img = cv2.imread(all_img_list[0], cv2.IMREAD_UNCHANGED).nbytes print('data size per image is: ', data_size_per_img) data_size = data_size_per_img * len(all_img_list) env = lmdb.open(lmdb_save_path, map_size=data_size * 10) txn = env.begin(write=True) pbar = ProgressBar(len(all_img_list)) for idx, (path, key) in enumerate(zip(all_img_list, keys)): pbar.update('Write {}'.format(key)) key_byte = key.encode('ascii') data = dataset[key] if read_all_imgs else cv2.imread( path, cv2.IMREAD_UNCHANGED) if 'flow' in mode: H, W = data.shape assert H == H_dst and W == W_dst, 'different shape.' else: H, W, C = data.shape assert H == H_dst and W == W_dst and C == 3, 'different shape.' txn.put(key_byte, data) if not read_all_imgs and idx % BATCH == 0: txn.commit() txn = env.begin(write=True) txn.commit() env.close() print('Finish writing lmdb.') #### create meta information meta_info = {} if mode == 'GT': meta_info['name'] = 'Vimeo90K_train_GT' elif mode == 'LR': meta_info['name'] = 'Vimeo90K_train_LR' elif mode == 'flow': meta_info['name'] = 'Vimeo90K_train_flowx4' channel = 1 if 'flow' in mode else 3 meta_info['resolution'] = '{}_{}_{}'.format(channel, H_dst, W_dst) key_set = set() for key in keys: if mode == 'flow': a, b, _, _ = key.split('_') else: a, b, _ = key.split('_') key_set.add('{}_{}'.format(a, b)) meta_info['keys'] = list(key_set) pickle.dump(meta_info, open(osp.join(lmdb_save_path, 'meta_info.pkl'), "wb")) print('Finish creating lmdb meta info.')
def sgbminnerblock(sorting_value, basis, expect_basis, expect_brownian_basis, \ riskfree_price, riskfree_delta, adjusted_price, adjusted_delta,\ timespan, bundles): """Perform the SGBM backward algorithm with given forward simulations This algorithm collects all the necessary data from the forward simulation separately and then perform the backward step sequentially with respect to the time discretization, starting from the last time period. Within each time period, first all samples are sorted according to the sorting value and partition them into bundles containing simular sorting value paths. In each bundle, servaral regressions are completed and the resulting coefficients are used to calculate the target value (option prices) within each bundle. Finally, all the results are collected and a new cycle begins, untill we ratch time zero. :param sorting_value: Array of the values of sorting function :param basis: Array of the regression basis for each path at every time step :param expect_basis: Array of the expectation of the regression basis for each path at every time step :param expect_brownian_basis: Array of the expectation of the regression basis multipled by a Brownian motion at each dimension for each path at every time step :param riskfree_price: Array for the riskfree option price for each paths at expiry :param riskfree_delta: Array for the riskfree option hedging for each paths at expiry :param adjusted_price: Array for the risk-adjusted option price for each paths at expiry :param adjusted_delta: Array for the risk-adjusted option hedging for each paths at expiry :param timespan: Array of time partition points :param bundles: Number of bundles in all time step (except the starting one) :type sorting value: N x (M+1) array :type basis: N x Q x M array :type expect_basis: N x Q x M array :type expect_brownian_basis: N x Q x d x M array :type riskfree_price: N x 1 array :type riskfree_delta: N x d array :type adjusted_price: N x 1 array :type adjusted_delta: N x d array :type timespan: M+1 array :type bundles: int :returns: (average riskfree option price, average risk adjusted option price) :rtype: (scaler, scaler) .. todo:: Allowing variation of bundle size between time steps """ num_of_time_points = np.size(timespan) # M+1 num_of_samples = basis.shape[0] # N adaptive_bundles = bundles * np.ones(num_of_time_points, dtype=int) adaptive_bundles[0] = 1 # Calculate the number of bundle for each time step. # (preparation for adaptive number of bundles) num_of_bundles = np.amax(adaptive_bundles) bundle_size = np.zeros((num_of_time_points, num_of_bundles), dtype=int) p_bar = ProgressBar(num_of_time_points - 1, prefix='backward process', suffix='', decimals=2, barLength=100) # The followings are the main backward in time algorithm. for j in range(num_of_time_points - 1): p_bar.update(j) # Calculate the number of paths in each bundle. c_bundles_num = adaptive_bundles[-j - 2] bundle_size[-j - 2, 0:c_bundles_num] = int(num_of_samples / c_bundles_num) bundle_size[-j - 2, 0:num_of_samples - c_bundles_num * int(num_of_samples / c_bundles_num)] += 1 # Sort the prices and deltas based on the order of sorting value order_p = np.argsort(sorting_value[:, -j - 2]) # Collect all the values that we have to perform regression on regression_coeff = np.zeros( (c_bundles_num, NUM_REGRESSION, BASIS_ORDER)) regression_unknown = regression_variable(basis.shape[0], riskfree_price, \ riskfree_delta, adjusted_price, adjusted_delta) # Collect all the basis related values in this time step and put them in order sorted_basis = basis[order_p, :, -j - 1] sorted_expect_basis = expect_basis[order_p, :, -j - 1] sorted_expect_brownian_basis = expect_brownian_basis[order_p, :, :, -j - 1] sorted_regression_unknown = regression_unknown[order_p, :] try: # Distribute the relavent information into different bundle and run the regression bundle_range = np.empty((c_bundles_num, 2), dtype=int) for i in range(c_bundles_num): bundle_range[i] = \ np.array([np.sum(bundle_size[-j-2, 0:i]), np.sum(bundle_size[-j-2, 0:i+1])]) for i in range(c_bundles_num): bundle_regression_unknown = \ sorted_regression_unknown[bundle_range[i, 0]: bundle_range[i, 1]] bundle_basis = sorted_basis[bundle_range[i, 0]:bundle_range[i, 1]] regression_coeff[i] = regressioninbundle(bundle_basis, \ bundle_regression_unknown) except ExceedBoundError: raise ExceedBoundError # Calculate the previous value based on problem specific numerical scheme theta_parameters = np.array( [0, 1]) # Adjustment parameters for the integration approximation delta_t = timespan[-j - 1] - timespan[-j - 2] for i in range(c_bundles_num): bundle_expect_basis = sorted_expect_basis[bundle_range[ i, 0]:bundle_range[i, 1]] bundle_expect_brownian_basis = \ sorted_expect_brownian_basis[bundle_range[i, 0]: bundle_range[i, 1]] riskfree_delta[order_p[bundle_range[i, 0]: bundle_range[i, 1]], :], \ riskfree_price[order_p[bundle_range[i, 0]: bundle_range[i, 1]], :], \ adjusted_delta[order_p[bundle_range[i, 0]: bundle_range[i, 1]], :], \ adjusted_price[order_p[bundle_range[i, 0]: bundle_range[i, 1]], :] = \ numerical_scheme(theta_parameters, delta_t, regression_coeff[i], \ bundle_expect_basis, bundle_expect_brownian_basis) p_bar.update(num_of_time_points - 1) return (np.mean(riskfree_price[:, :]), np.mean(adjusted_price[:, :]))
for i in range(out_W): idx = int(indices_W[i][0]) out_2[:, i, 0] = out_1_aug[:, idx:idx + kernel_width, 0].mv(weights_W[i]) out_2[:, i, 1] = out_1_aug[:, idx:idx + kernel_width, 1].mv(weights_W[i]) out_2[:, i, 2] = out_1_aug[:, idx:idx + kernel_width, 2].mv(weights_W[i]) return out_2.numpy() print('Read images...') pbar = ProgressBar(len(img_list)) for i, v in enumerate(img_list): pbar.update('Read {}'.format(v)) img = cv2.imread(v, cv2.IMREAD_UNCHANGED) if len(img.shape) < 3: img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) dataset.append(img) down_img = imresize_np(img, scale, True) down_dataset.append(down_img) # print(img.shape, down_img.shape) data_size += img.nbytes down_data_size += down_img.nbytes env = lmdb.open(lmdb_save_path, map_size=data_size * 10) down_env = lmdb.open(down_lmdb_save_path, map_size=down_data_size * 10) print('Finish reading {} images.\nWrite lmdb...'.format(len(img_list))) pbar = ProgressBar(len(img_list)) with env.begin(write=True) as txn: # txn is a Transaction object
class FFmpegConcatCommand(FFmpegCommand): """An ffmpeg shell command with a complex concat filter.""" _expect_patterns = [r"time=(\d\d):(\d\d):(\d\d\.\d\d)"] def __init__(self, input_options=[], output_options=[], quiet=False, max_progress=100, has_audio=False, has_video=False, process_audio=True, process_video=True, audio_codec="pcm_s16le", video_codec="h264"): super().__init__(input_options, output_options) self.progress = ProgressBar(max_value=max_progress, quiet=quiet) self.has_video = has_video self.process_video = process_video self.video_codec = video_codec if (self.has_video): self.prepend_output_options(["-map", "[vconc]"]) if (self.process_video): self.prepend_output_options(["-pix_fmt", "yuv420p"]) self.prepend_output_options(["-codec:v", self.video_codec]) self.has_audio = has_audio self.process_audio = process_audio self.audio_codec = audio_codec if (self.has_audio): if (self.process_audio): self.prepend_output_options(["-ac", "1", "-map", "[anorm]"]) else: self.prepend_output_options(["-map", "[aconc]"]) self.prepend_output_options(["-codec:a", self.audio_codec]) self.filters = [] def append_filter(self, filter): """Append a filter to the filters list.""" if (filter): self.filters.append(filter) def append_normalisation_filter(self): """Append a normalisation audio filter to the complex filter.""" if (self.has_audio): self.append_filter("[aconc] dynaudnorm=r=0.25:f=10:b=1 [anorm]") def append_concat_filter(self, frame_type, segments=[]): """Append a concat filter to the filters list""" # Ignore frame segments. if frame_type in ["a", "v"]: if (len(segments) > 1): self.append_filter( "{inspecs} concat=n={n}:v={v}:a={a} [{t}conc]".format( inspecs=" ".join( [s.output_stream_specifier() for s in segments]), n=len(segments), v=int(frame_type == "v"), a=int(frame_type == "a"), t=frame_type)) elif (len(segments) == 1): self.append_filter("{inspec} {a}null [{t}conc]".format( inspec=segments[0].output_stream_specifier(), a=frame_type if frame_type == "a" else "", t=frame_type)) def build_complex_filter(self): """Build the complete complex filter. Filters in the filtergraph are separated by ";". """ return "{f}".format(f=";".join(self.filters)) def argument_string(self, quote=False): """Return the list of arguments as a string.""" args = ( self._base_options + self.input_options + ["-filter_complex", self.build_complex_filter()] + self.output_options) if quote: return " ".join([ShellCommand.shellquote(a) for a in args]) else: return " ".join(args) def argument_list(self): """Return a combined list of all arguments.""" return ( self._base_options + self.input_options + ["-filter_complex", self.build_complex_filter()] + self.output_options) def process_pattern(self, pat): """Respond to a pexpect pattern. Return True on EOF.""" if (pat == 1): elapsed = datetime.timedelta( hours=int(self.process.match.group(1)), minutes=int(self.process.match.group(2)), seconds=float(self.process.match.group(3))) self.progress.update(elapsed.total_seconds()) return (pat == 0)
def printStatistics(self): SERVICE_LEVEL = 20 total_calls = len(self.demandList) missed_calls = 0 delay = 0 total_delay = 0 max_delay = 0 out_svl = 0 unanswered_calls = 0 print('Calculating demand statistics...') for call in self.demandList: if 'answered_interval' not in call: unanswered_calls += 1 missed_calls += 1 out_svl += 1 else: if call['interval'] != call['answered_interval']: missed_calls += 1 delay = call['answered_interval'] - call['interval'] total_delay += delay if delay > max_delay: max_delay = delay if delay > SERVICE_LEVEL: out_svl += 1 print('Calculating resource statistics...') utilization = 0 available = 0 for res in self.resourceList: available += self.SHIFT_LENGTH for moment in res['utilization']: if moment > 1: utilization += 1 res_utilization = round(100 * utilization / available, 2) if missed_calls > 0: avg_delay = round(total_delay / missed_calls, 2) asa = round(total_delay / total_calls, 0) else: avg_delay = 0 asa = 0 unanswered_perc = round(100 * unanswered_calls / total_calls, 2) calc_svl = round(100 - (100 * out_svl / total_calls), 2) delayed_perc = round(100 * missed_calls / total_calls, 2) print(f'Total calls: {total_calls}') print(f'Number of resources: {self.NUM_RESOURCES}') print(f'Service level ({SERVICE_LEVEL}): {calc_svl}%') print(f'Delayed calls: {missed_calls} ({delayed_perc}%)') print(f'Unanswered calls: {unanswered_calls} ({unanswered_perc}%)') print(f'Average seconds to answer: {asa:.0f}') print(f'Average delay: {avg_delay}') print(f'Max delay: {max_delay}') print(f'Resource Utilization: {res_utilization}%') if self.WRITE_TO_FILE: import sqlite3 import csv conn = sqlite3.connect('perfmodel.db') crsr = conn.cursor() crsr.execute("""DROP TABLE IF EXISTS tblDemand;""") conn.commit() crsr.execute("""CREATE TABLE tblDemand (ID BIGINT PRIMARY KEY, INTERVAL BIGINT, DURATION BIGINT, SKILL STRING);""") conn.commit() print('Writing to file...') with open('demand_stats.csv', 'w') as csv_file: writer = csv.DictWriter(csv_file, fieldnames=self.demandList[0].keys(), lineterminator="\n") writer.writeheader() for w_ in self.demandList: writer.writerow(w_) # print(f"""INSERT INTO tblDemand VALUES ({w_['id']}, {w_['interval']}, {w_['duration']},'{w_['skill']}');""") # crsr.execute(f"""INSERT INTO tblDemand VALUES ({w_['id']}, {w_['interval']}, {w_['duration']},'{w_['skill']}');""") conn.commit() crsr.execute("""DROP TABLE IF EXISTS tblResources;""") conn.commit() crsr.execute("""CREATE TABLE tblResources (ID BIGINT, SCHEDULE TINYINT, UTILIZATION BIGINT, SKILLS STRING);""") conn.commit() toolbar_width = 40 pb = ProgressBar(toolbar_width=toolbar_width) with open('resource_stats.csv', 'w') as csv_file: writer = csv.DictWriter(csv_file, fieldnames=['id', 'interval', 'schedule', 'utilization'], lineterminator="\n") writer.writeheader() for k, w_ in enumerate(self.resourceList): # print(f"""INSERT INTO tblResources VALUES ({w_['id']}, '{w_['skills'][0]}');""") # crsr.execute(f"""INSERT INTO tblResources VALUES ({w_['id']}, '{w_['skills'][0]}');""") pb.update(k, len(self.resourceList)) for i_ in range(self.NUM_INTERVALS): row = {'id': w_['id'], 'interval': str(i_), 'schedule': w_['schedule'][i_], 'utilization': w_['utilization'][i_]} # crsr.execute(f"""INSERT INTO tblResources VALUES ({w_['id']}, {w_['schedule'][i_]}, {w_['utilization'][i_]}, '{w_['skills'][0]}');""") writer.writerow(row) pb.update(1, 1) pb.clean() conn.commit() conn.close()