def generate_html(config, output_dir): articles, categories, months, pages = index_blog_structure(config) file_loader = FileSystemLoader( os.path.join('themes', config['theme'], 'templates')) env = Environment(loader=file_loader) env.globals = get_globals(config, articles, categories, months, pages) clear_dir(output_dir) generate_home(config, output_dir, env) generate_archives_index(config, output_dir, env) for category in categories: generate_category_collection(config, output_dir, env, category) for month in months: generate_month_collection(config, output_dir, env, month) for article in articles: generate_article(config, output_dir, env, article) for page in pages: generate_page(config, output_dir, env, page) copy_assets(config, output_dir)
def test_convert_mp3(self): dir_convert = join(TEST_DIR_RAW, CONVERTED_DIR) clear_dir(dir_convert) assert not os.listdir(dir_convert), "Dir not empty" convert_mp3_to_wav(TEST_DIR_RAW, TEST_LIST_AUDIO[0]) assert TEST_LIST_AUDIO[0].replace("mp3", "wav") in \ os.listdir(dir_convert), "MP3 conversion failed"
def diagnosis_by_fuzzing_entropy(program, fuzzing_dir, entropy_threshold, ratio_min, ratio_max, fuzzed_files_per_iter=10, stop_iter=500, pre_fuzz_count=0): seedfiles_dir = os.path.join(fuzzing_dir, consts.SEEDFILES) matrix_file = None for granularity in [DLL_GRANULARITY, FUNCTION_GRANULARITY, DOMINATOR_GRANULARITY, XREF_GRANULARITY]: instances_dir = utils.clear_dir(os.path.join(fuzzing_dir, consts.INSTANCES, granularity, str(entropy_threshold))) current_entropy = float('inf') previous_entropy = float('-inf') tracing_data = generate_tracing_data(granularity, matrix_file) matrix_file = os.path.join(fuzzing_dir, consts.FUZZING_MATRIX.format("{0}_{1}".format(granularity, str(entropy_threshold)))) if os.path.exists(matrix_file): os.remove(matrix_file) working_dir = utils.clear_dir(os.path.join(fuzzing_dir, consts.WORKING_DIR, granularity, str(entropy_threshold))) diagnosis_result = os.path.join(fuzzing_dir,consts.DLL_DIAGNOSIS_RESULT if granularity == DLL_GRANULARITY else consts.FUNCTION_DIAGNOSIS_RESULT) for seed_example in utils.get_files_in_dir(seedfiles_dir): shutil.copy2(seed_example, instances_dir) instance_path = os.path.join(instances_dir, os.path.basename(seed_example)) run_debugger_on_files(program, [instance_path], working_dir, config, granularity, tracing_data) fuzzed_files = fuzz_project_dir(seedfiles_dir, instances_dir, pre_fuzz_count, ratio_min, ratio_max) run_debugger_on_files(program, fuzzed_files, working_dir, config, granularity, tracing_data) iter_ind = 0 while abs(current_entropy - previous_entropy) > entropy_threshold: fuzzed_files = fuzz_project_dir(seedfiles_dir, instances_dir, fuzzed_files_per_iter, ratio_min, ratio_max) run_debugger_on_files(program, fuzzed_files, working_dir, config, granularity, tracing_data) diagnoser.campaign_matrix.create_matrix_for_dir(working_dir, diagnosis_result, matrix_file) sfl_matrix = readPlanningFile(matrix_file) sfl_matrix.diagnose() results = Diagnosis_Results(sfl_matrix.diagnoses, sfl_matrix.initial_tests, sfl_matrix.error) previous_entropy = current_entropy current_entropy = results.component_entropy iter_ind = iter_ind + 1 if iter_ind > stop_iter: break
def generate(): posts = utils.get_posts() ppp = config['posts_per_page'] pages = int(math.ceil(float(len(posts)) / ppp)) utils.clear_dir('site/page') for i in range(pages): page_content = render_template('frontend/index.html', config=config, frontend=True, current=i + 1, first=(i == 0), last=(i == pages - 1), posts=posts[i * ppp:(i + 1) * ppp]) file('site/page/%s.html' % (i + 1), 'w').write( page_content.encode(config['encoding'])) if i == 0: file('site/index.html', 'w').write( page_content.encode(config['encoding'])) not_found_content = render_template('404.html', config=config, frontend=True) file('site/404.html', 'w').write( not_found_content.encode(config['encoding'])) utils.clear_dir('site/posts') infos = utils.get_post_infos() feed = AtomFeed(config['title'], feed_url=config['url_root'] + '/posts.atom', url=config['url_root']) for info in infos: with open('posts/%s' % info['filename'], 'r') as f: content = f.read().decode(config['encoding']) title = utils.get_title(content) content = utils.postprocess_post_content(info['slug'], content, False) html_content = render_template('frontend/post.html', config=config, frontend=True, title=title, content=content) file('site/posts/%s.html' % info['slug'], 'w').write( html_content.encode(config['encoding'])) feed_content = render_template('feed.html', config=config, content=content) feed.add(title, feed_content, content_type='html', url=make_external('/posts/' + info['slug']), author='Tony Wang', published=utils.date_localize_from_utc(info['time'], True), updated=utils.date_localize_from_utc(info['time'], True)) file('site/posts.atom', 'w').write(str(feed.get_response().iter_encoded(config['encoding']).next())) return 'Done!'
def pred_pano_labels(pano_id, path_to_gsv_scrapes, GSV_IMAGE_WIDTH, GSV_IMAGE_HEIGHT, model_dir, num_threads=4, save_labeled_pano=True, verbose=False): ''' takes a panorama id and returns a dict of the filtered predictions''' path_to_folder = os.path.join(path_to_gsv_scrapes, pano_id[:2], pano_id) path_to_xml = path_to_folder + ".xml" (GSV_IMAGE_WIDTH, GSV_IMAGE_HEIGHT) = utils.extract_width_and_height(path_to_xml) now = time.time() temp = os.path.join('temp', 'crops') if not os.path.exists(temp): os.makedirs(temp) if not os.path.exists('viz'): os.makedirs('viz') utils.clear_dir(temp) make_sliding_window_crops(pano_id, GSV_IMAGE_WIDTH, GSV_IMAGE_HEIGHT, path_to_gsv_scrapes, num_threads=num_threads, verbose=verbose) model_name = utils.get_model_name() model_path = os.path.join(model_dir, model_name + '.pt') preds = predict_from_crops("temp", model_path, verbose=verbose) preds_loc = write_predictions_for_every_pano(path_to_gsv_scrapes, preds, verbose=verbose) if (len(preds_loc) == 0): return None pred = read_predictions_from_file(preds_loc) pred_dict = non_max_sup(pred, radius=150, clip_val=4.5, ignore_ind=1, verbose=verbose) if save_labeled_pano: pano_root = os.path.join(path_to_gsv_scrapes, pano_id[:2], pano_id) out_img = os.path.join("viz", pano_id + "_viz.jpg") show_predictions_on_image(pano_root, GSV_IMAGE_WIDTH, GSV_IMAGE_HEIGHT, pred_dict, out_img, show_coords=False, show_box=True, verbose=verbose) utils.clear_dir(temp) if verbose: print("{} took {} seconds".format(pano_id, time.time() - now)) return pred_dict
def make_directories(out_path, split, df): tmp_path = os.path.join(out_path, 'tmp') utils.make_dir(tmp_path) utils.clear_dir(tmp_path) split_path = os.path.join(out_path, split) utils.make_dir(split_path) if 'label' in df.columns: label_cat = df.label.astype('category') for label in label_cat.cat.categories: label_path = os.path.join(split_path, label) utils.make_dir(label_path) return tmp_path, split_path
def set_workdir(self, workdir): self.workdir = workdir self.log_dir = os.path.join(self.workdir, "LOG_DIR") utils.clear_dir(self.log_dir) if not os.path.isdir(self.log_dir): os.mkdir(self.log_dir) try: self.log_file = open(os.path.join(self.log_dir, "log.txt"), "w") except Exception as e: print(e, file=sys.stderr) exit(-1)
def clear_save_name(self): """ Clears all saved content for SAVE_NAME. """ utils.clear_dir(self.model_save_dir) utils.clear_dir(self.summary_save_dir) utils.clear_dir(self.log_save_dir) utils.clear_dir(self.image_save_dir) print('Clear stuff in {}'.format(os.path.join(self.save_dir)))
def template_test(): try: for f_name in session['f_names']: app.logger.info( f'******* clear out {app.root_path}/testoutput{f_name} from = {session}' ) clear_dir(out_path=app.root_path, f_name=f_name) except: pass session['my_encounters'] = [] session['f_names'] = [] session['resource_list'] = [] app.logger.info(f'******* sessions = {session}') #cache.get('f_name') #clear upload files if present in cache. #cache.clear() # clear all the cache *TODO switch over to db* my_string = '''This is a simple Flask App FHIR Facade which: For single "real-time" Notifications: 1. Fetches *Admit* and *Discharge* Encoounters from the {ref_server} Reference Server 1. Builds the Da Vinci Notifications Message Bundle 1. Submits the Message to the nominated endpoint using the `$process-message` operation 1. Receives and displays the $process-message operation response from the server For a Batch Transaction of multiple Notification: 1. Fetches all the relevant *Admit* and *Discharge* Encoounters from the {ref_server} Reference Server 1. Builds a transaction Bundle with: 1. the Da Vinci Notifications Message Bundle as entries 1. `POST` for the request method 1. `/$process-message` for the request url 1. Submits the transaction Bundle to the nominated endpoint using the `POST` operation 1. Receives and displays the "transaction-response" response from the server. '''.format(ref_server=ref_server_name) return render_template( 'template.html', ref_server=ref_server_name, enc_list=enc_list, title="Index", )
def download(save_dir, rewrite=False): save_dir = utils.get_dir() if rewrite: utils.clear_dir(save_dir) filename = DATA_URL.split(os.pathsep)[-1] filepath = os.path.join(save_dir, filename) if not os.path.isfile(filepath): def _progress(count, block_size, total_size): sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename, float(count * block_size) / float(total_size) * 100.0)) sys.stdout.flush() filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress) print() statinfo = os.stat(filepath) print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') tarfile.open(filepath, 'r:gz').extractall(save_dir)
def batch_save_pano_labels(pano_ids, path_to_gsv_scrapes, model_dir, num_threads=4, verbose=False): ''' takes a panorama id and returns a dict of the filtered predictions''' start = time.time() crops = os.path.join('temp', 'crops') if not os.path.exists(crops): os.makedirs(crops) if not os.path.exists('viz'): os.makedirs('viz') utils.clear_dir(crops) for pano_id in pano_ids: now = time.time() pano_root = os.path.join(path_to_gsv_scrapes, pano_id[:2], pano_id) path_to_xml = pano_root + ".xml" (GSV_IMAGE_WIDTH, GSV_IMAGE_HEIGHT) = utils.extract_width_and_height(path_to_xml) make_sliding_window_crops(pano_id, GSV_IMAGE_WIDTH, GSV_IMAGE_HEIGHT, path_to_gsv_scrapes, num_threads=num_threads, verbose=verbose) model_name = "20ep_sw_re18_2ff2" model_path = os.path.join(model_dir, model_name + '.pt') preds = predict_from_crops("temp", model_path, verbose=verbose) preds_loc = write_predictions_for_every_pano(preds, path_to_gsv_scrapes + pano_id[:2], "labels.csv", verbose=verbose) utils.clear_dir(crops) print("{} took {} seconds".format(pano_id, time.time() - now)) print("total time: {} seconds".format(time.time() - start))
def clear_save_name(): """ Clears all saved content for SAVE_NAME. """ utils.clear_dir(MODEL_SAVE_DIR) utils.clear_dir(SUMMARY_SAVE_DIR) utils.clear_dir(IMG_SAVE_DIR) print('Clear stuff in {}'.format(os.path.join(SAVE_DIR, SAVE_NAME)))
def generate_data(input_data, date_after, path_to_panos, ignore_null, number_agree, path_to_summary, verbose, num_threads): crops = os.path.join("single", "crops") if not os.path.exists(crops): os.makedirs(crops) utils.clear_dir(crops) existing_labels = labels_already_made(path_to_panos) add_to_summary = {} dict_valid = read_validation_data(input_data, date_after, existing_labels, add_to_summary, number_agree, verbose) dict_image = generate_image_date(dict_valid, existing_labels, verbose) make_crop_threading(dict_image, path_to_panos, verbose, num_threads) get_results(verbose) rows_dict = exact_labels(ignore_null) labels_list = generate_labelrows(dict_valid) new_labels = write_summary_file(rows_dict, labels_list, add_to_summary, path_to_summary) if (verbose): print("Number of new labels is " + str(len(new_labels))) update_labels_already_made(new_labels, path_to_panos) utils.clear_dir(crops) if os.path.exists(path_to_completelabels): os.remove(path_to_completelabels)
def __init__(self): git(['checkout', 'master']) git(['commit', '--allow-empty', '-m', 'octoblog-publish notification']) git(['checkout', 'dev']) print('---> generating blog to www/ directory') clear_dir('www') config = json.load(open('config.json', 'r')) generate_html(config, 'www') print('---> done') SaveCommand() print('---> pushing contents of www/ to master') git(['checkout', 'master']) clear_dir(os.getcwd()) git(['checkout', 'dev', '--', 'www']) flatten('www') git(['add', '.']) git(['commit', '-m', '"octoblog-publish files"']) git(['push', 'origin', 'master']) git(['checkout', 'dev']) print('---> done')
def ReduceSingleThread(content_file): print u'创建倒排索引...' worker = ReduceWorker() reader = IOWorker([content_file], 20000) doc_list = [] count = 0 save_count = 0 start_time = time.clock() clear_dir('tmp') while 1: task_raw_data = reader.GetLines() data_count = len(task_raw_data) if data_count == 0: break doc_list = Converter.ToDocList(task_raw_data, count) count += data_count term_inverter_list = worker.run(doc_list) term_inverter_string = Converter.TermInverterToString( term_inverter_list) save_count += 1 IOWorker.SaveText('tmp\\%d' % save_count, term_inverter_string) print_speed(start_time, count) print u'保存完毕,共%d个临时文件' % save_count
def main(argv): FLAGS = gflags.FLAGS gflags.DEFINE_string('srcfile', 'json/sports1m_test.json', 'jason file to read from[json/sports1m_test.json]') gflags.DEFINE_string('savedir', 'tmp', 'dstination directory to save files[tmp]') gflags.DEFINE_string('video_format', 'mp4', 'video format[mp4]') gflags.DEFINE_string('video_resolution', '360p', 'video resolution 360p, 720p and so on[360p]') gflags.DEFINE_integer('min_len', 4, 'Minimal length(4 seconds)') gflags.DEFINE_integer('max_len', 30, 'Maximal length (30 seconds)') gflags.DEFINE_string( 'dstfile', None, 'dstination json file[json/sports1m_test_(duration).json]') gflags.DEFINE_boolean('rewrite', True, 'rewrite everything in saved dir[True]') argv = FLAGS(argv) srcfile = FLAGS.srcfile dstfile = FLAGS.dstfile savedir = os.path.abspath(FLAGS.savedir) savedir = utils.get_dir(savedir) if FLAGS.rewrite: utils.clear_dir(savedir) raw_json_data = read_json(srcfile) if not dstfile: srcname, srcext = os.path.splitext(srcfile) dstfile = '{:s}-{:02d}-{:02d}{:s}'.format(srcname, FLAGS.min_len, FLAGS.max_len, '.txt') download(raw_json_data, save_dir=savedir, dstfile=dstfile, max_len=FLAGS.max_len, min_len=FLAGS.min_len)
def SlowMovie(vid_in_path, slow_factor=2, continuous_fine_tuning=False, tmp_dir='./slowed_movie_frames/', cpu_max_size=320): if np.log2(slow_factor) % 1 != 0: raise ValueError("Slow factor must be a power of 2!") def write_frame(frame, frame_no): out_path = os.path.join(tmp_dir, '{0:09d}.bmp'.format(frame_no)) assert (not os.path.exists(out_path)) cv2.imwrite(out_path, frame) def recursive_predict_and_write(f0, f2, frame0_no, frame_diff): #Interpolate the middle frame with torch.no_grad(): f1 = net(f0, f2)['output_im'] offset = frame_diff // 2 #Recursively predict the extra frames if frame_diff > 2: recursive_predict_and_write(f0, f1, frame0_no, offset) recursive_predict_and_write(f1, f2, frame0_no + offset, offset) #Write the predicted frame write_frame(fh.tensor_to_numpy_bgr(f1[0]), frame0_no + offset) cap = cv2.VideoCapture(vid_in_path) if cap is None or not cap.isOpened(): raise RuntimeError('Unable to open video: ', os.path.abspath(vid_in_path)) vid_dir, vid_in_name = os.path.split(vid_in_path) vid_out_name = os.path.splitext(vid_in_name)[0] + '_{0}x_slow.mp4'.format( slow_factor) vid_out_path = vid_dir + vid_out_name assert (not os.path.exists(vid_out_path)) #Total number of frames n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) real_frames = Queue() if not os.path.exists(tmp_dir): os.mkdir(tmp_dir) utils.clear_dir(tmp_dir) fh = utils.FrameHandler(None) def compute_inbetween_frames(real_frames, idx): frame0_no, frame0 = real_frames.queue[idx] frame1_no, frame1 = real_frames.queue[idx + 1] #Convert the frames to tensors with minibatch size 1 frame0 = default_collate([frame0]) frame1 = default_collate([frame1]) recursive_predict_and_write(frame0, frame1, frame0_no * slow_factor, slow_factor) video_long_edge = max(cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) if not torch.cuda.is_available() and video_long_edge > cpu_max_size: print( "Because we are using CPU, all images will be resized to {0} on the long edge for speed." .format(cpu_max_size)) scale = cpu_max_size / video_long_edge for frame_num in tqdm(range(n_frames)): ret, frame = cap.read() if not torch.cuda.is_available() and video_long_edge > cpu_max_size: frame = cv2.resize(frame, (0, 0), fx=scale, fy=scale) assert (ret) write_frame(frame, frame_num * slow_factor) real_frames.put((frame_num, fh.bgr_to_tensor(frame))) if real_frames.qsize() == 4: #We have four real frames in the queue, which lets us finetune if we want if continuous_fine_tuning: net.cft.finetune_4(real_frames) #Compute the intermediate frames between the two middle frames in the queue compute_inbetween_frames(real_frames, 1) #In the very beginning and end of our video, we need to predict the first and last frame pair, respectively if real_frames.queue[0][0] == 0: compute_inbetween_frames(real_frames, 0) if real_frames.queue[3][0] == n_frames - 1: compute_inbetween_frames(real_frames, 2) real_frames.get() fps = cap.get(cv2.CAP_PROP_FPS) ffmpeg_command = 'ffmpeg -f image2 -r {fps} -i "{tmp}%09d.bmp" -c h264 -crf 17 -y "{vid_out_path}"'.format( fps=fps, tmp=tmp_dir, vid_out_path=vid_out_path) print('Running', ffmpeg_command) print( "if it doesn't generate a movie you probably don't have ffmpeg installed." ) os.system(ffmpeg_command)
def analysis_pipeline(): Config.prepare_env() # 1. find relevant docs x = input("Are you sure you want to [find relevant docs]? (y/n)") logger.debug("Step 1. user input: {}".format(x)) if x == "y" or x == "Y": logger.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>") logger.info("Step 1. [find relevant docs]") data = utils.load_csv(Config.corpus_file_csv, "title", "content")[:5000] logger.info("csv row num: {:,}".format(len(data))) rel_doc = RelevantDoc(data) rel_doc.find_relevant_texts( output_file="output/relevant/relevant_docs.csv") # 2. cluster docs x = input("Are you sure you want to [cluster documents]? (y/n)") logger.debug("user input: {}".format(x)) if x == "y" or x == "Y": alg = input( "Which cluster method you want to use a:[kmeans], b:[dbscan] ? (a/b)" ) logger.debug("Step 2. user input: {}".format(alg)) alg_dic = {"a": "kmeans", "b": "dbscan"} if alg in alg_dic: alg = alg_dic[alg] else: raise ValueError("Invalid cluster method choice: {}".format(alg)) logger.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>") logger.info("Step 2. [cluster docs]") data = utils.load_csv("output/relevant/relevant_docs.csv", "score", "text") utils.clear_dir(Config.cluster_dir) cluster = Cluster(data) cluster.cluster_docs(alg) # 3. text summarization x = input("Are you sure you want to [summarize documents]? (y/n)") logger.debug("Step 3. user input: {}".format(x)) if x == "y" or x == "Y": logger.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>") logger.info("Step 3. [summary docs]") x = input( "Use a:[original documents] or b:[cutted doc pieces (shorter)]? (a/b)" ) utils.clear_dir(Config.summary_dir) root = Config.cluster_dir des_dir = Config.summary_dir for f in sorted(os.listdir(root)): if os.path.isfile(os.path.join(root, f)): if not f.endswith(".out"): continue src_file = os.path.join(root, f) out_file = os.path.join(des_dir, f[:f.rfind(".out")] + ".sum") summary_model = Summary() if x == "b" or x == "B": summary_model.split_long_docs(src_file) summary_model.summary_docs(src_file=src_file + ".split", output_file=out_file) else: summary_model.summary_docs(src_file=src_file, output_file=out_file) logger.info("summaries for each cluster are saved in dir: {}".format( Config.summary_dir)) logger.info("finish summary all.") Config.release_resources() logger.info("finish pipeline.")
cfg["corpus"]["emb_dim"], cfg["model"]["layers"], ) if __name__ == "__main__": args = parser.parse_args(const.TRAIN_MODE) assert args.config with open(args.config, "r") as file: cfg = yaml.safe_load(file) out = Path(args.out) if out.is_dir(): utils.clear_dir(out) else: out.mkdir(parents=True, exist_ok=True) train = Path(args.train) dev = Path(args.dev) # tensorflow.config.experimental.list_physical_devices('CPU')) # Instanciating word embedding model vlayer = TextVectorization( standardize=None, max_tokens=cfg["corpus"]["voc_len"], output_mode="int", output_sequence_length=cfg["corpus"]["seq_len"], )
def tar_gz(in_file, out_dir, clear=True): if clear: clear_dir(out_dir) sudo_call("tar", ["xaf", path.expanduser(in_file), "-C", path.abspath(out_dir)]) sudo_call("sync")
def test_open_wav(self): dir_convert = join(TEST_DIR_RAW, CONVERTED_DIR) assert os.listdir(dir_convert), "Empty dir" test_wav = join(dir_convert, os.listdir(dir_convert)[0]) read_wav(test_wav) clear_dir(dir_convert)
async def cache_clear(): try: utils.clear_dir('cache') except: pass return RedirectResponse("/cache")
def dispatch(self, obj_response, client_data): if 'loaded_custom_wall' in client_data: self.__custom_img_data = client_data['loaded_custom_wall'] automode_files[ 'wallFilePath'] = self.__on_custom_img_loaded_to_canvas() if 'loaded_custom_mask' in client_data: self.__custom_img_data = client_data['loaded_custom_mask'] automode_files[ 'maskFilePath'] = self.__on_custom_img_loaded_to_canvas() if 'loaded_custom_sticker' in client_data: self.__custom_img_data = client_data['loaded_custom_sticker'] automode_files[ 'stickerFilePath'] = self.__on_custom_img_loaded_to_canvas() if 'loaded_wall_gallery' in client_data: self.__wall_gallery_manager.reload_walls_and_masks_gallery( obj_response) if 'loaded_sticker_gallery' in client_data: self.__sticker_gallery_manager.reload_stickers_gallery( obj_response) if 'loaded_gallery_wall_file' in client_data: self.__wall_gallery_manager.on_input_new_wall_bttn_click( obj_response, client_data['loaded_gallery_wall_file']) if 'loaded_gallery_mask_file' in client_data: self.__wall_gallery_manager.on_input_custom_mask_bttn_click( obj_response, client_data['loaded_gallery_mask_file']) if 'loaded_gallery_sticker_file' in client_data: self.__sticker_gallery_manager.on_input_new_sticker_bttn_click( obj_response, client_data['loaded_gallery_sticker_file']) if 'clicked_gallery_wall_mask' in client_data: self.__wall_gallery_manager.on_wall_gallery_img_click( obj_response, client_data['clicked_gallery_wall_mask']) automode_files[ 'wallFilePath'] = self.__wall_gallery_manager._clicked_img_path if 'clicked_gallery_sticker' in client_data: self.__sticker_gallery_manager.on_sticker_gallery_img_click( obj_response, client_data['clicked_gallery_sticker']) automode_files[ 'stickerFilePath'] = self.__sticker_gallery_manager._clicked_img_path if 'delGalleryImg' in client_data: self.__on_delete_bttn_pressed(client_data['delGalleryImg']) if 'automode_settings' in client_data: automode_settings['sticker_center'] = client_data[ 'automode_settings'][0] automode_settings['repeat_x'] = client_data['automode_settings'][1] automode_settings['repeat_y'] = client_data['automode_settings'][2] automode_settings['opacity'] = client_data['automode_settings'][3] self.__auto_mode_manager.response_processed_image(obj_response) if 'downloaded' in client_data: utils.clear_dir(TMP_FOLDER)
datefmt='%Y-%m-%d %H:%M:%S') if arcpy.CheckExtension("Spatial") == "Available": arcpy.AddMessage("Checking out Spatial") arcpy.CheckOutExtension("Spatial") else: arcpy.AddError("Unable to get spatial analyst extension") arcpy.AddMessage(arcpy.GetMessages(0)) sys.exit(0) arcpy.env.workspace = s.TEMP_DIR arcpy.env.scratchWorkspace = s.TEMP_DIR arcpy.env.overwriteOutput = True utils.set_arc_env(s.ECOCOMMUNITIES_FE) utils.clear_dir(s.TRIAL_DIR) utils.mkdir(os.path.join(s.INPUT_DIR, 'fire')) utils.mkdir(os.path.join(s.INPUT_DIR, 'garden')) utils.mkdir(os.path.join(s.INPUT_DIR, 'pond')) utils.clear_dir(s.TEMP_DIR) utils.mkdir(s.TEMP_DIR) logging.info('creating full extent ecocommunities') ecocommunities_fe = arcpy.Raster(s.ECOCOMMUNITIES_FE) # TODO: Is this obsolete? Should Lenape sites not already be in ecocomm grid? - ask Eric lenape_sites = os.path.join(s.TEMP_DIR, 'lenape_sites.tif') arcpy.PolygonToRaster_conversion(in_features=s.BUFFER_FE, value_field='RASTERVALU', out_rasterdataset=lenape_sites, cellsize=s.CELL_SIZE) ecocommunities_fe = arcpy.sa.Con(
# grads = opt.compute_gradients(loss) # train_op = opt.apply_gradients(grads, global_step=global_step) # train_step = opt.apply_gradients(grads) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_step = tf.train.AdamOptimizer(lr).minimize(loss, global_step=global_step) # train_step = tf.train.MomentumOptimizer(lr, momentum=0.9).minimize(loss, global_step=global_step) # loss = tf.Print(loss, [lr, global_step]) sess = tf.Session() log_dir = 'local/logs' checkpoint_dir = 'local/checkpoints' utils.clear_dir(log_dir) utils.clear_dir(checkpoint_dir) saver = tf.train.Saver() summary_all = tf.summary.merge_all() train_writer = tf.summary.FileWriter(join(log_dir, 'train'), sess.graph) # test_writer = tf.summary.FileWriter() tf.global_variables_initializer().run(session=sess) step = 0 best_iou = 0 best_epoch = 0 exp_start_time = time.time() for epoch in range(1, num_epochs + 1):
logging.basicConfig( filename=s.LOGFILE, # use same log file as initiate_disturbance # filemode='w', # defaults to 'a' for append format='%(asctime)s %(levelname)s: %(message)s', level=logging.DEBUG, datefmt='%Y-%m-%d %H:%M:%S') if arcpy.CheckExtension("Spatial") == "Available": arcpy.AddMessage("Checking out Spatial") arcpy.CheckOutExtension("Spatial") else: arcpy.AddError("Unable to checkout spatial analyst extension") arcpy.AddMessage(arcpy.GetMessages(0)) sys.exit(0) utils.clear_dir(s.TEMP_DIR) arcpy.env.workspace = s.TEMP_DIR scratchdir = os.path.join(s.TEMP_DIR, 'scratch') utils.mkdir(scratchdir) arcpy.env.scratchWorkspace = scratchdir # arcpy.env.workspace = os.path.join(s.TEMP_DIR, 'working.gdb') # arcpy.env.scratchWorkspace = os.path.join(s.TEMP_DIR, 'working.gdb') arcpy.env.overwriteOutput = True utils.set_arc_env(s.ecocommunities) if s.FROMSCRATCH: utils.clear_dir(s.OUTPUT_DIR) utils.mkdir(os.path.join(s.OUTPUT_DIR, 'fire', 'burn_rasters')) utils.mkdir(os.path.join(s.OUTPUT_DIR, 'garden')) utils.mkdir(os.path.join(s.OUTPUT_DIR, 'pond'))
def __init__(self): os.makedirs('preview', exist_ok=True) clear_dir('preview') config = json.load(open('config.json', 'r')) config['url'] = 'file:///' + os.getcwd() + '/preview' generate_html(config, 'preview')