def find_matches(subscriptions, reddit, database): Logger.log('Finding Matches...', Color.GREEN) subreddits = {} matches = [] for subscription in subscriptions: subreds = subscription.data[Subscription.SUBREDDITS] for subreddit in subreds: if subreddit.lower() not in [k.lower() for k in subreddits.keys()]: Logger.log(subreddit.lower(), Color.CYAN) submissions = reddit.get_submissions(subreddit.lower()) temp = [] for sub in submissions: temp.append(sub) subreddits[subreddit.lower()] = temp submissions = subreddits[subreddit.lower()] # submissions = reddit.get_submissions(subreddit) num = 0 for submission in submissions: num += 1 is_match, mismatched_keys = MatchFinder.is_match(subscription, submission) if is_match: already_exists = database.check_if_match_exists(subscription.username, subscription.to_string(), submission.permalink) if not already_exists: matches.append((subscription, submission)) return matches
def _validate_building_data(self, b_dict): """ Ensure a dictionary containing building information is actually valid for updating purposes. The main goal is to validate the presence and format of b_id and/or l_b_id. If no b_id is present but a l_b_id is valid, it is set as current b_id, which ensures the building does not get discarded. Arguments: - b_dict: a dictionary representing a building Return value: True if data is valid, False otherwise """ b_id = b_dict.get("b_id", "") l_b_id = b_dict.get("l_b_id", "") if not Building.is_valid_bid(b_id): if Building.is_valid_bid(l_b_id): Logger.warning( "Invalid building id: \"{}\"".format(b_id), "- legacy id", l_b_id, "will be used instead." ) b_dict["b_id"] = l_b_id else: Logger.error( "Building discarded:", "Invalid building id", b_id, "and no valid legacy id is present" ) return False return True
def prepare_rooms(self, floor_id, rooms): """ Transform a list of rooms in a dictionary indexed by room id. Arguments: - floor_id: a string representing the floor identifier, - rooms: a list of rooms. Returns: a dictionary of rooms. Validate the r_id using Building.is_valid_rid function and discard rooms with invalid id. Create and return a dictionary of validated rooms. """ result = {} discarded_rooms = set() for r in map(self.sanitize_room, rooms): if not Building.is_valid_rid(r["r_id"]): discarded_rooms.add(r["r_id"]) continue if "cat_id" in r: r["cat_id"] = RoomCategory.get_cat_id_by_name(r.get("cat_name", "")) del r["cat_name"] r_id = r["r_id"] del r["r_id"] result[r_id] = r if discarded_rooms: Logger.warning( "Rooms discarded from floor", floor_id, "for having an invalid room id:", ", ".join(discarded_rooms) ) return result
def build_element(link_type, label): # find item which is desired link_text = None if label.find(':') != -1: link_text = label[label.find(':') + 1:] label = label[:label.find(':')] result = Globals.get_url_by_name(label, link_type) item = result[0] item_field = result[1] a = etree.Element('a') a.set('data-href', 'Alink') if item_field: a.text = link_text or item_field.href_name a.set('href', '#{item_field.href_id}'.format(item_field=item_field)) elif item: a.text = link_text or item.href_name a.set('href', '#{item.href_id}'.format(item=item)) else: Logger.instance().warning('Link not found %s %s' % (link_type, label)) return '' a.set('text', a.text) return a
def __val(self): """ Validation function during the train phase. """ self.seg_net.eval() start_time = time.time() for j, data_tuple in enumerate(self.val_loader): # Change the data type. inputs = Variable(data_tuple[0].cuda(async=True), volatile=True) targets = Variable(data_tuple[1].cuda(async=True), volatile=True) # Forward pass. outputs = self.seg_net(inputs) # Compute the loss of the val batch. loss_pixel = self.pixel_loss(outputs, targets) loss = loss_pixel self.val_losses.update(loss.data[0], inputs.size(0)) # Update the vars of the val phase. self.batch_time.update(time.time() - start_time) start_time = time.time() self.module_utilizer.save_net(self.seg_net, self.iters) # Print the log info & reset the states. Log.info( 'Test Time {batch_time.sum:.3f}s, ({batch_time.avg:.3f})\t' 'Loss {loss.avg:.8f}\n'.format( batch_time=self.batch_time, loss=self.val_losses)) self.batch_time.reset() self.val_losses.reset() self.seg_net.train()
class Configuration: def __init__(self, config_file = 'planner.ini'): self.logger = Logger().getLogger("configuration.Configuration") self._config_file = config_file current_directory = os.path.dirname(os.path.abspath(__file__)) self._config_file_path = os.path.join(current_directory, '../etc/' + config_file) self.logger.debug('Initialize Configuration with ' + self._config_file_path) self.config = ConfigParser.ConfigParser() self.config.read(self._config_file_path) def get(self, section,option): if self.config.has_section(section) and self.config.has_option(section,option) : return self.config.get(section, option) else: return None def getDict(self, section,option): '''dict example: {1:'aaa',2:'bbb'}''' value = self.get(section, option) if value is not None: return ast.literal_eval(value) else: return value
class UBSan: def get_name(self): return self.name def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "UBSan" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) self.neg_count = 0 os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="clang", CFLAGS="-g -fsanitize=undefined -fsanitize=integer", LD="clang") self.pipeline.run_bechmark(self, [], 2) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): print len(self.tp_set) return self.tp_set def get_fp_set(self): print len(self.fp_set) return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): if len(stderr) > 0: print(stderr) if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if "runtime error" in (stdout + stderr).lower(): if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(stdout, i, cur_dir, j, "NEG") self.neg_count += 1 def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") self.neg_count += 1 def cleanup(self): self.pipeline.clean_benchmark() self.logger.close_log()
def get_vote_loss(self, key): if key == 'vote_loss': return VoteLoss() else: Log.error('Vote loss: {} is not valid.'.format(key)) exit(1)
def seg_net(self): key = self.configer.get('network', 'model_name') if key == 'erf_net': return ERFNet(self.configer.get('network', 'out_channels')) else: Log.error('Model: {} not valid!'.format(key)) exit(1)
def _run(self, timeout): timeout = min([max_wait_time, timeout]) * self.scale def target(): Logger.instance().info('Running command with time limit {:1.2f} s: {} in {}'.format(timeout, self.args.command, self.args.cwd)) self.process = Popen(self.args.command, stdout=self.out, stderr=self.err, stdin=self.inn, cwd=self.args.cwd) Logger.instance().info('started PID {}'.format(self.process.pid)) self.process.wait() # process itself is not limited but there is global limit Logger.instance().info('Command finished with %d' % self.process.returncode) thread = threading.Thread(target=target) thread.start() thread.join(GlobalTimeout.time_left()) if thread.is_alive(): Logger.instance().info('Terminating process') self.terminated = True self.global_terminate = GlobalTimeout.time_left() < 0 try: self.process.terminate() except Exception as e: print(e) try: self.process.kill() except Exception as e: print(e) thread.join()
def _print_merge_analysis(klass, source, target, building): method_name = "analyse_"+source+"_to_"+target merge_info = getattr(FloorMergeAnalysis, method_name)(building) if not building.get_path(source+".floors"): return if not building.get_path(target+".floors"): return with Logger.info("{} -> {} Merge Analysis".format(source.upper(), target.upper())): for f_id, count, which in merge_info: total_rooms = count["total_rooms"] identified_rooms = data_and_percent(count["identified_rooms"], total_rooms) non_identified_rooms = data_and_percent(count["non_identified_rooms"], total_rooms) message = "Floor {:5} | ".format(f_id) message += "Rooms: {:<4} | ".format(total_rooms) message += "With Id.: {:<12} | ".format(identified_rooms) message += "No Id: {:<12}".format(non_identified_rooms) with Logger.info(message): if count["non_identified_rooms"]: Logger.warning( source, "knows about room(s)", ", ".join(which["non_identified_rooms"]), "but", target, "does not" )
def __monkey_checker(self): __act = self.__android_helper.getCurAct() __app = self.__android_helper.getCurApp() Logger.d("App: %s, Act: %s " % (__act, __app)) # 第一次进入,设置Act为当前Act if self.__current_act == '': self.__current_act = __act self.__max_thinking_time = SETTING().get_max_thinging_time() self.__think_max_count = random.randint(1, self.__max_thinking_time) Logger.d("New Acticity, Max thing time = %s s" % self.__think_max_count) #如果两次相同,发送2次Back按键,并设置Act为空 elif self.__current_act == __act: if self.__think_count == self.__think_max_count: self.__android_helper.senKey(4) self.__android_helper.senKey(4) self.__current_act = '' self.__think_count = 0 Logger.d("Seam Acticity Max count: Back.") else: self.__think_count += 1 Logger.d("Seam Activity think count " + str(self.__think_count)) #如果两次不相同,则设置Act为当前act else: self.__current_act = __act self.__max_thinking_time = SETTING().get_max_thinging_time() self.__think_max_count = random.randint(1, self.__max_thinking_time) self.__think_count = 0 Logger.d("Diff Activity think count empty, Reset Max thing time = %s s" % self.__think_max_count)
class RRApp(object): """ Abstract base class for all rr job renderer aplications. """ def __init__(self): self.log = Logger(debug=True) def version(self): raise NotImplementedError() def open_scene(self): raise NotImplementedError() def set_env(self, name, value): if value is not None: os.environ[name] = value self.log.info('Environmental variable "%s" set to "%s"' % (name, value)) else: self.log.info('Can not set environment "%s" to "%s"' % (name, value)) def start_kso_server(self): """ This function perform Keep Scene Open RR functionality. Start TCP server and listen for commands from client. """ KSO_HOST = "localhost" KSO_PORT = 7774 server = rrKSOServer((KSO_HOST, KSO_PORT), rrKSOTCPHandler) server.handle_command() def start_render(self): raise NotImplementedError()
def resolve_room_categories(klass, building, floor_dict=None): """ Given a building, perform the mapping between it's dxf rooms and their relative categories. It does not save the building, which is responsibility of the caller. Arguments: - building: a Building object whose dxf rooms we want to process. - floor_dict: an (optional) dxf floor to limit the rooms to process. If None, all the current building floors will be used instead. Returns value: an integer representing the amount of rooms matched. The category name saved in place in each room dictionary, under the key "cat_name" """ categorized_rooms = 0 target_floors = floor_dict and [floor_dict] or building.get("dxf") and building.get("dxf")["floors"] or [] cats = klass.get_room_categories_dict() for floor_dict in target_floors: categorized_rooms += klass._resolve_room_categories_for_floor(floor_dict, cats) if categorized_rooms: Logger.info(categorized_rooms, "rooms were categorized") return categorized_rooms
def vis_peaks(self, heatmap, ori_img, name='default', vis_dir=PEAK_DIR, scale_factor=1, img_size=(368, 368)): vis_dir = os.path.join(self.configer.get('project_dir'), vis_dir) if not os.path.exists(vis_dir): Log.error('Dir:{} not exists!'.format(vis_dir)) os.makedirs(vis_dir) if not isinstance(heatmap, np.ndarray): if len(heatmap.size()) != 3: Log.error('Heatmap size is not valid.') exit(1) heatmap = heatmap.data.squeeze().cpu().numpy().transpose(1, 2, 0) if not isinstance(ori_img, np.ndarray): ori_img = DeNormalize(mean=[128.0, 128.0, 128.0],std=[256.0, 256.0, 256.0])(ori_img) ori_img = ori_img.data.cpu().squeeze().numpy().transpose(1, 2, 0) for j in range(self.configer.get('num_keypoints')): peaks = self.__get_peaks(heatmap[:, :, j].data.cpu().numpy()) image_path = os.path.join(vis_dir, '{}_{}.jpg'.format(name, j)) for peak in peaks: image = cv2.circle(ori_img, (peak[0], peak[1]), self.configer.get('vis', 'circle_radius'), (0,255,0), thickness=-1) image = self.scale_image(image, scale_factor, img_size) cv2.imwrite(image_path, image)
def __init__(self, quiet_start=0, quiet_end=0): if quiet_end < quiet_start: Logger.log('Invalid Quiet Hours.', Color.RED) exit() self.quiet_start = quiet_start self.quiet_stop = quiet_end self.is_quiet = False
def test(self): base_dir = os.path.join(self.configer.get('project_dir'), 'val/results/pose', self.configer.get('dataset'), 'test') if not os.path.exists(base_dir): os.makedirs(base_dir) test_img = self.configer.get('test_img') test_dir = self.configer.get('test_dir') if test_img is None and test_dir is None: Log.error('test_img & test_dir not exists.') exit(1) if test_img is not None and test_dir is not None: Log.error('Either test_img or test_dir.') exit(1) if test_img is not None: filename = test_img.rstrip().split('/')[-1] save_path = os.path.join(base_dir, filename) self.__test_img(test_img, save_path) else: for filename in self.__list_dir(test_dir): image_path = os.path.join(test_dir, filename) save_path = os.path.join(base_dir, filename) self.__test_img(image_path, save_path)
def select_pose_model(self): key = self.configer.get('method') if key == 'open_pose': if self.configer.get('phase') == 'train': return OpenPose(self.configer) else: return OpenPoseTest(self.configer) elif key == 'conv_pose_machine': if self.configer.get('phase') == 'train': return ConvPoseMachine(self.configer) else: return ConvPoseMachineTest(self.configer) elif key == 'associative_embedding': if self.configer.get('phase') == 'train': return AssociativeEmbedding(self.configer) else: return AssociativeEmbeddingTest(self.configer) elif key == 'fashion_ai': if self.configer.get('phase') == 'train': return FashionAI(self.configer) else: return FashionAITest(self.configer) else: Log.error('Pose Model: {} is not valid.'.format(key)) exit(1)
def select_det_model(self): key = self.configer.get('method') if key == 'pose_top_down': return ConvPoseMachine(self.configer) else: Log.error('Det Model: {} is not valid.'.format(key))
def information_exception(username): Logger.log( 'information exception caught\n' + 'username: '******'\n' + 'stacktrace: ' + '\n' + traceback.format_exc() + '\n\n', Color.RED)
def subscriptions_exception(username): Logger.log( 'subscriptions exception caught\n' + 'username: '******'\n' + 'stacktrace: ' + '\n' + traceback.format_exc() + '\n\n', Color.RED)
def __test_img(self, image_path, save_path): if self.configer.get('dataset') == 'cityscape': self.__test_cityscape_img(image_path, save_path) elif self.configer.get('dataset') == 'laneline': self.__test_laneline_img(image_path, save_path) else: Log.error('Dataset: {} is not valid.'.format(self.configer.get('dataset'))) exit(1)
def information(username): Logger.log( '-------------------------------\n' + ' INFORMATION\n' + 'username: '******'\n' + '-------------------------------\n\n', Color.GREEN)
def unsubscribe_all_exception(username): Logger.log( 'unsubscribe all exception caught\n' + 'username: '******'\n' + 'stacktrace: ' + '\n' + traceback.format_exc() + '\n\n', Color.RED)
def get_relation_loss(self, key): if key == 'embedding_loss': return EmbeddingLoss(num_keypoints=self.configer.get('data', 'num_keypoints'), l_vec=self.configer.get('capsule', 'l_vec')) else: Log.error('Relation loss: {} is not valid.'.format(key)) exit(1)
def subscriptions(username): Logger.log( '-------------------------------\n' + ' SUBSCRIPTIONS\n' + 'username: '******'\n' + '-------------------------------\n\n', Color.GREEN)
def unsubscribe_all(username): Logger.log( '-------------------------------\n' + ' UNSUBSCRIBE ALL\n' + 'username: '******'\n' + '-------------------------------\n\n', Color.RED)
def handle_unsubscribe_from_num_message(database, message, payload): Logger.log('Unsub from num') removed = database.remove_subscription_by_number(str(message.author), int(payload)) subs = database.get_subscriptions_by_user(str(message.author)) if removed: message.reply(inbox.compose_unsubscribe_from_num_message(str(message.author), removed, subs)) else: message.reply(inbox.compose_unsubscribe_invalid_sub_message(str(message.author), subs)) message.mark_as_read()
def check_for_commands(self): Logger.log('Checking for commands') commands = CommandHandler.get_commands(self.reddit) if CommandHandler.PAUSE in commands: self.run = False if CommandHandler.RUN in commands: self.run = True if CommandHandler.KILL in commands: exit()
class Valgrind: def get_name(self): return self.name def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "Valgrind" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) self.neg_count = 0 os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="gcc", CFLAGS="", LD="gcc") self.pipeline.run_bechmark(self, ["valgrind", "--error-exitcode=10"], 6) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): print len(self.tp_set) return self.tp_set def get_fp_set(self): print len(self.fp_set) return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): print(stderr) if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if exit_code != 0: if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(stdout, i, cur_dir, j, "NEG") def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") def cleanup(self): self.pipeline.clean_benchmark() self.logger.close_log()
class Net(object): def __init__(self, class_num=CLASS_NUM, init_build=True): net_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=False, intra_op_parallelism_threads=1, gpu_options=tf.GPUOptions(allow_growth=True # force_gpu_compatible=True )) # net_config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1 self.__loaded = False self.run_time = RUN_TIME self.summary_path = os.path.join(ROOT_PATH, 'log', 'summary_%s' % self.run_time) self.logger = Logger('net') self.class_num = class_num self.sess = tf.Session(config=net_config) self.graph = self.sess.graph self.img = None self.label = None self.train_step = None self.val_step = None self.val_data = None self.val_summary = None self.tower_loss = [] self.tower_grads = [] self.tower_preds = [] self.tower_accuracy = [] self.prediction = None self.accuracy = None self.loss = None self.trainer = None self.summary = None self.writer = None self.saver = None self.init_build = init_build if init_build: with self.graph.as_default(), tf.variable_scope('net_base'): self.img = tf.placeholder(tf.float32, [None, SIZE, SIZE, MOD_NUM], name='img') self.label = tf.placeholder(tf.float32, [None, CLASS_NUM], name='label') # self.inputs = tf.reshape(self.img, [-1, SIZE, SIZE, MOD_NUM]) # self.inputs = tf.identity(self.img, 'inputs') self.train_step = tf.train.get_or_create_global_step( self.graph) with tf.variable_scope('validation_net'): self.val_step = tf.Variable(0, dtype=tf.int32, name='v_step') self.val_data = { 'avg_acc': tf.Variable(0.0, dtype=tf.float32, name='avg_acc'), 'tp': tf.Variable(0.0, dtype=tf.float32, name='tp'), 'tn': tf.Variable(0.0, dtype=tf.float32, name='tn'), 'fp': tf.Variable(0.0, dtype=tf.float32, name='fp'), 'fn': tf.Variable(0.0, dtype=tf.float32, name='fn') } for name, var in self.val_data.items(): tf.summary.scalar(name, var, ['validation_summary']) self.val_summary = tf.summary.merge( self.graph.get_collection('validation_summary'), name='val_summary') def start(self): if not self.__loaded: self.sess.run(tf.global_variables_initializer()) self.__loaded = True def train(self, file_list, val_file_list, batch_size=BATCH_SIZE, val_batch_size=BATCH_SIZE, repeat_time=REPEAT_NUM, val_interval=VAL_INTERVAL, summary_interval=SUMMARY_INTERVAL): with self.graph.as_default(): self.start() self.logger.info('Train start') _, next_batch = generate_dataset(file_list, batch_size, repeat_time=repeat_time) while True: try: img, label = self.sess.run(next_batch) _, train_step, summary, accuracy, loss = self.sess.run( fetches=[ self.trainer, self.train_step, self.summary, self.accuracy, self.loss ], feed_dict={ self.img: img, self.label: label }) if train_step % val_interval == 0: self.validate(val_file_list, val_batch_size) elif train_step % summary_interval == 0: self.writer.add_summary(summary, train_step) self.logger.info( 'Training summary %d, accuracy: %f, loss %f' % (train_step, accuracy, loss)) except tf.errors.OutOfRangeError: break self.logger.info('Train end') def predict(self, img): return self.sess.run(self.prediction, feed_dict={self.img: img}) def whole_predict(self, tfr_name, batch_size=BATCH_SIZE, with_label=True, final_result=False, min_connect_tumor_num=MIN_CONNECT_TUMOR_NUM, min_tumor_num=MIN_TUMOR_NUM, random=True): with self.graph.as_default(): _, next_batch = generate_dataset([tfr_name], None, train=False, shuffle=False, batch=False) tfr_predict = [] tfr_imgs, tfr_labels = self.sess.run(next_batch) tfr_size = len(tfr_labels) if random: rank = np.arange(tfr_size) for _ in range(WHOLE_REPEAT_NUM): rank_copy = rank.copy() np.random.shuffle(rank_copy) predict = [] for i in range(0, tfr_size, batch_size): imgs = tfr_imgs[rank_copy][i:i + batch_size] if len(imgs) < batch_size: imgs = np.concatenate( [imgs] * batch_size).astype(int)[:batch_size] predict.append(self.predict(imgs)) predict = np.concatenate(predict)[:tfr_size] pair = list(zip(predict, rank_copy)) pair.sort(key=lambda x: x[1]) tfr_predict.append(np.stack(np.array(pair).T[0])) tfr_predict = np.array(tfr_predict).mean(axis=0).argmax(axis=1) else: for i in range(0, tfr_size, batch_size): imgs = tfr_imgs[i:i + batch_size] if len(imgs) < batch_size: imgs = np.concatenate( [imgs] * batch_size).astype(int)[:batch_size] tfr_predict.append(self.predict(imgs)) tfr_predict = np.concatenate(tfr_predict).argmax( axis=1)[:tfr_size] tfr_zero_predict_pos = np.where(tfr_predict == 0)[0] for pos in tfr_zero_predict_pos: if 0 < pos < tfr_predict.size - 1: if tfr_predict[pos - 1] == tfr_predict[pos + 1] == 1: tfr_predict[pos] = 1 tfr_one_predict_pos = np.where(tfr_predict == 1)[0] for pos in tfr_one_predict_pos: if 0 < pos < tfr_predict.size - 1: if tfr_predict[pos - 1] == tfr_predict[pos + 1] == 0: tfr_predict[pos] = 0 if final_result: tfr_predict = self.validate_slice_to_whole( tfr_predict, min_tumor_num, min_connect_tumor_num) if with_label: return tfr_predict, tfr_labels.argmax(axis=1) return tfr_predict def validate_slice_to_whole(self, predict, min_tumor_num=MIN_TUMOR_NUM, min_connect_tumor_num=MIN_CONNECT_TUMOR_NUM): tumor_predict_pos = np.where(predict == 1)[0] if tumor_predict_pos.size >= min_tumor_num: if min_connect_tumor_num in np.convolve( predict, np.ones(min_connect_tumor_num), mode='same'): return 1 return 0 def validate_report(self, predict_list, label_list, test, tumor_as_1=True, quiet=False): tumor_num, normal_num = int(tumor_as_1), int(not tumor_as_1) tumor_list = predict_list[label_list == tumor_num] # tumor result normal_list = predict_list[label_list == normal_num] # normal result if tumor_list.size and normal_list.size: tp_rate = (tumor_list == tumor_num).sum() / \ len(tumor_list) # t -> f fn_rate = 1 - tp_rate # t -> t tn_rate = (normal_list == normal_num).sum() / \ len(normal_list) # f -> f fp_rate = 1 - tn_rate # f -> t elif not tumor_list.size: tn_rate = (normal_list == normal_num).sum() / \ len(normal_list) # f -> f fp_rate = 1 - tn_rate # f -> t fn_rate = 0.0 tp_rate = 0.0 elif not normal_list.size: tp_rate = (tumor_list == tumor_num).sum() / \ len(tumor_list) # t -> f fn_rate = 1 - tp_rate # t -> t fp_rate = 0.0 tn_rate = 0.0 avg_accuracy = np.equal(predict_list, label_list).mean() train_step = self.sess.run(self.train_step) if not quiet: report_result = '(train step %d) Average accuary %f, TP %f, TN %f, FP %f, FN %f' % ( train_step, avg_accuracy, tp_rate, tn_rate, fp_rate, fn_rate) self.logger.info('Validation end') self.logger.info(report_result) with open(os.path.join(self.summary_path, 'result'), 'a+') as file: file.write(report_result + '\n') if not test: self.sess.run([ tf.assign(self.val_data['avg_acc'], avg_accuracy), tf.assign(self.val_data['tp'], tp_rate), tf.assign(self.val_data['tn'], tn_rate), tf.assign(self.val_data['fp'], fp_rate), tf.assign(self.val_data['fn'], fn_rate) ]) self.writer.add_summary(self.sess.run(self.val_summary), train_step) return { 'avg_accuracy': avg_accuracy, 'tp_rate': tp_rate, 'tn_rate': tn_rate, 'fp_rate': fp_rate, 'fn_rate': fn_rate } def validate(self, file_list, batch_size=BATCH_SIZE, test=False): with self.graph.as_default(): _, next_batch = generate_dataset(file_list, batch_size, train=False, shuffle=True, batch=True) val_step_initer = tf.variables_initializer([self.val_step]) self.sess.run(val_step_initer) label_list = [] predict_list = [] while True: try: img, label = self.sess.run(next_batch) accuracy, loss, predict, val_step = self.sess.run( fetches=[ self.accuracy, self.loss, self.prediction, self.val_step.assign_add(1) ], feed_dict={ self.img: img, self.label: label }) if val_step % VAL_SUMMARY_INTERVAL == 0: self.logger.info( 'Validation summary %d, accuracy: %f, loss: %f' % (val_step, accuracy, loss)) predict_list.append(predict.argmax(axis=1)) label_list.append(label.argmax(axis=1)) except tf.errors.OutOfRangeError: break predict_list = np.concatenate(predict_list) label_list = np.concatenate(label_list) self.validate_report(predict_list, label_list, test, tumor_as_1=True) return predict_list, label_list def whole_validate(self, file_list, batch_size=BATCH_SIZE, test=False, condition=None, min_connect_tumor_num=MIN_CONNECT_TUMOR_NUM, min_tumor_num=MIN_TUMOR_NUM): if condition is None: condition = lambda tfr_name, tfr_labels: 1 in tfr_labels with self.graph.as_default(): file_num = len(file_list) label_list = np.zeros((file_num), dtype=np.int64) predict_list = np.zeros((file_num), dtype=np.int64) for num, tfr_name in enumerate(file_list): predict_list[num], tfr_labels = self.whole_predict( tfr_name, batch_size, with_label=True, final_result=True, min_connect_tumor_num=min_connect_tumor_num, min_tumor_num=min_tumor_num) if condition(tfr_name, tfr_labels): label_list[num] = 1 self.validate_report(predict_list, label_list, test, tumor_as_1=True) return predict_list, label_list def whole_validate_best(self, file_list, batch_size=BATCH_SIZE, test=False, condition=None): if condition is None: condition = lambda tfr_name, tfr_labels: 1 in tfr_labels with self.graph.as_default(): file_num = len(file_list) label_list = np.zeros((file_num), dtype=np.int64) predict_slice_list = list() for num, tfr_name in enumerate(file_list): tfr_predict, tfr_labels = self.whole_predict( tfr_name, batch_size, with_label=True, final_result=False) predict_slice_list.append(tfr_predict) if condition(tfr_name, tfr_labels): label_list[num] = 1 result = dict() for min_connect_tumor_num in range(1, 10): for min_tumor_num in range(1, 25): func = partial(self.validate_slice_to_whole, min_connect_tumor_num=min_connect_tumor_num, min_tumor_num=min_tumor_num) predict_list = np.array(list(map(func, predict_slice_list))) tfr_result = self.validate_report(predict_list, label_list, test, tumor_as_1=True, quiet=True) result[(min_connect_tumor_num, min_tumor_num)] = tfr_result['avg_accuracy'] sorted_result = sorted(result.items(), key=lambda x: x[1], reverse=True) best_result = sorted_result[0][1] best_pairs = [ '(min_connect_tumor_num=%d, min_tumor_num=%d)' % (connect_tumor_num, tumor_num) for (connect_tumor_num, tumor_num), result in sorted_result if result == best_result ] report_result = 'Best params is %s with accuary %f' % ( ', '.join(best_pairs), best_result) self.logger.info(report_result) with open(os.path.join(self.summary_path, 'result'), 'a+') as file: file.write(report_result + '\n') def save(self, model_name): if self.__loaded: model_path = os.path.join(self.summary_path, 'model', model_name) self.saver.save(self.sess, model_path) self.logger.info('Save model %s' % model_path) def load(self, model_path): if not self.__loaded: self.run_time = model_path.strip('/').split('/')[-3][8:] self.summary_path = os.path.join(ROOT_PATH, 'log', 'summary_%s' % self.run_time) self.logger.reset_log_path(self.summary_path) self.logger.info('Change run time back to %s' % self.run_time) if not self.init_build: self.saver = tf.train.import_meta_graph(os.path.join( self.summary_path, 'model', 'model.meta'), clear_devices=True) self.img = self.graph.get_tensor_by_name('net_base/img:0') self.label = self.graph.get_tensor_by_name('net_base/label:0') self.train_step = self.graph.get_tensor_by_name( 'net_base/global_step:0') self.val_step = self.graph.get_tensor_by_name( 'net_base/validation_net/v_step:0') self.val_data = { 'avg_acc': self.graph.get_tensor_by_name( 'net_base/validation_net/avg_acc:0'), 'tp': self.graph.get_tensor_by_name( 'net_base/validation_net/tp:0'), 'tn': self.graph.get_tensor_by_name( 'net_base/validation_net/tn:0'), 'fp': self.graph.get_tensor_by_name( 'net_base/validation_net/fp:0'), 'fn': self.graph.get_tensor_by_name( 'net_base/validation_net/fn:0') } self.val_summary = tf.summary.merge( self.graph.get_collection('validation_summary'), name='val_summary') self.saver.restore(self.sess, model_path) self.__loaded = True self.logger.info('Load model %s' % model_path)
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait # 显性等待时间 from utils import assertion from tools.common.BrowserDriver import BrowserDriver from HTMLTestRunner_cn import HTMLTestRunner from utils.file_read import YamlRead from utils.file_write import YamlWrite import unittest import time import os import sys import random logger = Logger('logger').getlog() class Test_NamePlateScreen(unittest.TestCase): def setUp(self): self.sc = Config() self.b = BrowserDriver() self.driver = self.b.OpenBrowser() self.driver.implicitly_wait(5) # 全局隐形等待30S self.imgs = [] self.ym = YamlWrite() # 每次执行用例前都登陆网页 self.b.by_find_element('name', 'account').send_keys( self.sc.getConfig('User').get('username')) self.b.by_find_element('name', 'password').send_keys( self.sc.getConfig('User').get('password'))
from utils.logger import Logger from utils.sampler import SamplerConfig from utils.loss import LossConfig from utils.optim import OptimConfig from utils.dataloader import LoaderConfig from model.gan import GanConfig, GanSchema, BaseGan from model.zoo import ModelConfig from dataset.config import DatasetConfig from matplotlib import pyplot as plt import os logger = Logger(cat='test') @BaseGan.register('Vanilla') class VanillaGan(BaseGan): @BaseGan.add_post_iterate_hook def vis_gen(self, iter_num): if (iter_num + 1) % 1000 != 0: return self.eval_mode() x_generated = self.sampling(100, 100) xrange = ((self.dataset.data[:, 0].min() - 1, self.dataset.data[:, 0].max() + 1), (self.dataset.data[:, 1].min() - 1, self.dataset.data[:, 1].max() + 1)) fig = plt.figure() plt.hist2d(x_generated[:, 0], x_generated[:, 1], bins=100, range=xrange) plt.savefig(os.path.join(logger.img_dir, 'hmap_gen_{}.jpg'.format(iter_num))) plt.close() fig = plt.figure()
from selenium import webdriver from utils.logger import Logger logger = Logger(logger="Browser").logger #浏览器驱动配置 class Browser(object): def __init__(self): self.driver = webdriver.Chrome() logger.info("Starting browser:%s" % self.driver) if __name__ == '__main__': dr1 = Browser() dr = dr1.driver dr.get('http://www.baidu.com') dr.quit()
print(model, '\n') # TODO: Remove and handle with checkpoints if not args.train: print("Loading Model Weights ...") evaluation_state_dict = torch.load(args.eval_ckpt) model_dict = model.state_dict(full_dict=True) model_dict.update(evaluation_state_dict) model.load_state_dict(model_dict) model.eval() if args.train: val_dataset.set_label_usage(dataset.return_labels) # Create logger logger = Logger(os.path.join(job_path, 'logs')) # Get trainer trainer_creator = getattr(TrainerLoader, args.model) trainer = trainer_creator(args, model, dataset, data_loader, logger, device) if args.train: evaluator = trainer_creator(args, model, val_dataset, val_data_loader, logger, device) evaluator.train = False if args.train: print("Training ...") else: print("Evaluating ...") vars(args)['num_epochs'] = 1
def main(): """ Run main program """ parser = create_parser() options, args = parse_args(parser) # create instance of formatter from ist.ist_formatter_module import ISTFormatter formatter = ISTFormatter() # read input json file with open(options.input, 'r') as fp: json_data = json.load(fp) ist_info = { 'version': json_data['version']['flow123d_version'] if 'version' in json_data else 'Input reference' } json_data = json_data[ 'ist_nodes'] if 'ist_nodes' in json_data else json_data # filter out unsupported types, they won't be formatted items = list() for json_item in json_data: input_type = json_item[ 'input_type'] if 'input_type' in json_item else None if input_type in registered_nodes: item = registered_nodes[input_type]() item.parse(json_item) items.append(item) else: Logger.instance().info(' - item type not supported: %s' % str(json_item)) # if we have all items parsed we create references for item in items: if getattr(item, 'input_type', InputType.UNKNOWN) == InputType.MAIN_TYPE: if item.input_type == InputType.RECORD: for key in getattr(item, 'keys', []): if key.type.get_reference().input_type == InputType.ARRAY: key.type.get_reference().subtype.get_reference( ).add_link(item) else: key.type.get_reference().get_generic_root().add_link( item) if item.input_type == InputType.ABSTRACT_RECORD: for imp in getattr(item, 'implementations', []): imp.get_reference().add_link(item) # disable sort for now (type and name) keep items order unchanged # items = sorted(items, key=lambda x: '{}{}'.format(x.input_type.value, x.name)) # convert to tex format if options.format.lower() in ('tex', 'latex'): Logger.instance().info('-' * 80) Logger.instance().info('Formatting ist to tex format') from ist.utils.texlist2 import TexList TexList.PRETTY_FORMAT = options.debug formatter.json2latex(items, options.output, info=ist_info) if os.path.isfile(options.output): print('Ok: File "{:s}" created'.format(options.output)) sys.exit(0) else: print('Error: File "{:s}" does not exists'.format(options.output)) sys.exit(1) # convert to HTML format if options.format.lower() in ('html', 'html5', 'www', 'htm'): Logger.instance().info('-' * 80) Logger.instance().info('Formatting ist to html format') formatter.json2html(items, options.output, info=ist_info) if os.path.isfile(options.output): print('Ok: File "{:s}" created'.format(options.output)) sys.exit(0) else: print('Error: File "{:s}" does not exists'.format(options.output)) sys.exit(1) if options.format.lower() in ('markdown', 'md'): Logger.instance().info('Testing markdown') text = ''' # Using markdown in description **Description field** supports markdown syntax (support is partial and some techniques may not work in Python markdown implementation). ## Links Link to record [[root]] selection [[DG_output_fields]] or abstract [[Transport]]. All links are in the same format. If `link_name` is specified in `attributes` (let say DG_output_fields has link_name of DG), we can use that [[DG]] Record and Selection types offer links to their keys/values, so you can write [[DG#porosity]], to specify link text use following syntax [[DG#porosity:poro]] or link to key in Root item [[root#flow123d_version]] Every name should be unique, if `link_name` is duplicate first occurrence will be used! To avoid conflict with `link_name` or `name` we can use type specification like this [[record#root]]. 3 types are registered: 1. type RECORD supporting prefixes: - r - record 2. type SELECTION supporting prefixes: - s - selection 3. type ABSTRACT supporting prefixes: - a - ar - abstract ## Basics We can write **bold** statements (or *italic* if needed). We can also ~~strkkethrg~~ strikethrough some text to express some change. Another usage can be in lists, we can write both unordered and ordered list. Important is to place one empty line before list starts. Unordered list: - important item - another important item Ordered list have same rules: 1. item number 1 2. and item number 2 To write code section with monospaced font, such as variable use `this` syntax. **Note** Use line breaks \\n chars sparely, and only break text-flow if necessarily. Full markdown specification can be found [here](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet) but since Python is used to parse markdown, there may be slight differences. (($q_w$)) ''' o = htmltree() o.description(text) print(o.dump()) sys.exit(0) Logger.instance().error("Error: Unsupported format '{:s}'".format( options.format)) sys.exit(1)
def log(*args, **kwargs): msg = copy.deepcopy(request.json) Logger().info({"body": msg, "url": request.url}) return f(*args, **kwargs)
_, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels).sum().item() num_iters += 1 return correct / total, running_loss / num_iters if __name__ == "__main__": if len(sys.argv) < 2: raise ValueError('Please enter config file path.') # Read configs configFile = sys.argv[1] with open(configFile, 'r') as f: config = yaml.safe_load(f) # Create a logger logger = Logger(config['logdir']) logger.log_string(datetime.datetime.now()) logger.log_string(config) logger.backup_files(['train.py', 'qpu_ops.py', 'qpu_layers.py']) if config['dataset'] == 'ntu': logger.backup_files(['data/ntu/feeder.py']) if config['dataset'] == 'fpha': logger.backup_files(['data/fpha/feeder.py']) # Train train(config, logger)
message = 'KFold training and evaluation has been done.\n' message += f'F1 majority voting - Avg: {f1_majority_mean}, Std: {f1_majority_std}\n' message += f'F1 optimized - Avg: {f1_optimized_mean}, Std: {f1_optimized_std}\n' message += f'Threshold - Avg: {threshold_mean}' logger.post(message) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--batch-size', default=512, type=int) parser.add_argument('--device-ids', metavar='N', type=int, nargs='+', default=[0]) parser.add_argument('--max-workers', default=2, type=int) parser.add_argument('--debug', action='store_true') args = parser.parse_args().__dict__ log_dir = LOG_DIR.joinpath(f'{SCRIPT_NAME}') logger = Logger(SCRIPT_NAME, log_dir=log_dir, webhook_url=SLACK_URL, overwrite=True) logger.info(f'Script starts with command line arguments: {args}') try: main(logger, args) logger.post('===== Script completed successfully! =====') except Exception as e: logger.exception(e)
def _init_logger(self): logger = Logger(os.path.join(self.model_dir, 'logs')) return logger
def __init__(self): self.logger = Logger(__name__, PathMgr.get_log_path()) self.historical_data_provider = DBProvider()
def __init__(self, logger=Logger(__name__, None)): self.logger = logger
def main(): new_exp = input("Is this a new experiment? [Y/N]") if new_exp == 'Y': # capture the config path from the run arguments # then process the json configuration file config_filename = '/home/ADAMGE/action_recognition/action_recognition_v1/configs/params.json' elif new_exp == 'N': config_filename = input( "Enter the full path of the config file in the old experiment folder" ) else: print("Wrong input") exit() paths_filename = '/home/ADAMGE/action_recognition/action_recognition_v1/configs/paths.json' config = process_config(config_filename, paths_filename, new_exp) # create the experiments dirs and write the JSON file to the dir create_dirs([config.summary_dir, config.checkpoint_dir], config_filename) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7) sess_config = tf.ConfigProto(gpu_options=gpu_options) # create tensorflow session sess = tf.Session(config=sess_config) # create an instance of the model you want model = ExampleModel(config) # create your data generator data_train = DataGenerator(model, config, sess, 'train', shuffle=True, augment=False) data_validate = DataGenerator(model, config, sess, 'validate', shuffle=False, augment=False) # create tensorboard logger logger = Logger(sess, config) # create trainer and pass all the previous components to it trainer = ExampleTrainer(sess, model, data_train, data_validate, config, logger) # restore mobile net model.restore_mobile_net(sess) # load model if exists - only the lstm if new_exp == 'N': model.load(sess) # training trainer.train()
class Agent(object): def __init__(self, config, policy, global_episode, n_agent=0, agent_type='exploration', log_dir=''): print(f"Initializing agent {n_agent}...") self.config = config self.n_agent = n_agent self.agent_type = agent_type self.max_steps = config['max_ep_length'] self.num_episode_save = config['num_episode_save'] self.global_episode = global_episode self.local_episode = 0 self.log_dir = log_dir # Create environment self.env_wrapper = create_env_wrapper(config) self.env_wrapper.env.set_agent(self.n_agent) self.ou_noise = OUNoise(dim=config["action_dim"], low=config["action_low"], high=config["action_high"]) self.ou_noise.reset() self.actor = policy # Logger log_path = f"{log_dir}/agent-{n_agent}" self.logger = Logger(log_path) def update_actor_learner(self, learner_w_queue): """Update local actor to the actor from learner. """ if learner_w_queue.empty(): return source = learner_w_queue.get() target = self.actor for target_param, source_param in zip(target.parameters(), source): w = torch.tensor(source_param).float() target_param.data.copy_(w) def run(self, training_on, replay_queue, learner_w_queue, update_step): # Initialise deque buffer to store experiences for N-step returns self.exp_buffer = deque() best_reward = -float("inf") rewards = [] while training_on.value: episode_reward = 0 num_steps = 0 self.local_episode += 1 self.global_episode.value += 1 self.exp_buffer.clear() if self.local_episode % 100 == 0: print(f"Agent: {self.n_agent} episode {self.local_episode}") ep_start_time = time.time() print("call reset on agent {}".format(self.n_agent)) state = self.env_wrapper.reset() print(state.shape) print("called reset on agent {}".format(self.n_agent)) self.ou_noise.reset() self.env_wrapper.env.resume_simulator() done = False angle_avg = [] distance_avg = [] while not done: action = self.actor.get_action(state) if self.agent_type == "supervisor": action = self.env_wrapper.env.get_supervised_action() elif self.agent_type == "exploration": action = self.ou_noise.get_action(action, num_steps) action = action.squeeze(0) else: action = action.detach().cpu().numpy().flatten() next_state, reward, done = self.env_wrapper.step(action) angle_avg.append(state[0]) distance_avg.append(math.hypot(state[1], state[2])) episode_reward += reward state = self.env_wrapper.normalise_state(state) reward = self.env_wrapper.normalise_reward(reward) self.exp_buffer.append((state, action, reward)) # We need at least N steps in the experience buffer before we can compute Bellman # rewards and add an N-step experience to replay memory if len(self.exp_buffer) >= self.config['n_step_returns']: state_0, action_0, reward_0 = self.exp_buffer.popleft() discounted_reward = reward_0 gamma = self.config['discount_rate'] for (_, _, r_i) in self.exp_buffer: discounted_reward += r_i * gamma gamma *= self.config['discount_rate'] if not replay_queue.full(): replay_queue.put([ state_0, action_0, discounted_reward, next_state, done, gamma ]) state = next_state if done or num_steps == self.max_steps: print("agent {} done steps: {}/{}".format( self.n_agent, num_steps, self.max_steps)) # add rest of experiences remaining in buffer while len(self.exp_buffer) != 0: #print("agent {} exp_buffer_len {}".format(self.n_agent, len(self.exp_buffer))) state_0, action_0, reward_0 = self.exp_buffer.popleft() discounted_reward = reward_0 gamma = self.config['discount_rate'] for (_, _, r_i) in self.exp_buffer: #print("agent {} exp_buffer_len {}".format(self.n_agent, len(self.exp_buffer))) discounted_reward += r_i * gamma gamma *= self.config['discount_rate'] replay_queue.put([ state_0, action_0, discounted_reward, next_state, done, gamma ]) break num_steps += 1 #print("agent {} finished if".format(self.n_agent)) # Log metrics step = update_step.value if self.agent_type == "exploitation": self.logger.scalar_summary("agent/angle", np.rad2deg(np.mean(angle_avg)), step) self.logger.scalar_summary("agent/angle_var", np.rad2deg(np.var(angle_avg)), step) self.logger.scalar_summary("agent/distance", np.mean(distance_avg), step) self.logger.scalar_summary("agent/distance_var", np.var(distance_avg), step) observation_image = self.env_wrapper.env.get_current_observation_image( ) if num_steps == self.max_steps: self.logger.image_summar("agent/observation_end", observation_image, num_steps) else: self.logger.image_summar( "agent/observation_p_{:2.3f}".format( discounted_reward), observation_image, num_steps) self.logger.scalar_summary("agent/reward", episode_reward, step) self.logger.scalar_summary("agent/episode_timing", time.time() - ep_start_time, step) # Saving agent if self.local_episode % self.num_episode_save == 0 or episode_reward > best_reward: if episode_reward > best_reward: best_reward = episode_reward self.save( f"local_episode_{self.local_episode}_reward_{best_reward:4f}" ) print("reward is: {} step: {} ".format(episode_reward, step)) rewards.append(episode_reward) if (self.agent_type == "exploration" or self.agent_type == "supervisor" ) and self.local_episode % self.config['update_agent_ep'] == 0: self.update_actor_learner(learner_w_queue) # while not replay_queue.empty(): # replay_queue.get() # Save replay from the first agent only # if self.n_agent == 0: # self.save_replay_gif() #print(f"Agent {self.n_agent} done.") def save(self, checkpoint_name): last_path = f"{self.log_dir}" process_dir = f"{self.log_dir}/agent_{self.n_agent}" if not os.path.exists(process_dir): os.makedirs(process_dir) if not os.path.exists(last_path): os.makedirs(last_path) model_fn = f"{process_dir}/{checkpoint_name}.pt" torch.save(self.actor, model_fn) model_fn = f"{last_path}/best.pt" torch.save(self.actor, model_fn) def save_replay_gif(self): dir_name = "replay_render" if not os.path.exists(dir_name): os.makedirs(dir_name) state = self.env_wrapper.reset() self.env_wrapper.env.resume_simulator() for step in range(self.max_steps): action = self.actor.get_action(state) action = action.cpu().detach().numpy() next_state, reward, done = self.env_wrapper.step(action) img = self.env_wrapper.render() plt.imsave(fname=f"{dir_name}/{step}.png", arr=img) state = next_state if done: break fn = f"{self.config['env']}-{self.config['model']}-{step}.gif" make_gif(dir_name, f"{self.log_dir}/{fn}") shutil.rmtree(dir_name, ignore_errors=False, onerror=None) print("fig saved to ", f"{self.log_dir}/{fn}")
import time from utils.network import check_internet_connected, start_subprocess_to_login_csust_bg from utils.config import load_config from utils.tips import info from utils.logger import Logger from utils.wifi import LinkWifi wifi = LinkWifi() logger = Logger('./out.log') config = load_config('config/keep_network_connect.json') logger.write(info(f"config: {config}\n")) while True: # 判断是否已连接到网络 if check_internet_connected() is False: # 连接WiFi if wifi.connect_wifi('csust-bg') == wifi.status('connected'): logger.write(info("connecting")) # 启动子进程,登录csust-bg if start_subprocess_to_login_csust_bg(config['start_process'], config['check_max_line']) is True: logger.write(info("network has been connected!")) else: logger.write(info("connect failed!")) # 运行出现错误,或者尝试次数过多 # 等待N秒 time.sleep(config['check_per_second'])
# Memory memory = Memory(args.mem_size, state_dim, action_dim, args) # Algorithm drla = NASReptiLe(state_dim, action_dim, max_action, args) # Action noise a_noise = GaussianNoise(action_dim, sigma=args.gauss_sigma) # Logger fields = [ "eval_score", "total_steps", "train_scores", "sampled_archi", "log_alphas" ] logger = Logger(args.output, fields) # Train ite = 0 total_steps = 0 while total_steps < args.max_steps: ite += 1 fitness = [] c_losses = [] a_losses = [] pop = drla.sample_pop(args.pop_size) # Update actors and critic if total_steps >= args.start_steps:
def main(): # Train model best_val_loss = np.inf best_epoch = 0 # AL grouped must use controlled training dataset train_dataset = OneGraphDataset.load_one_graph_data( 'train_causal_vel_' + args.suffix, train_data_min_max=None, size=args.train_size, self_loop=args.self_loop, control=args.grouped, control_nodes=args.input_atoms, variations=args.variations, need_grouping=args.need_grouping) if args.val_grouped: # To see control loss, val and test should be grouped valid_dataset = OneGraphDataset.load_one_graph_data( 'valid_causal_vel_' + args.val_suffix, train_data_min_max=[train_dataset.mins, train_dataset.maxs], size=args.val_size, self_loop=args.self_loop, control=True, control_nodes=args.input_atoms, variations=args.val_variations, need_grouping=args.val_need_grouping) valid_sampler = RandomPytorchSampler(valid_dataset) valid_data_loader = DataLoader(valid_dataset, batch_size=args.val_bs, shuffle=False, sampler=valid_sampler) # test_data = load_one_graph_data( # 'test_'+args.val_suffix, size=args.test_size, self_loop=args.self_loop, control=True, control_nodes=args.input_atoms, variations=4) # test_sampler = RandomPytorchSampler(test_data) # test_data_loader = DataLoader( # test_data, batch_size=args.val_bs, shuffle=False, sampler=test_sampler) else: valid_dataset = OneGraphDataset.load_one_graph_data( 'valid_causal_vel_' + args.val_suffix, train_data_min_max=[train_dataset.mins, train_dataset.maxs], size=args.val_size, self_loop=args.self_loop, control=False) valid_data_loader = DataLoader(valid_dataset, batch_size=args.val_bs, shuffle=True) # test_data = load_one_graph_data( # 'test_'+args.val_suffix, size=args.val_size, self_loop=args.self_loop, control=False) # test_data_loader = DataLoader( # test_data, batch_size=args.val_bs, shuffle=True) if args.sampler == 'random': sampler = RandomDatasetSampler(train_dataset, args) elif args.sampler == 'uncertainty': sampler = MaximalEntropyDatasetSampler(train_dataset, args) else: print('Only random and uncertainty samplers are supported for now!') logger = Logger(save_folder) print('Doing initial validation before training...') val_control(args, log_prior, logger, save_folder, valid_data_loader, -1, decoder, rel_rec, rel_send, scheduler) data_idx = [] nodes = [] # Start with one group of each node for i in range(args.input_atoms): # group_size=args.variations new_data_idx = sampler.sample([i], args.variations, 200) uncertain_nodes = torch.LongTensor([i] * args.variations * 200).cuda() data_idx.append(new_data_idx) nodes.append(uncertain_nodes) data_idx = torch.cat(data_idx) nodes = torch.cat(nodes) al_train_dataset = ALIndexDataset(train_dataset, data_idx, nodes) # while len(al_train_dataset.idxs) < args.budget: for epoch in range(500): # epoch = int(len(al_train_dataset)/args.log_data_size) # Sampler batch_size is fixed to be #value variations for a node # control_nodes = sampler.criterion(decoder.rel_graph, k=args.topk) # new_data_idx = sampler.sample( # control_nodes, args.variations, args.sample_num_groups) # uncertain_nodes = torch.LongTensor( # control_nodes).repeat_interleave(args.variations*args.sample_num_groups).cuda() # al_train_dataset.update(new_data_idx, uncertain_nodes) # with open(os.path.join(save_folder, str(epoch)+'_queries.pt'), 'wb') as f: # torch.save( # [al_train_dataset.nodes, al_train_dataset.idxs], f) # print('sampled nodes this episode', # control_nodes, new_data_idx, len(al_train_dataset)) # Only need to be a normal dataloader since the grouping and indexing are already implemented in al_train_dataset. Just make sure shuffle=False. train_data_loader = DataLoader(al_train_dataset, batch_size=args.train_bs, shuffle=False) # for j in range(10): # print('epoch and j', epoch, j) nll, nll_lasttwo, kl, mse, control_constraint_loss, lr, rel_graphs, rel_graphs_grad, a, b, c, d, e, f = train_control( args, log_prior, optimizer, save_folder, train_data_loader, valid_data_loader, decoder, rel_rec, rel_send, epoch) if epoch % args.train_log_freq == 0: logger.log('train', decoder, epoch, nll, nll_lasttwo, kl=kl, mse=mse, control_constraint_loss=control_constraint_loss, lr=lr, rel_graphs=rel_graphs, rel_graphs_grad=rel_graphs_grad, msg_hook_weights=a, nll_train_lasttwo=b, nll_train_lasttwo_5=c, nll_train_lasttwo_10=d, nll_train_lasttwo__1=e, nll_train_lasttwo_1=f) if epoch % args.val_log_freq == 0: _ = val_control(args, log_prior, logger, save_folder, valid_data_loader, epoch, decoder, rel_rec, rel_send, scheduler) scheduler.step() print("Optimization Finished!") print("Best Epoch: {:04d}".format(logger.best_epoch)) if args.save_folder: print("Best Epoch: {:04d}".format(logger.best_epoch), file=meta_file) meta_file.flush() test_control(test_data_loader) if meta_file is not None: print(save_folder) meta_file.close()
def __init__(self): self.logger = Logger(self.__class__.__name__ or __name__, PathMgr.get_log_path())
def mcapi_playback(name_interface): """Main function that calibrates the robot, get it into a default waiting position then launch the main control loop once the user has pressed the Enter key Args: name_interface (string): name of the interface that is used to communicate with the robot """ name_replay = "/home/odri/git/thomasCbrs/log_eval/test_3/06_nl/" # name_replay = "/home/odri/git/thomasCbrs/log_eval/vmax_nl/" # replay_q = np.loadtxt(name_replay + "_q.dat", delimiter=" ") # replay_v = np.loadtxt(name_replay + "_v.dat", delimiter=" ") # replay_tau = np.loadtxt(name_replay + "_tau.dat", delimiter=" ") qtsid_full = np.load(name_replay + "qtsid.npy" , allow_pickle = True) vtsid_full = np.load(name_replay + "vtsid.npy" , allow_pickle = True) tau_ff = np.load(name_replay + "torques_ff.npy" , allow_pickle = True) replay_q = qtsid_full[7:,:].transpose() replay_v = vtsid_full[6:,:].transpose() replay_tau = tau_ff.transpose() N_SIMULATION = replay_q.shape[0] # Default position after calibration # q_init = replay_q[0, 1:] q_init = replay_q[0, :] if SIMULATION: device = PyBulletSimulator() qc = None else: device = Solo12(name_interface, dt=DT) qc = QualisysClient(ip="140.93.16.160", body_id=0) if LOGGING: logger = Logger(device, qualisys=qc, logSize=N_SIMULATION) # Number of motors nb_motors = device.nb_motors # Initiate communication with the device and calibrate encoders if SIMULATION: device.Init(calibrateEncoders=True, q_init=q_init, envID=0, use_flat_plane=True, enable_pyb_GUI=True, dt=DT) else: device.Init(calibrateEncoders=True, q_init=q_init) # Wait for Enter input before starting the control loop put_on_the_floor(device, q_init) # CONTROL LOOP *************************************************** t = 0.0 t_max = (N_SIMULATION-1) * DT i = 0 P = 7 * np.ones(12) D = 0.5 * np.ones(12) q_des = np.zeros(12) v_des = np.zeros(12) tau_ff = np.zeros(12) while ((not device.hardware.IsTimeout()) and (t < t_max)): device.UpdateMeasurment() # Retrieve data from IMU and Motion capture # Set desired quantities for the actuators device.SetDesiredJointPDgains(P, D) # device.SetDesiredJointPosition(replay_q[i, 1:]) # device.SetDesiredJointVelocity(replay_v[i, 1:]) # device.SetDesiredJointTorque(replay_tau[i, 1:]) device.SetDesiredJointPosition(replay_q[i, :]) device.SetDesiredJointVelocity(replay_v[i, :]) device.SetDesiredJointTorque(replay_tau[i, :]) # Call logger if LOGGING: logger.sample(device, qualisys=qc) # Send command to the robot device.SendCommand(WaitEndOfCycle=True) if ((device.cpt % 1000) == 0): device.Print() t += DT i += 1 # DAMPING TO GET ON THE GROUND PROGRESSIVELY ********************* t = 0.0 t_max = 2.5 while ((not device.hardware.IsTimeout()) and (t < t_max)): device.UpdateMeasurment() # Retrieve data from IMU and Motion capture # Set desired quantities for the actuators device.SetDesiredJointPDgains(np.zeros(12), 0.1 * np.ones(12)) device.SetDesiredJointPosition(np.zeros(12)) device.SetDesiredJointVelocity(np.zeros(12)) device.SetDesiredJointTorque(np.zeros(12)) # Send command to the robot device.SendCommand(WaitEndOfCycle=True) if ((device.cpt % 1000) == 0): device.Print() t += DT # FINAL SHUTDOWN ************************************************* # Whatever happened we send 0 torques to the motors. device.SetDesiredJointTorque([0]*nb_motors) device.SendCommand(WaitEndOfCycle=True) if device.hardware.IsTimeout(): print("Masterboard timeout detected.") print("Either the masterboard has been shut down or there has been a connection issue with the cable/wifi.") device.hardware.Stop() # Shut down the interface between the computer and the master board # Save the logs of the Logger object if LOGGING: logger.saveAll() print("Log saved") if SIMULATION: # Disconnect the PyBullet server (also close the GUI) device.Stop() print("End of script") quit()
class BaseDAO(object): def __init__(self): self.logger = Logger(self.__class__.__name__ or __name__, PathMgr.get_log_path()) @staticmethod def get_connection(): db_config = ConfigMgr.get_db_config() return mysql.connector.connect(host=db_config['host'], user=db_config['user'], password=db_config['password'], database=db_config['database']) @staticmethod def python_value_to_sql_value(val): if val is not None: if type(val) is float: return '{:.5f}'.format(val) else: return str(val) else: return 'null' @staticmethod def mysql_format(template, *args): mysql_args = map(BaseDAO.python_value_to_sql_value, args) return template.format(*mysql_args) def select(self, query, cursor=None): #self.logger.info('query:%s' % query) conn = None if cursor is None: conn = BaseDAO.get_connection() cursor = conn.cursor() try: cursor.execute(query) rows = cursor.fetchall() return rows except Exception as e: error_message = "Query:{}, error message: {}, Stack Trace: {}".format( query, str(e), traceback.format_exc()) self.logger.exception(error_message) finally: if conn: conn.close() def execute_query(self, query, cursor=None): #self.logger.info('query:%s' % query) conn = None if cursor is None: conn = BaseDAO.get_connection() cursor = conn.cursor() try: cursor.execute(query) if conn: conn.commit() except mysql.connector.IntegrityError: pass except Exception as e: error_message = "Query:{}, error message: {}, Stack Trace: {}".format( query, str(e), traceback.format_exc()) self.logger.exception(error_message) finally: if conn: conn.close() def execute_query_list(self, query_list): conn = BaseDAO.get_connection() cursor = conn.cursor() try: query_for_log_exception = None for query in query_list: #self.logger.info('query:%s' % query) query_for_log_exception = query cursor.execute(query) conn.commit() except Exception as e: error_message = "Query:{}, error message: {}, Stack Trace: {}".format( query_for_log_exception, str(e), traceback.format_exc()) self.logger.exception(error_message) finally: conn.close()
if level == "debug": return logging.DEBUG if level == "info": return logging.INFO if level == "warn": return logging.WARN if level == "error": return logging.ERROR INIT_FILE_PATH = "config/init.ini" NUMBER_OF_FRAMES = -1 CAMERA_NAME = socket.gethostname() app_conf = AppConf(INIT_FILE_PATH) logger = Logger(level=get_conf_log_level(app_conf.log_level)) cap = cv2.VideoCapture(0) count = NUMBER_OF_FRAMES with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor: while count > 0 or NUMBER_OF_FRAMES == -1: try: # Capture frame-by-frame ret, frame = cap.read() if ret: cv2.imshow('frame', frame) vi = V3ioImage(logger, frame, CAMERA_NAME) img_json = vi.image_json logger.debug(img_json) future = { executor.submit(
def main_worker(gpu_idx, configs): configs.gpu_idx = gpu_idx configs.device = torch.device('cpu' if configs.gpu_idx is None else 'cuda:{}'.format(configs.gpu_idx)) if configs.distributed: if configs.dist_url == "env://" and configs.rank == -1: configs.rank = int(os.environ["RANK"]) if configs.multiprocessing_distributed: # For multiprocessing distributed training, rank needs to be the # global rank among all the processes configs.rank = configs.rank * configs.ngpus_per_node + gpu_idx dist.init_process_group(backend=configs.dist_backend, init_method=configs.dist_url, world_size=configs.world_size, rank=configs.rank) configs.subdivisions = int(64 / configs.batch_size / configs.ngpus_per_node) else: configs.subdivisions = int(64 / configs.batch_size) configs.is_master_node = (not configs.distributed) or ( configs.distributed and (configs.rank % configs.ngpus_per_node == 0)) if configs.is_master_node: logger = Logger(configs.logs_dir, configs.saved_fn) logger.info('>>> Created a new logger') logger.info('>>> configs: {}'.format(configs)) tb_writer = SummaryWriter(log_dir=os.path.join(configs.logs_dir, 'tensorboard')) else: logger = None tb_writer = None # model model = create_model(configs) # load weight from a checkpoint if configs.pretrained_path is not None: assert os.path.isfile(configs.pretrained_path), "=> no checkpoint found at '{}'".format(configs.pretrained_path) model.load_state_dict(torch.load(configs.pretrained_path)) if logger is not None: logger.info('loaded pretrained model at {}'.format(configs.pretrained_path)) # resume weights of model from a checkpoint if configs.resume_path is not None: assert os.path.isfile(configs.resume_path), "=> no checkpoint found at '{}'".format(configs.resume_path) model.load_state_dict(torch.load(configs.resume_path)) if logger is not None: logger.info('resume training model from checkpoint {}'.format(configs.resume_path)) # Data Parallel model = make_data_parallel(model, configs) # Make sure to create optimizer after moving the model to cuda optimizer = create_optimizer(configs, model) lr_scheduler = create_lr_scheduler(optimizer, configs) configs.step_lr_in_epoch = True if configs.lr_type in ['multi_step'] else False # resume optimizer, lr_scheduler from a checkpoint if configs.resume_path is not None: utils_path = configs.resume_path.replace('Model_', 'Utils_') assert os.path.isfile(utils_path), "=> no checkpoint found at '{}'".format(utils_path) utils_state_dict = torch.load(utils_path, map_location='cuda:{}'.format(configs.gpu_idx)) optimizer.load_state_dict(utils_state_dict['optimizer']) lr_scheduler.load_state_dict(utils_state_dict['lr_scheduler']) configs.start_epoch = utils_state_dict['epoch'] + 1 if configs.is_master_node: num_parameters = get_num_parameters(model) logger.info('number of trained parameters of the model: {}'.format(num_parameters)) if logger is not None: logger.info(">>> Loading dataset & getting dataloader...") # Create dataloader train_dataloader, train_sampler = create_train_dataloader(configs) if logger is not None: logger.info('number of batches in training set: {}'.format(len(train_dataloader))) if configs.evaluate: val_dataloader = create_val_dataloader(configs) precision, recall, AP, f1, ap_class = evaluate_mAP(val_dataloader, model, configs, None) print('Evaluate - precision: {}, recall: {}, AP: {}, f1: {}, ap_class: {}'.format(precision, recall, AP, f1, ap_class)) print('mAP {}'.format(AP.mean())) return for epoch in range(configs.start_epoch, configs.num_epochs + 1): if logger is not None: logger.info('{}'.format('*-' * 40)) logger.info('{} {}/{} {}'.format('=' * 35, epoch, configs.num_epochs, '=' * 35)) logger.info('{}'.format('*-' * 40)) logger.info('>>> Epoch: [{}/{}]'.format(epoch, configs.num_epochs)) if configs.distributed: train_sampler.set_epoch(epoch) # train for one epoch train_one_epoch(train_dataloader, model, optimizer, lr_scheduler, epoch, configs, logger, tb_writer) if not configs.no_val: val_dataloader = create_val_dataloader(configs) print('number of batches in val_dataloader: {}'.format(len(val_dataloader))) precision, recall, AP, f1, ap_class = evaluate_mAP(val_dataloader, model, configs, logger) val_metrics_dict = { 'precision': precision.mean(), 'recall': recall.mean(), 'AP': AP.mean(), 'f1': f1.mean(), 'ap_class': ap_class.mean() } if tb_writer is not None: tb_writer.add_scalars('Validation', val_metrics_dict, epoch) # Save checkpoint if configs.is_master_node and ((epoch % configs.checkpoint_freq) == 0): model_state_dict, utils_state_dict = get_saved_state(model, optimizer, lr_scheduler, epoch, configs) save_checkpoint(configs.checkpoints_dir, configs.saved_fn, model_state_dict, utils_state_dict, epoch) if not configs.step_lr_in_epoch: lr_scheduler.step() if tb_writer is not None: tb_writer.add_scalar('LR', lr_scheduler.get_lr()[0], epoch) if tb_writer is not None: tb_writer.close() if configs.distributed: cleanup()
from adapters.data_adapter import IDataAdapter from adapters.hist_data_adapter import HistDataAdapter from adapters.influx_sensordata import InfluxSensorData from models.arima import train_or_load_ARIMA from models.autoarima import train_or_load_autoARIMA from models.expsmoothing import train_or_load_expSmoothing from models.lstm import train_or_load_LSTM from models.modelholder import ModelHolder from preprocessing.imputing import impute_simple_imputer from preprocessing.moving_average import moving_average from utils.config import default_config from utils.logger import Logger from utils.threading import to_thread from utils.timer import Timer logger: Logger = Logger(module_name="Model 1") script_timer = Timer() # structure of time series: rows: instances, cols: variables, depth: series of values def print_header() -> None: """ Prints a header for a more pleasant console user experience. Not that this is at all important, but i felt like it. Don't try to change my mind. """ logger.info( "##############################################################") logger.info( "# Datascience in Techno-Socio-Economic Systems #") logger.info(
def __init__(self): ##The top config #self.data_root = '/media/hhy/data/USdata/MergePhase1/test_0.3' #self.log_dir = '/media/hhy/data/code_results/MILs/MIL_H_Attention' self.root = '/remote-home/my/Ultrasound_CV/data/Ruijin/clean' self.log_dir = '/remote-home/my/hhy/Ultrasound_MIL/experiments/PLN1/base/fold4' if not os.path.exists(self.log_dir): os.makedirs(self.log_dir) ##training config self.lr = 1e-4 self.epoch = 50 self.resume = -1 self.batch_size = 1 self.net = Attention() self.net.cuda() self.optimizer = Adam(self.net.parameters(), lr=self.lr) self.lrsch = torch.optim.lr_scheduler.MultiStepLR( self.optimizer, milestones=[10, 30, 50, 70], gamma=0.5) self.logger = Logger(self.log_dir) self.train_transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.RandomResizedCrop((224, 224)), transforms.RandomHorizontalFlip(0.5), transforms.RandomVerticalFlip(0.5), transforms.ColorJitter(0.25, 0.25, 0.25, 0.25), transforms.ToTensor() ]) self.test_transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) self.label_name = "手术淋巴结情况(0未转移;1转移)" self.trainbag = RuijinBags(self.root, [0, 1, 2, 3], self.train_transform, label_name=self.label_name) self.testbag = RuijinBags(self.root, [4], self.test_transform, label_name=self.label_name) train_label_list = list( map(lambda x: int(x['label']), self.trainbag.patient_info)) pos_ratio = sum(train_label_list) / len(train_label_list) print(pos_ratio) train_weight = [(1 - pos_ratio) if x > 0 else pos_ratio for x in train_label_list] # self.train_sampler = WeightedRandomSampler(weights=train_weight, num_samples=len(self.trainbag)) self.train_loader = DataLoader(self.trainbag, batch_size=self.batch_size, num_workers=8, shuffle=True) self.val_loader = DataLoader(self.testbag, batch_size=self.batch_size, shuffle=False, num_workers=8) if self.resume > 0: self.net, self.optimizer, self.lrsch, self.loss, self.global_step = self.logger.load( self.net, self.optimizer, self.lrsch, self.loss, self.resume) else: self.global_step = 0 # self.trainer = MTTrainer(self.net, self.optimizer, self.lrsch, self.loss, self.train_loader, self.val_loader, self.logger, self.global_step, mode=2) self.trainer = MILTrainer(self.net, self.optimizer, self.lrsch, None, self.train_loader, self.val_loader, self.logger, self.global_step)
def main(args): args = prepare_log_dir(args) # set dtype for training chainer.global_config.dtype = args.dtype train_dataset = BaseImageDataset( args.train_file, args.image_size, root=os.path.dirname(args.train_file), dtype=chainer.get_dtype(), ) validation_dataset = BaseImageDataset( args.val_file, args.image_size, root=os.path.dirname(args.val_file), dtype=chainer.get_dtype(), ) train_iter = MultiprocessIterator(train_dataset, batch_size=args.batch_size, shuffle=True) validation_iter = MultiprocessIterator(validation_dataset, batch_size=args.batch_size, repeat=False) net = HandwritingNet() model = L.Classifier(net, label_key='has_text') tensorboard_handle = SummaryWriter(log_dir=args.log_dir) optimizer = Adam(alpha=args.learning_rate) optimizer.setup(model) if args.save_gradient_information: optimizer.add_hook( TensorboardGradientPlotter(tensorboard_handle, args.log_interval), ) # log train information everytime we encouter a new epoch or args.log_interval iterations have been done log_interval_trigger = ( lambda trainer: (trainer.updater.is_new_epoch or trainer.updater.iteration % args.log_interval == 0) and trainer.updater.iteration > 0 ) updater = StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = Trainer(updater, (args.num_epoch, 'epoch'), out=args.log_dir) data_to_log = { 'log_dir': args.log_dir, 'image_size': args.image_size, # 'num_layers': args.num_layers, 'keep_aspect_ratio': train_dataset.keep_aspect_ratio, 'net': get_import_info(net), } for argument in filter(lambda x: not x.startswith('_'), dir(args)): data_to_log[argument] = getattr(args, argument) def backup_train_config(stats_cpu): iteration = stats_cpu.pop('iteration') epoch = stats_cpu.pop('epoch') elapsed_time = stats_cpu.pop('elapsed_time') for key, value in stats_cpu.items(): tensorboard_handle.add_scalar(key, value, iteration) if iteration == args.log_interval: stats_cpu.update(data_to_log) stats_cpu.update({ "epoch": epoch, "iteration": iteration, "elapsed_time": elapsed_time, }) trainer.extend( extensions.snapshot_object(net, net.__class__.__name__ + '_{.updater.iteration}.npz'), trigger=lambda trainer: trainer.updater.is_new_epoch or trainer.updater.iteration % args.snapshot_interval == 0, ) trainer.extend( extensions.snapshot(filename='trainer_snapshot', autoload=args.resume is not None), trigger=(args.snapshot_interval, 'iteration') ) trainer.extend( TensorboardEvaluator( validation_iter, model, device=args.gpu, tensorboard_handle=tensorboard_handle ), trigger=(args.test_interval, 'iteration'), ) logger = Logger( os.path.dirname(os.path.realpath(__file__)), args.log_dir, postprocess=backup_train_config, trigger=log_interval_trigger, exclusion_filters=['*logs*', '*.pyc', '__pycache__', '.git*'], resume=args.resume is not None, ) trainer.extend(logger, trigger=log_interval_trigger) trainer.extend( extensions.PrintReport( ['epoch', 'iteration', 'main/loss', 'main/accuracy', 'validation/main/accuracy'], log_report='Logger', ), trigger=log_interval_trigger, ) trainer.extend(extensions.ExponentialShift('alpha', 0.1, optimizer=optimizer), trigger=(10, 'epoch')) trainer.extend(extensions.ProgressBar(update_interval=10)) trainer.run()
class Config(object): ''' This config is for single modality BM classification ''' def __init__(self): ##The top config #self.data_root = '/media/hhy/data/USdata/MergePhase1/test_0.3' #self.log_dir = '/media/hhy/data/code_results/MILs/MIL_H_Attention' self.root = '/remote-home/my/Ultrasound_CV/data/Ruijin/clean' self.log_dir = '/remote-home/my/hhy/Ultrasound_MIL/experiments/PLN1/base/fold4' if not os.path.exists(self.log_dir): os.makedirs(self.log_dir) ##training config self.lr = 1e-4 self.epoch = 50 self.resume = -1 self.batch_size = 1 self.net = Attention() self.net.cuda() self.optimizer = Adam(self.net.parameters(), lr=self.lr) self.lrsch = torch.optim.lr_scheduler.MultiStepLR( self.optimizer, milestones=[10, 30, 50, 70], gamma=0.5) self.logger = Logger(self.log_dir) self.train_transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.RandomResizedCrop((224, 224)), transforms.RandomHorizontalFlip(0.5), transforms.RandomVerticalFlip(0.5), transforms.ColorJitter(0.25, 0.25, 0.25, 0.25), transforms.ToTensor() ]) self.test_transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) self.label_name = "手术淋巴结情况(0未转移;1转移)" self.trainbag = RuijinBags(self.root, [0, 1, 2, 3], self.train_transform, label_name=self.label_name) self.testbag = RuijinBags(self.root, [4], self.test_transform, label_name=self.label_name) train_label_list = list( map(lambda x: int(x['label']), self.trainbag.patient_info)) pos_ratio = sum(train_label_list) / len(train_label_list) print(pos_ratio) train_weight = [(1 - pos_ratio) if x > 0 else pos_ratio for x in train_label_list] # self.train_sampler = WeightedRandomSampler(weights=train_weight, num_samples=len(self.trainbag)) self.train_loader = DataLoader(self.trainbag, batch_size=self.batch_size, num_workers=8, shuffle=True) self.val_loader = DataLoader(self.testbag, batch_size=self.batch_size, shuffle=False, num_workers=8) if self.resume > 0: self.net, self.optimizer, self.lrsch, self.loss, self.global_step = self.logger.load( self.net, self.optimizer, self.lrsch, self.loss, self.resume) else: self.global_step = 0 # self.trainer = MTTrainer(self.net, self.optimizer, self.lrsch, self.loss, self.train_loader, self.val_loader, self.logger, self.global_step, mode=2) self.trainer = MILTrainer(self.net, self.optimizer, self.lrsch, None, self.train_loader, self.val_loader, self.logger, self.global_step) def update(self, dicts): for k, v in dicts.items(): self.__setattr__(k, v) @staticmethod def auto_argparser(cfg, description=None): """Generate argparser from config file automatically (experimental) """ # partial_parser = ArgumentParser(description=description) # partial_parser.add_argument('config', help='config file path') # parsed, unparsed_args = partial_parser.parse_known_args() # cfg = Config.fromfile(parsed.config) parser = ArgumentParser(description=description) add_args(parser, cfg) parsed_args = parser.parse_args() update_cfg(cfg, parsed_args) return cfg
#!/usr/bin/env python3 import os from utils.logger import Logger from utils.config import Config from utils.environment import Env from utils.arguments import get_arguments __appName__ = 'TrailerTech' __author__ = 'JsAddiction' __version__ = '0.1.0' __description__ = 'Download Trailers for your movie library.' CONFIG_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'settings.ini') LOG_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'TrailerTech.log') env = Env() args = get_arguments(__appName__, __description__, __version__) config = Config(CONFIG_PATH) logger = Logger(LOG_PATH, config.log_level, config.log_to_file, quiet=args.quiet)
from asyncio import sleep, gather, create_task from bs4 import BeautifulSoup as bs4 from random import SystemRandom import models.errors as errors from models.pydantic import BookInfo from utils.logger import Logger logger = Logger.get(__file__) class WebPage: """ Представляет собой Web-страницу """ __headers = { 'Host' : 'elibrary.asu.ru', 'Connection' : 'close', 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36', 'Accept' : 'image/webp,image/apng,image/*,*/*;q=0.8', 'Accept-Encoding' : 'gzip, deflate', 'Accept-Language' : 'en-GB,en;q=0.9,ru-RU;q=0.8,ru;q=0.7,en-US;q=0.6' } def __init__(self, session, link): """ session = aiohttp.Session link = link for book page """ self.session = session
def train(args): # Configuration os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') input_height, input_width = args.input_size logger = Logger(log_root='logs/', name=args.logger_name) for k, v in args.__dict__.items(): logger.add_text('configuration', "{}: {}".format(k, v)) # Dataset train_loader, val_loader = get_data_loaders(args) batchs_in_val = math.ceil(len(val_loader.dataset) / args.validate_batch) print("Train set size:", len(train_loader.dataset)) print("Val set size:", len(val_loader.dataset)) # Network if args.use_noise is True: noise_layers = { 'crop': '((0.4,0.55),(0.4,0.55))', 'cropout': '((0.25,0.35),(0.25,0.35))', 'dropout': '(0.25,0.35)', 'jpeg': '()', 'resize': '(0.4,0.6)', } # This is a combined noise used in the paper else: noise_layers = dict() encoder = Encoder(input_height, input_width, args.info_size) noiser = Noiser(noise_layers, torch.device('cuda')) decoder = Decoder(args.info_size) discriminator = Discriminator() encoder.cuda() noiser.cuda() decoder.cuda() discriminator.cuda() # Optimizers optimizer_enc = torch.optim.Adam(encoder.parameters()) optimizer_dec = torch.optim.Adam(decoder.parameters()) optimizer_dis = torch.optim.Adam(discriminator.parameters()) # Training dir_save = 'ckpt/{}'.format(logger.log_name) os.makedirs(dir_save, exist_ok=True) os.makedirs(dir_save + '/images/', exist_ok=True) os.makedirs(dir_save + '/models/', exist_ok=True) training_losses = defaultdict(AverageLoss) info_fix = torch.randint(0, 2, size=(100, args.info_size)).to(device, dtype=torch.float32) image_fix = None for image, _ in val_loader: image_fix = image.cuda() # 100 images for validate, the first batch break global_step = 1 for epoch in range(1, args.epochs + 1): # Train one epoch for image, _ in train_loader: image = image.cuda() batch_size = image.shape[0] info = torch.randint(0, 2, size=(batch_size, args.info_size)).to(device, dtype=torch.float32) encoder.train() noiser.train() decoder.train() discriminator.train() # ---------------- Train the discriminator ----------------------------- optimizer_dis.zero_grad() # train on cover y_real = torch.ones(batch_size, 1).cuda() y_fake = torch.zeros(batch_size, 1).cuda() d_on_cover = discriminator(image) encoded_image = encoder(image, info) d_on_encoded = discriminator(encoded_image.detach()) if args.relative_loss: d_loss_on_cover = F.binary_cross_entropy_with_logits(d_on_cover - torch.mean(d_on_encoded), y_real) d_loss_on_encoded = F.binary_cross_entropy_with_logits(d_on_encoded - torch.mean(d_on_cover), y_fake) d_loss = d_loss_on_cover + d_loss_on_encoded else: d_loss_on_cover = F.binary_cross_entropy_with_logits(d_on_cover, y_real) d_loss_on_encoded = F.binary_cross_entropy_with_logits(d_on_encoded, y_fake) d_loss = d_loss_on_cover + d_loss_on_encoded d_loss.backward() optimizer_dis.step() # --------------Train the generator (encoder-decoder) --------------------- optimizer_enc.zero_grad() optimizer_dec.zero_grad() d_on_cover = discriminator(image) encoded_image = encoder(image, info) noised_and_cover = noiser([encoded_image, image]) noised_image = noised_and_cover[0] decoded_info = decoder(noised_image) d_on_encoded = discriminator(encoded_image) if args.relative_loss: g_loss_adv = \ (F.binary_cross_entropy_with_logits(d_on_encoded - torch.mean(d_on_cover), y_real) + F.binary_cross_entropy_with_logits(d_on_cover - torch.mean(d_on_encoded), y_fake)) * 0.5 g_loss_enc = F.mse_loss(encoded_image, image) g_loss_dec = F.mse_loss(decoded_info, info) else: g_loss_adv = F.binary_cross_entropy_with_logits(d_on_encoded, y_real) g_loss_enc = F.mse_loss(encoded_image, image) g_loss_dec = F.mse_loss(decoded_info, info) g_loss = args.adversarial_loss_constant * g_loss_adv + \ args.encoder_loss_constant * g_loss_enc + \ args.decoder_loss_constant * g_loss_dec g_loss.backward() optimizer_enc.step() optimizer_dec.step() decoded_rounded = decoded_info.detach().cpu().numpy().round().clip(0, 1) bitwise_avg_err = \ np.sum(np.abs(decoded_rounded - info.detach().cpu().numpy())) / \ (batch_size * info.shape[1]) losses = { 'g_loss': g_loss.item(), 'g_loss_enc': g_loss_enc.item(), 'g_loss_dec': g_loss_dec.item(), 'bitwise_avg_error': bitwise_avg_err, 'g_loss_adv': g_loss_adv.item(), 'd_loss_on_cover': d_loss_on_cover.item(), 'd_loss_on_encoded': d_loss_on_encoded.item(), 'd_loss': d_loss.item() } if logger: for name, loss in losses.items(): logger.add_scalar(name + '_iter', loss, global_step) training_losses[name].update(loss) global_step += 1 if logger: logger.add_scalar('d_loss_epoch', training_losses['d_loss'].avg, epoch) logger.add_scalar('g_loss_epoch', training_losses['g_loss'].avg, epoch) # Validate each epoch info_random = torch.randint(0, 2, size=(100, args.info_size)).to(device, dtype=torch.float32) image_random = None choice = random.randint(0, batchs_in_val - 2) # print(choice) for i, (image, _) in enumerate(val_loader): if i < choice: continue if image.shape[0] < 100: continue image_random = image.cuda() # Grub the first batch break encoder.eval() noiser.eval() decoder.eval() discriminator.eval() encoded_image_random = encoder(image_random, info_random) noised_and_cover_random = noiser([encoded_image_random, image_random]) noised_image_random = noised_and_cover_random[0] decoded_info_random = decoder(noised_image_random) encoded_image_fix = encoder(image_fix, info_fix) noised_and_cover_fix = noiser([encoded_image_fix, image_fix]) noised_image_fix = noised_and_cover_fix[0] decoded_info_fix = decoder(noised_image_fix) decoded_rounded_fix = decoded_info_fix.detach().cpu().numpy().round().clip(0, 1) bitwise_avg_err_fix = \ np.sum(np.abs(decoded_rounded_fix - info_fix.detach().cpu().numpy())) / \ (100 * info_fix.shape[1]) decoded_rounded_random = decoded_info_random.detach().cpu().numpy().round().clip(0, 1) bitwise_avg_err_random = \ np.sum(np.abs(decoded_rounded_random - info_random.detach().cpu().numpy())) / \ (100 * info_random.shape[1]) stack_image_random = exec_val(image_random, encoded_image_random, os.path.join(dir_save, 'images', 'random_epoch{:0>3d}.png'.format(epoch))) stack_image_fix = exec_val(image_fix, encoded_image_fix, os.path.join(dir_save, 'images', 'fix_epoch{:0>3d}.png'.format(epoch))) if logger: logger.add_scalar('fix_err_ratio', bitwise_avg_err_fix, epoch) logger.add_scalar('random_err_ratio', bitwise_avg_err_random, epoch) logger.add_image('image_rand', stack_image_random, epoch) logger.add_image('image_fix', stack_image_fix, epoch) torch.save(encoder.state_dict(), '{}/models/encoder-epoch{:0>3d}.pth'.format(dir_save, epoch)) torch.save(decoder.state_dict(), '{}/models/decoder-epoch{:0>3d}.pth'.format(dir_save, epoch)) if args.use_noise: torch.save(noiser.state_dict(), '{}/models/noiser-epoch{:0>3d}.pth'.format(dir_save, epoch)) torch.save(discriminator.state_dict(), '{}/models/discriminator-epoch{:0>3d}.pth'.format(dir_save, epoch))
def initialize_logger(config_file): Logger().load_configs(config_file)