def meta_train_epoch(self, epoch): self.model.to(self.device) self.model.train() self.mtrloader.dataset.set_mode('train') dlen = len(self.mtrloader.dataset) trloss = 0 y_all, y_pred_all =[], [] pbar = tqdm(self.mtrloader) for x, g, acc in pbar: self.optimizer.zero_grad() y_pred = self.forward(x, g) y = acc.to(self.device) loss = self.model.mseloss(y_pred, y.unsqueeze(-1)) loss.backward() self.optimizer.step() y = y.tolist() y_pred = y_pred.squeeze().tolist() y_all += y y_pred_all += y_pred pbar.set_description(get_log( epoch, loss, y_pred, y, self.acc_std, self.acc_mean)) trloss += float(loss) return trloss/dlen, pearsonr(np.array(y_all), np.array(y_pred_all))[0]
def meta_validation(self, epoch): self.model.to(self.device) self.model.eval() valoss = 0 self.mtrloader.dataset.set_mode('valid') dlen = len(self.mtrloader.dataset) y_all, y_pred_all =[], [] pbar = tqdm(self.mtrloader) with torch.no_grad(): for x, g, acc in pbar: y_pred = self.forward(x, g) y = acc.to(self.device) loss = self.model.mseloss(y_pred, y.unsqueeze(-1)) y = y.tolist() y_pred = y_pred.squeeze().tolist() y_all += y y_pred_all += y_pred pbar.set_description(get_log( epoch, loss, y_pred, y, self.acc_std, self.acc_mean, tag='val')) valoss += float(loss) return valoss/dlen, pearsonr(np.array(y_all), np.array(y_pred_all))[0]
def initialize(): """ Initialize static-gen environment. """ config = generate_config() os.system("mkdir -p " + config['project_name']) os.system("cd " + config['project_name']) log = get_log(config['project_name'] + "/" + config['logfile']) with open(config['project_name'] + "/config.yml", "w") as config_file: yaml.dump(config, config_file, default_flow_style=False) log.info("Created config file") os.system("mkdir -p " + config['project_name'] + "/" + config['asset_directory']) log.info("Created asset directory") os.system("mkdir -p " + config['project_name'] + "/" + config['template_directory']) log.info("Created template directory") os.system("mkdir -p " + config['project_name'] + "/" + config['static_directory']) log.info("Created static directory") os.system("mkdir -p " + config['project_name'] + "/" + config['cms_directory']) log.info("Created cms directory")
def __init__( self, api_key=None, timeout=10, mixins=[], access_token=None, refresh_token=None, client_id=None, **extra_options ): super(BaseClient, self).__init__() # reverse so that the first one in the list because the first parent mixins.reverse() for mixin_class in mixins: if mixin_class not in self.__class__.__bases__: self.__class__.__bases__ = (mixin_class,) + self.__class__.__bases__ self.api_key = api_key or extra_options.get("api_key") self.access_token = access_token or extra_options.get("access_token") self.refresh_token = refresh_token or extra_options.get("refresh_token") self.client_id = client_id or extra_options.get("client_id") self.log = utils.get_log("hapipy") if self.api_key and self.access_token: raise Exception("Cannot use both api_key and access_token.") if not (self.api_key or self.access_token or self.refresh_token): raise Exception("Missing required credentials.") self.options = {"api_base": "api.hubapi.com"} if not _PYTHON25: self.options["timeout"] = timeout self.options.update(extra_options) self._prepare_connection_type()
def __init__(self, api_key=None, timeout=10, mixins=[], access_token=None, refresh_token=None, client_id=None, **extra_options): super(BaseClient, self).__init__() # reverse so that the first one in the list because the first parent mixins.reverse() for mixin_class in mixins: if mixin_class not in self.__class__.__bases__: self.__class__.__bases__ = ( mixin_class, ) + self.__class__.__bases__ self.api_key = api_key or extra_options.get('api_key') self.access_token = access_token or extra_options.get('access_token') self.refresh_token = refresh_token or extra_options.get( 'refresh_token') self.client_id = client_id or extra_options.get('client_id') self.log = utils.get_log('hapipy') if self.api_key and self.access_token: raise Exception("Cannot use both api_key and access_token.") if not (self.api_key or self.access_token or self.refresh_token): raise Exception("Missing required credentials.") self.options = {'api_base': 'api.hubapi.com'} if not _PYTHON25: self.options['timeout'] = timeout self.options.update(extra_options) self._prepare_connection_type()
def main(): log = get_log() # Load datasets feature_train1, target_train1 = get_dataset(hyp['train_dataset1'], transform=False, scaler=False) feature_test1, target_test1 = get_dataset(hyp['test_dataset1'], transform=False, scaler=False) feature_train2, target_train2 = get_dataset(hyp['train_dataset2'], transform=False, scaler=False) feature_test2, target_test2 = get_dataset(hyp['test_dataset2'], transform=False, scaler=False) feature_train_val = feature_train1 + feature_test1 + feature_train2 + feature_test2 target_train_val = target_train1 + target_test1 + target_train2 + target_test2 ALL_features = feature_train1 + feature_train2 ALL_labels = target_train1 + target_train2 # Define model param_grid = { 'learning_rate': 0.1, 'num_boost_round': 150, 'boosting_type': 'gbdt', 'objective': 'regression', 'max_depth': 6, 'num_leaves': 45, 'bagging_fraction': 0.9, 'bagging_freq': 5, # 'reg_alpha': 0.04, 'reg_lambda': 0.12, 'metric': 'rmse' } # Model # model = LinearRegression() # model = DecisionTreeRegressor() # model = RandomForestRegressor() model = lgb.LGBMRegressor(**param_grid) # Train model.fit(ALL_features, ALL_labels) # Predict predict_results = model.predict(feature_test1) log.info(((predict_results > 0) * (predict_results < 0.6)).sum()) for metric in [R2, MAE, MSE, EVS]: score = metric(target_test1, predict_results) log.info(metric.__name__ + ': ' + str(score)) # Save joblib.dump( model, '/home/twsf/work/CRGNet/density_tools/weights/gbm_{}_64.pkl'.format( dataset.lower()))
def search_log(request): """ @summary:日志查询接口 """ form = SearchLogForm(request.POST) if not form.is_valid(): return render_json({"result": False, "data": {}, "message": u"参数错误"}) key = form.cleaned_data["key"] start = form.cleaned_data["start"] app_id = form.cleaned_data["app_id"] is_success, error_code, data = get_log(form.cleaned_data["size"], start, key, request.COOKIES['bk_token'], app_id) if key and start == 0: save_search_history(request.user, app_id, key, data.get("took"), data.get("total") if data else 0) if is_success: return render_json({"result": True, "data": data, "message": ""}) else: result = False if error_code == "20001": config_add_check = Config.objects.filter(app_id=app_id, publish_status=0, publish_type=0) config_off_check = Config.objects.filter(app_id=app_id, publish_status=0, publish_type=1) if config_off_check.count() < config_add_check.count(): message = u"查询接口出错,请联系管理员查看" error_code = "20002" else: message = u"请创建采集项!" else: message = u"请修改查询关键词重试!" result = True return render_json({ "result": result, "data": { "total": 0, "took": 0, "log_list": [] }, "message": u"查询失败,%s" % message, "code": error_code })
async def run(self, client): for guild in client.guilds: print(guild) with ILock(str(guild.id)): activity_log = read_activity_log(guild.id) for member in guild.members: if not member.bot: if member.voice: print(f"{member} está conectado. Progreso guardado.") log = get_log(member.voice) if log["type"] == "JOINED": if activity_log[str(member)][-1]["type"] == "ONLINE": activity_log[str( member)][-1]["timestamp"] = time() else: log["type"] = "ONLINE" activity_log[str(member)].append(log) save_activity_log(guild.id, activity_log)
def app(): st.markdown("# Plots ") workout = get_log() exercises = st.multiselect("Exercise", list(workout)) if exercises: # for exercise in exercises: fig = px.scatter(x=workout[exercise].index, y=workout[exercise].values) fig.update_layout( # title="Plot Title", xaxis_title="Date", yaxis_title=exercise[0], font=dict(family="Courier New, monospace", size=18, color="#7f7f7f"), ) st.plotly_chart(fig)
def meta_validation(self, epoch): self.model.to(self.device) self.model.eval() valoss = 0 self.mtrloader.dataset.set_mode('valid') dlen = len(self.mtrloader.dataset) y_all, y_pred_all = [], [] pbar = tqdm(self.mtrloader) with torch.no_grad(): for batch in pbar: batch_loss = 0 y_batch, y_pred_batch = [], [] for x, g, acc in batch: y_pred = self.forward(x, decode_ofa_mbv3_to_igraph(g)) y = acc.to(self.device) batch_loss += self.model.mseloss(y_pred, y) y = y.squeeze().tolist() y_pred = y_pred.squeeze().tolist() y_batch.append(y) y_pred_batch.append(y_pred) y_all.append(y) y_pred_all.append(y_pred) valoss += float(batch_loss) pbar.set_description( get_log(epoch, batch_loss, y_pred_batch, y_batch, self.acc_std, self.acc_mean, tag='val')) return valoss / dlen, pearsonr(np.array(y_all), np.array(y_pred_all))[0]
def export_log(request): """ @summary:日志下载接口 """ key = request.GET.get('key', '') start = request.GET.get('start', '') app_id = request.GET.get('app_id', '') size = request.GET.get('size', '') is_success, error_code, data = get_log(size, start, key, request.COOKIES['bk_token'], app_id) file_abs = "log_%s.txt" % app_id def file_iterator(file_name, chunk_size=512): with open(file_name) as f: while True: c = f.read(chunk_size) if c: yield c else: break with open(file_abs, "wt") as f: for k in data.get("log_list"): log_k = [ k.get("moduleName"), " ", str(k.get("tagTime")), " ", k.get("log").encode("utf-8"), " ", k.get("serverIp"), " ", k.get("localTime"), "\n" ] f.writelines(log_k) f.writelines("Finish it") f.close() response = StreamingHttpResponse(file_iterator(file_abs), content_type='text/plain') response['Content-Type'] = 'application/octet-stream' response['Content-Disposition'] = 'attachment;filename="{0}"'.format( file_abs) return response
def meta_train_epoch(self, epoch): self.model.to(self.device) self.model.train() self.mtrloader.dataset.set_mode('train') dlen = len(self.mtrloader.dataset) trloss = 0 y_all, y_pred_all = [], [] pbar = tqdm(self.mtrloader) for batch in pbar: batch_loss = 0 y_batch, y_pred_batch = [], [] self.optimizer.zero_grad() for x, g, acc in batch: y_pred = self.forward(x, decode_ofa_mbv3_to_igraph(g)) y = acc.to(self.device) batch_loss += self.model.mseloss(y_pred, y) y = y.squeeze().tolist() y_pred = y_pred.squeeze().tolist() y_batch.append(y) y_pred_batch.append(y_pred) y_all.append(y) y_pred_all.append(y_pred) batch_loss.backward() trloss += float(batch_loss) self.optimizer.step() pbar.set_description( get_log(epoch, batch_loss, y_pred_batch, y_batch, self.acc_std, self.acc_mean)) return trloss / dlen, pearsonr(np.array(y_all), np.array(y_pred_all))[0]
def __init__(self, api_key=None, timeout=10, mixins=[], access_token=None, refresh_token=None, client_id=None, password=None, username=None, portal_id=None, **extra_options): super(BaseClient, self).__init__() # reverse so that the first one in the list because the first parent mixins.reverse() for mixin_class in mixins: if mixin_class not in self.__class__.__bases__: self.__class__.__bases__ = (mixin_class,) + self.__class__.__bases__ self.api_key = api_key or extra_options.get('api_key') self.access_token = access_token or extra_options.get('access_token') self.refresh_token = refresh_token or extra_options.get('refresh_token') self.client_id = client_id or extra_options.get('client_id') self.portal_id = portal_id or extra_options.get('portal_id') self.password = password or extra_options.get('password') self.username = username or extra_options.get('username') self.log = utils.get_log('happy') if not ((self.api_key or self.access_token or self.refresh_token) and self.portal_id): raise Exception("Missing required credentials.") self.options = { 'api_base': 'api.hubapi.com', 'portal_id': portal_id } if client_id and portal_id and password and username: tokens = self._new_refresh_token(username, password, portal_id, client_id) self.refresh_token = tokens['refresh_token'] self.access_token = tokens['access_token'] if not _PYTHON25: self.options['timeout'] = timeout self.options.update(extra_options) self._prepare_connection_type()
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an AS IS BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and# # limitations under the License. import re import time from utils import get_log LOG = get_log(__name__) # Maximum Bytes Per Packet CHUNK_SIZE = 512 * 1024 # B class FileLikeProxy: def __init__(self, transfer_object, callback, speed_limit='1mb'): self.__callback = callback if callback else lambda size, length, obj_id, name: True self.resp = transfer_object['resource'].get_ref_image( transfer_object['id']) self.length = (self.resp.length if self.resp.length else transfer_object['size']) self.id = transfer_object['id'] self.name = transfer_object['name'] self.percent = self.length / 100
import torch.optim as optim import torch.nn as nn from utils import accuracy, writeOFFfile, get_log, from models import GAE, GCNencoder, GCNdecoder import visdom from PairGraphDataset import GraphDataset # ------------------------------------------------------------------------------------------ sample_dir = 'reconst' check_point = 'check_points' first_train_tag = True first_val_tag = True logger = get_log('log.txt') # Create a directory if not exists if not os.path.exists(sample_dir): os.makedirs(sample_dir) if not os.path.exists(check_point): os.makedirs(check_point) # Device configuration device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') # Training settings parser = argparse.ArgumentParser() parser.add_argument('--no-cuda', action='store_true', default=False,
def train_model(train_dataset, train_num_each, val_dataset, val_num_each): if if_load_old == True: pdb.set_trace() print("please choose the previous one") time_cur = '1586310709.4848218' else: time_cur = time.time() writer = SummaryWriter(summary_dir + str(time_cur)) logger = utils.get_log('log/' + str(time_cur) + '.txt') # num_train = len(train_dataset) # num_val = len(val_dataset) train_useful_start_idx = get_useful_start_idx(sequence_length, train_num_each) val_useful_start_idx = get_useful_start_idx(sequence_length, val_num_each) # train_idx = [] # for i in range(num_train_we_use): # for j in range(sequence_length): # train_idx.append(train_we_use_start_idx[i] + j) val_idx = [] for i in range(len(val_useful_start_idx)): for j in range(sequence_length): val_idx.append(val_useful_start_idx[i] + j) # num_train_all = len(train_idx) num_val_all = len(val_idx) # print('num of train dataset: {:6d}'.format(num_train)) # print('num train start idx : {:6d}'.format(len(train_useful_start_idx))) # print('last idx train start: {:6d}'.format(train_useful_start_idx[-1])) # print('num of train we use : {:6d}'.format(num_train_we_use)) # print('num of all train use: {:6d}'.format(num_train_all)) # print('num of valid dataset: {:6d}'.format(num_val)) # print('num valid start idx : {:6d}'.format(len(val_useful_start_idx))) # print('last idx valid start: {:6d}'.format(val_useful_start_idx[-1])) # print('num of valid we use : {:6d}'.format(num_val_we_use)) # print('num of all valid use: {:6d}'.format(num_val_all)) val_loader = DataLoader(val_dataset, batch_size=val_batch_size, sampler=SeqSampler(val_dataset, val_idx), num_workers=workers, pin_memory=True) #select data to train X = train_useful_start_idx select_num = math.floor(len(X) * quary_portion) #every time choose 10% if is_first_selection is True: pdb.set_trace() print("this is first selectin!!!! please check your parameter in .sh") import random mask = [1 for n in range(0, len(X))] selected = random.sample(X, select_num) for i in range(len(X)): if X[i] in selected: mask[i] = 0 unselected = [X[i] for i in range(len(X)) if X[i] not in selected] save_select_data(save_select_txt_path, selected, unselected, mask, time_cur) else: # load_select_data return: data['selected'],data['unselected'],data['mask'] selected, unselected, mask = load_select_data( os.path.join(save_select_txt_path, json_name)) if select_chose == 'non_local': print("this is non_local select") test_idx = [] for i in range(len(unselected)): for j in range(sequence_length): test_idx.append(unselected[i] + j) num_test_all = len(test_idx) subset = Subset(train_dataset, test_idx) selected, unselected, mask = non_local_select( val_model_path, subset, sequence_length, X, select_num, selected, unselected, mask) elif select_chose == 'DBN': print("this is DBN select") test_idx = [] for i in range(len(unselected)): for j in range(sequence_length): test_idx.append(unselected[i] + j) num_test_all = len(test_idx) subset = Subset(train_dataset, test_idx) selected, unselected, mask = DBN_select(val_model_path, subset, sequence_length, X, select_num, selected, unselected, mask) elif select_chose == 'random': print("this is random select") test_idx = [] for i in range(len(unselected)): for j in range(sequence_length): test_idx.append(unselected[i] + j) num_test_all = len(test_idx) selected, unselected, mask = random_select_data( X, select_num, selected, unselected, mask) pdb.set_trace() selected = [ selected[i] for i in range(len(selected)) if selected[i] in test_idx ] else: print( "just using old load select data to train without select new data" ) # pdb.set_trace() if is_save_json is True: save_select_data(save_select_txt_path, selected, unselected, mask, time_cur) pdb.set_trace() # save_dir = save_dir_base + '/' + str(time_cur) + '_' + str(learning_rate) + '_tbs' + str(train_batch_size) \ # + '_seq' + str(sequence_length) + '_opt' + str(optimizer_choice) + '_crop' + str(crop_type) + '_adjlr' \ # + '_adamgamma' + str(adamgamma) + '_adamstep' + str(adam_step) + '_weight_decay' + str(adamweightdecay) + '_block_num' + str(block_num) if train_mode == 'RESLSTM' or train_mode == 'RESLSTM_DBN': save_dir = save_dir_base + '/' + str(train_mode) + '/' + str(time_cur) + 'txtname' + json_name + '_' + str(learning_rate) + '_tbs' + str(train_batch_size) \ + '_seq' + str(sequence_length) + '_opt' + str(optimizer_choice) + '_crop' + str(crop_type) \ + '_sgdstep' + str(sgd_step) + '_sgd_gamma' + str(sgd_gamma) + '_sgd_adjust_lr' + str(sgd_adjust_lr)+ '_weight_decay' + str(weight_decay) elif train_mode == 'RESLSTM_NOLOCAL' or train_mode == 'RESLSTM_NOLOCAL_dropout0.2': save_dir = save_dir_base + '/' + str(train_mode) + '/' + str(time_cur) + 'txtname' + json_name + '_' + str(learning_rate) + '_tbs' + str(train_batch_size) \ + '_seq' + str(sequence_length) + '_opt' + str(optimizer_choice) + '_crop' + str(crop_type) \ + '_adamgamma' + str(adamgamma) + '_adamstep' + str(adam_step) + '_adamweightdecay' + str(adamweightdecay) + '_block_num' + str(block_num) if if_load_old == True: # Check if a checkpoint is in there if len([name for name in os.listdir(save_dir)]) > 0: print("Loading old model") else: print("nothing to load") pdb.set_trace() else: os.makedirs(save_dir) if train_mode == 'RESLSTM': model = resnet_lstm() elif train_mode == 'RESLSTM_NOLOCAL': model = resnet_lstm_nonlocal() elif train_mode == 'RESLSTM_NOLOCAL_dropout0.2': model = resnet_lstm_nonlocal() chk = 'results_ResLSTM_Nolocal/RESLSTM_NOLOCAL/1572847215.642195txtname42974_1572767025.1601517.json_0.0005_tbs400_seq10_opt1_crop0_adamgamma0.1_adamstep3_adamweightdecay0.0001_block_num1/checkpoint_best-23.pt' print("Restoring: ", chk) # Load state = torch.load(chk) # newdict = {} # for k,v in state['state_dict'].items(): # if k[0:7] != 'module.': # name = 'module.' + k # newdict[name] = v # else: # newdict[k] = v model.load_state_dict(state['state_dict']) elif train_mode == 'RESLSTM_DBN': model = resnet_lstm_dropout() else: print("not implemented") pdb.set_trace() # print (model) # pdb.set_trace() if use_gpu: model = DataParallel(model) model.to(device) criterion = nn.CrossEntropyLoss(size_average=False) optimizer = None exp_lr_scheduler = None if multi_optim == 0: if optimizer_choice == 0: optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=momentum, dampening=dampening, weight_decay=weight_decay, nesterov=use_nesterov) if sgd_adjust_lr == 0: exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=sgd_step, gamma=sgd_gamma) elif sgd_adjust_lr == 1: exp_lr_scheduler = lr_scheduler.ReduceLROnPlateau( optimizer, 'min') elif optimizer_choice == 1: optimizer = optim.Adam(model.parameters(), lr=learning_rate) elif multi_optim == 1: if optimizer_choice == 0: optimizer = optim.SGD([ { 'params': model.module.share.parameters() }, { 'params': model.module.lstm.parameters(), 'lr': learning_rate }, { 'params': model.module.fc.parameters(), 'lr': learning_rate }, ], lr=learning_rate / 10, momentum=momentum, dampening=dampening, weight_decay=weight_decay, nesterov=use_nesterov) if sgd_adjust_lr == 0: exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=sgd_step, gamma=sgd_gamma) elif sgd_adjust_lr == 1: exp_lr_scheduler = lr_scheduler.ReduceLROnPlateau( optimizer, 'min') elif optimizer_choice == 1: # optimizer = optim.Adam([ # {'params': model.module.share.parameters()}, # {'params': model.module.lstm.parameters(), 'lr': learning_rate}, # {'params': model.module.fc.parameters(), 'lr': learning_rate}, # ], lr=learning_rate / 10) optim_params = list( filter(lambda p: p.requires_grad, model.parameters())) print('Optimizing %d paramters' % len(optim_params)) optimizer = optim.Adam(optim_params, lr=learning_rate, weight_decay=adamweightdecay) exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=adam_step, gamma=adamgamma) #check if need load old weigth, optimizer if if_load_old: # Find last, not last best checkpoint files = glob(save_dir + '/*') global_steps = np.zeros([len(files)]) for i in range(len(files)): # Use meta files to find the highest index if 'best' in files[i]: continue if 'checkpoint-' not in files[i]: continue # Extract global step nums = [int(s) for s in re.findall(r'\d+', files[i])] global_steps[i] = nums[-1] # Create path with maximum global step found chkPath = save_dir + '/checkpoint-' + str(int( np.max(global_steps))) + '.pt' print("Restoring: ", chkPath) # Load state = torch.load(chkPath) # Initialize model and optimizer newdict = {} for k, v in state['state_dict'].items(): if k[0:7] != 'module.': name = 'module.' + k newdict[name] = v else: newdict[k] = v model.load_state_dict(newdict) # model.load_state_dict(state['state_dict']) optimizer.load_state_dict(state['optimizer']) # pdb.set_trace() start_epoch = state['epoch'] best_epoch = int(np.max(global_steps)) best_val_accuracy = state['best_val_accuracy'] correspond_train_acc = state['correspond_train_acc'] else: start_epoch = 1 best_epoch = -1 best_val_accuracy = 0.0 correspond_train_acc = 0.0 if sv_init_model is not None: print("Restoring supervised model: ", sv_init_model) # Load state = torch.load(sv_init_model) # Initialize model and optimizer newdict = {} for k, v in state['state_dict'].items(): if k[0:7] != 'module.': name = 'module.' + k newdict[name] = v else: newdict[k] = v model.load_state_dict(newdict) best_model_wts = copy.deepcopy(model.module.state_dict()) for epoch in range(start_epoch, epochs + 1): np.random.shuffle(selected) train_idx = [] for i in range(len(selected)): for j in range(sequence_length): train_idx.append(selected[i] + j) num_train_all = len(train_idx) # subset = Subset(train_dataset,train_idx) # train_loader = DataLoader( # subset, # batch_size=train_batch_size, # sampler=SeqSampler(subset, train_idx), # num_workers=workers, # pin_memory=True # ) train_loader = DataLoader(train_dataset, batch_size=train_batch_size, sampler=SeqSampler(train_dataset, train_idx), num_workers=workers, pin_memory=True) # pdb.set_trace() # Sets the module in training mode. model.train() train_loss = 0.0 train_corrects = 0 batch_progress = 0.0 train_start_time = time.time() for data in train_loader: optimizer.zero_grad() # torch.cuda.empty_cache() with torch.set_grad_enabled(True): if use_gpu: inputs, labels = data[0].to(device), data[1].to(device) labels = labels[(sequence_length - 1)::sequence_length] else: inputs, labels = data[0], data[1] labels = labels[(sequence_length - 1)::sequence_length] # pdb.set_trace() inputs = inputs.view(-1, sequence_length, 3, 224, 224) # pdb.set_trace() outputs = model.forward(inputs) # pdb.set_trace() outputs = outputs[sequence_length - 1::sequence_length] _, preds = torch.max(outputs.data, 1) loss = criterion(outputs, labels) loss.backward() optimizer.step() train_loss += loss.data.item() batch_corrects = torch.sum(preds == labels.data) train_corrects += batch_corrects batch_acc = float( batch_corrects) / train_batch_size * sequence_length batch_progress += 1 if batch_progress * train_batch_size >= num_train_all: percent = 100.0 print('Batch progress: %s [%d/%d] Batch acc:%.2f' % (str(percent) + '%', num_train_all, num_train_all, batch_acc), end='\n') else: percent = round( batch_progress * train_batch_size / num_train_all * 100, 2) print('Batch progress: %s [%d/%d] Batch acc:%.2f' % (str(percent) + '%', batch_progress * train_batch_size, num_train_all, batch_acc), end='\r') train_elapsed_time = time.time() - train_start_time train_accuracy = float(train_corrects) / float( num_train_all) * sequence_length train_average_loss = train_loss / num_train_all * sequence_length # Sets the module in evaluation mode. model.eval() val_loss = 0.0 val_corrects = 0 val_start_time = time.time() val_progress = 0 with torch.no_grad(): for data in val_loader: # torch.cuda.empty_cache() if use_gpu: inputs, labels = data[0].to(device), data[1].to(device) labels = labels[(sequence_length - 1)::sequence_length] else: inputs, labels = data[0], data[1] labels = labels[(sequence_length - 1)::sequence_length] if crop_type == 0 or crop_type == 1: inputs = inputs.view(-1, sequence_length, 3, 224, 224) outputs = model.forward(inputs) elif crop_type == 5: inputs = inputs.permute(1, 0, 2, 3, 4).contiguous() inputs = inputs.view(-1, 3, 224, 224) outputs = model.forward(inputs) outputs = outputs.view(5, -1, 7) outputs = torch.mean(outputs, 0) elif crop_type == 10: inputs = inputs.permute(1, 0, 2, 3, 4).contiguous() inputs = inputs.view(-1, 3, 224, 224) outputs = model.forward(inputs) outputs = outputs.view(10, -1, 7) outputs = torch.mean(outputs, 0) outputs = outputs[sequence_length - 1::sequence_length] _, preds = torch.max(outputs.data, 1) loss = criterion(outputs, labels) val_loss += loss.data.item() val_corrects += torch.sum(preds == labels.data) val_progress += 1 if val_progress * val_batch_size >= num_val_all: percent = 100.0 print('Val progress: %s [%d/%d]' % (str(percent) + '%', num_val_all, num_val_all), end='\n') else: percent = round( val_progress * val_batch_size / num_val_all * 100, 2) print('Val progress: %s [%d/%d]' % (str(percent) + '%', val_progress * val_batch_size, num_val_all), end='\r') val_elapsed_time = time.time() - val_start_time val_accuracy = float(val_corrects) / float( num_val_all) * sequence_length val_average_loss = val_loss / num_val_all * sequence_length write_dict = { "train_loss": train_average_loss, "val_loss": val_average_loss, "train_accuracy": train_accuracy, "val_accuracy": val_accuracy } writer.add_scalars('scalar', write_dict, epoch) if optimizer_choice == 0: if sgd_adjust_lr == 0: exp_lr_scheduler.step() elif sgd_adjust_lr == 1: exp_lr_scheduler.step(val_average_loss) if optimizer_choice == 1: exp_lr_scheduler.step() if val_accuracy >= best_val_accuracy: if val_accuracy > best_val_accuracy: best_val_accuracy = val_accuracy correspond_train_acc = train_accuracy best_model_wts = copy.deepcopy(model.module.state_dict()) oldBestInd = best_epoch best_epoch = epoch if val_accuracy == best_val_accuracy: if train_accuracy > correspond_train_acc: correspond_train_acc = train_accuracy best_model_wts = copy.deepcopy(model.module.state_dict()) oldBestInd = best_epoch best_epoch = epoch # Delte previously best model if os.path.isfile(save_dir + '/checkpoint_best-' + str(oldBestInd) + '.pt'): os.remove(save_dir + '/checkpoint_best-' + str(oldBestInd) + '.pt') # Save currently best model state = { 'epoch': epoch, 'state_dict': best_model_wts, 'optimizer': optimizer.state_dict(), 'best_val_accuracy': best_val_accuracy, 'correspond_train_acc': correspond_train_acc } torch.save(state, save_dir + '/checkpoint_best-' + str(epoch) + '.pt') # If its not better, just save it delete the last checkpoint if it is not current best one # Save current model state = { 'epoch': epoch, 'state_dict': model.module.state_dict(), 'optimizer': optimizer.state_dict(), 'best_val_accuracy': best_val_accuracy, 'correspond_train_acc': correspond_train_acc } torch.save(state, save_dir + '/checkpoint-' + str(epoch) + '.pt') # Delete last one if os.path.isfile(save_dir + '/checkpoint-' + str(epoch - 1) + '.pt'): os.remove(save_dir + '/checkpoint-' + str(epoch - 1) + '.pt') logger.info("\n") logger.info('Epoch: %d/%d (%d h %d m %d s)' % (epoch, epochs, int(train_elapsed_time / 3600), int(np.mod(train_elapsed_time, 3600) / 60), int(np.mod(np.mod(train_elapsed_time, 3600), 60))) + time.strftime("%d.%m.-%H:%M:%S", time.localtime())) logger.info('validation time: %d h %d m %d s' % (int(val_elapsed_time / 3600), int(np.mod(val_elapsed_time, 3600) / 60), int(np.mod(np.mod(val_elapsed_time, 3600), 60))) + time.strftime("%d.%m.-%H:%M:%S", time.localtime())) logger.info("training loss: %6f" % train_average_loss) logger.info("validation loss: %6f" % val_average_loss) logger.info("train accu: %6f" % train_accuracy) logger.info("validation accu: %6f" % val_accuracy) logger.info("best val accu: %6f at Epoch %d" % (best_val_accuracy, best_epoch)) logger.info("best corresponding train accu: %6f" % correspond_train_acc) writer.close()
from lxml import etree from pyquery import PyQuery as pq from models import JsonEntity, Article, Account, Backpack, Ftp from config import get_mysql_new, GETCAPTCHA_URL, mongo_conn, ADD_COLLECTION, GET_ACCOUNT_FROM_MYSQL, JUDEG from utils import uploads_mysql, get_log, GetDrver, driver, get_captcha_path, time_strftime, save_name, abuyun_proxy from selenium.webdriver.common.by import By from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.common.exceptions import WebDriverException from PIL import Image from io import BytesIO from verification_code import captch_upload_image current_dir = os.getcwd() log = get_log('daily_collect') class AccountHttp(object): def __init__(self): self.url = 'https://weixin.sogou.com/weixin?type=1&s_from=input&query={}&ie=utf8&_sug_=n&_sug_type_=' self.account = '' self.name = '' self.search_name = '' self.tags = '' self.s = requests.Session() self.s.keep_alive = False # 关闭多余连接 self.s.adapters.DEFAULT_RETRIES = 5 # 增加重连次数 self.headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
nmp_layers = 4 elif args.dataset_name == 'hotel': args.num_iterations = 7437 nmp_layers = 1 elif args.dataset_name == 'univ': args.num_iterations = 5540 nmp_layers = 5 elif args.dataset_name == 'zara1': args.num_iterations = 6318 nmp_layers = 3 elif args.dataset_name == 'zara2': args.num_iterations = 5987 nmp_layers = 4 logger = get_log( os.path.join(sys.path[0], 'logs', args.checkpoint_name + '.txt'), __file__.split('/')[0]) def init_weights(m): classname = m.__class__.__name__ if classname.find('Linear') != -1: nn.init.kaiming_normal_(m.weight) def get_dtypes(args): long_dtype = torch.LongTensor float_dtype = torch.FloatTensor if args.use_gpu == 1: long_dtype = torch.cuda.LongTensor float_dtype = torch.cuda.FloatTensor
entry['it'] *= batch_size if args.list_keys > 0: keys = list_log_keys(hist[exp_name]) print 'keys in file %s (experiment "%s"):' % (fname, exp_name) for k in keys: print ' %s' % k if args.list_keys > 0: sys.exit(0) # sanity check for what_dict for k in what_dict: found = False for exp_name in hist: try: h = get_log(what_dict[k][0], hist[exp_name]) if len(h[0]) > 0: found = True break except BaseException: pass if not found: print 'WARNING: key %s=%s configured but not found in any history' % ( k, what_dict[k][0]) # prepare plot if args.grad > 0: fig, ax = plt.subplots(2, 1) ax0 = ax[0] else: fig, ax = plt.subplots(1, 1)
import os import random import re import time from ftplib import FTP import requests from config import mongo_conn from utils import get_log from pyquery import PyQuery as pq import datetime import hashlib import uuid import zipfile from utils import abuyun_proxy log = get_log('models').info class Article(object): def __init__(self): self.url = '' self.account = '' self.title = '' self.content = '' # 作者即公众号名称name self.author = '' self._from = '' self.time = '' self.image_url = '' self.readnum = '' self.likenum = ''
# # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an AS IS BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and# # limitations under the License. import re import time from utils import get_log LOG = get_log(__name__) # Maximum Bytes Per Packet CHUNK_SIZE = 512 * 1024 # B class FileLikeProxy: def __init__(self, transfer_object, callback, speed_limit='1mb'): self.__callback = callback if callback else lambda size, length, obj_id, name: True self.resp = transfer_object['resource'].get_ref_image( transfer_object['id']) self.length = ( self.resp.length if self.resp.length else transfer_object['size']) self.id = transfer_object['id'] self.name = transfer_object['name'] self.percent = self.length / 100
def __init__(self): self.config = yaml.load(open("config.yml")) self.log = get_log(self.config['logfile'])
def info(): utils.log('Explorer up!!! Virtutoso ' + str(query.memoized_query_sparql.cache_info())) return utils.get_log()
from utils import accuracy, writeOFFfile, get_log, reprocess, smootherror from models import GCNencoder, GCNdecoder , Discriminator,LSDiscriminator,ZDiscriminator from PairGraphDataset import GraphDataset, PairGLightColorDataset # ------------------------------------------------------------------------------------------ path_ae = 'fixedmodel' check_point = 'checkpoints/gan' generate_dir = 'generate' first_tag = True first_val_tag = True logger = get_log('train-GANlog.txt') # Create a directory if not exists if not os.path.exists(path_ae): os.makedirs(path_ae) if not os.path.exists(generate_dir): os.makedirs(generate_dir) # Device configuration device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') # visdom viz = visdom.Visdom(env='train-gan') # Training settings parser = argparse.ArgumentParser()