def meta_train_epoch(self, epoch):
self.model.to(self.device)
self.model.train()
self.mtrloader.dataset.set_mode('train')
dlen = len(self.mtrloader.dataset)
trloss = 0
y_all, y_pred_all =[], []
pbar = tqdm(self.mtrloader)
for x, g, acc in pbar:
self.optimizer.zero_grad()
y_pred = self.forward(x, g)
y = acc.to(self.device)
loss = self.model.mseloss(y_pred, y.unsqueeze(-1))
loss.backward()
self.optimizer.step()
y = y.tolist()
y_pred = y_pred.squeeze().tolist()
y_all += y
y_pred_all += y_pred
pbar.set_description(get_log(
epoch, loss, y_pred, y, self.acc_std, self.acc_mean))
trloss += float(loss)
return trloss/dlen, pearsonr(np.array(y_all),
np.array(y_pred_all))[0]
def meta_validation(self, epoch):
self.model.to(self.device)
self.model.eval()
valoss = 0
self.mtrloader.dataset.set_mode('valid')
dlen = len(self.mtrloader.dataset)
y_all, y_pred_all =[], []
pbar = tqdm(self.mtrloader)
with torch.no_grad():
for x, g, acc in pbar:
y_pred = self.forward(x, g)
y = acc.to(self.device)
loss = self.model.mseloss(y_pred, y.unsqueeze(-1))
y = y.tolist()
y_pred = y_pred.squeeze().tolist()
y_all += y
y_pred_all += y_pred
pbar.set_description(get_log(
epoch, loss, y_pred, y, self.acc_std, self.acc_mean, tag='val'))
valoss += float(loss)
return valoss/dlen, pearsonr(np.array(y_all),
np.array(y_pred_all))[0]
def initialize():
"""
Initialize static-gen environment.
"""
config = generate_config()
os.system("mkdir -p " + config['project_name'])
os.system("cd " + config['project_name'])
log = get_log(config['project_name'] + "/" + config['logfile'])
with open(config['project_name'] + "/config.yml", "w") as config_file:
yaml.dump(config, config_file, default_flow_style=False)
log.info("Created config file")
os.system("mkdir -p " + config['project_name'] + "/" +
config['asset_directory'])
log.info("Created asset directory")
os.system("mkdir -p " + config['project_name'] + "/" +
config['template_directory'])
log.info("Created template directory")
os.system("mkdir -p " + config['project_name'] + "/" +
config['static_directory'])
log.info("Created static directory")
os.system("mkdir -p " + config['project_name'] + "/" +
config['cms_directory'])
log.info("Created cms directory")
def __init__(
self,
api_key=None,
timeout=10,
mixins=[],
access_token=None,
refresh_token=None,
client_id=None,
**extra_options
):
super(BaseClient, self).__init__()
# reverse so that the first one in the list because the first parent
mixins.reverse()
for mixin_class in mixins:
if mixin_class not in self.__class__.__bases__:
self.__class__.__bases__ = (mixin_class,) + self.__class__.__bases__
self.api_key = api_key or extra_options.get("api_key")
self.access_token = access_token or extra_options.get("access_token")
self.refresh_token = refresh_token or extra_options.get("refresh_token")
self.client_id = client_id or extra_options.get("client_id")
self.log = utils.get_log("hapipy")
if self.api_key and self.access_token:
raise Exception("Cannot use both api_key and access_token.")
if not (self.api_key or self.access_token or self.refresh_token):
raise Exception("Missing required credentials.")
self.options = {"api_base": "api.hubapi.com"}
if not _PYTHON25:
self.options["timeout"] = timeout
self.options.update(extra_options)
self._prepare_connection_type()
def __init__(self,
api_key=None,
timeout=10,
mixins=[],
access_token=None,
refresh_token=None,
client_id=None,
**extra_options):
super(BaseClient, self).__init__()
# reverse so that the first one in the list because the first parent
mixins.reverse()
for mixin_class in mixins:
if mixin_class not in self.__class__.__bases__:
self.__class__.__bases__ = (
mixin_class, ) + self.__class__.__bases__
self.api_key = api_key or extra_options.get('api_key')
self.access_token = access_token or extra_options.get('access_token')
self.refresh_token = refresh_token or extra_options.get(
'refresh_token')
self.client_id = client_id or extra_options.get('client_id')
self.log = utils.get_log('hapipy')
if self.api_key and self.access_token:
raise Exception("Cannot use both api_key and access_token.")
if not (self.api_key or self.access_token or self.refresh_token):
raise Exception("Missing required credentials.")
self.options = {'api_base': 'api.hubapi.com'}
if not _PYTHON25:
self.options['timeout'] = timeout
self.options.update(extra_options)
self._prepare_connection_type()
def main():
log = get_log()
# Load datasets
feature_train1, target_train1 = get_dataset(hyp['train_dataset1'],
transform=False,
scaler=False)
feature_test1, target_test1 = get_dataset(hyp['test_dataset1'],
transform=False,
scaler=False)
feature_train2, target_train2 = get_dataset(hyp['train_dataset2'],
transform=False,
scaler=False)
feature_test2, target_test2 = get_dataset(hyp['test_dataset2'],
transform=False,
scaler=False)
feature_train_val = feature_train1 + feature_test1 + feature_train2 + feature_test2
target_train_val = target_train1 + target_test1 + target_train2 + target_test2
ALL_features = feature_train1 + feature_train2
ALL_labels = target_train1 + target_train2
# Define model
param_grid = {
'learning_rate': 0.1,
'num_boost_round': 150,
'boosting_type': 'gbdt',
'objective': 'regression',
'max_depth': 6,
'num_leaves': 45,
'bagging_fraction': 0.9,
'bagging_freq': 5,
# 'reg_alpha': 0.04,
'reg_lambda': 0.12,
'metric': 'rmse'
}
# Model
# model = LinearRegression()
# model = DecisionTreeRegressor()
# model = RandomForestRegressor()
model = lgb.LGBMRegressor(**param_grid)
# Train
model.fit(ALL_features, ALL_labels)
# Predict
predict_results = model.predict(feature_test1)
log.info(((predict_results > 0) * (predict_results < 0.6)).sum())
for metric in [R2, MAE, MSE, EVS]:
score = metric(target_test1, predict_results)
log.info(metric.__name__ + ': ' + str(score))
# Save
joblib.dump(
model,
'/home/twsf/work/CRGNet/density_tools/weights/gbm_{}_64.pkl'.format(
dataset.lower()))
def search_log(request):
"""
@summary:日志查询接口
"""
form = SearchLogForm(request.POST)
if not form.is_valid():
return render_json({"result": False, "data": {}, "message": u"参数错误"})
key = form.cleaned_data["key"]
start = form.cleaned_data["start"]
app_id = form.cleaned_data["app_id"]
is_success, error_code, data = get_log(form.cleaned_data["size"], start,
key, request.COOKIES['bk_token'],
app_id)
if key and start == 0:
save_search_history(request.user, app_id, key, data.get("took"),
data.get("total") if data else 0)
if is_success:
return render_json({"result": True, "data": data, "message": ""})
else:
result = False
if error_code == "20001":
config_add_check = Config.objects.filter(app_id=app_id,
publish_status=0,
publish_type=0)
config_off_check = Config.objects.filter(app_id=app_id,
publish_status=0,
publish_type=1)
if config_off_check.count() < config_add_check.count():
message = u"查询接口出错，请联系管理员查看"
error_code = "20002"
else:
message = u"请创建采集项！"
else:
message = u"请修改查询关键词重试!"
result = True
return render_json({
"result": result,
"data": {
"total": 0,
"took": 0,
"log_list": []
},
"message": u"查询失败，%s" % message,
"code": error_code
})
async def run(self, client):
for guild in client.guilds:
print(guild)
with ILock(str(guild.id)):
activity_log = read_activity_log(guild.id)
for member in guild.members:
if not member.bot:
if member.voice:
print(f"{member} está conectado. Progreso guardado.")
log = get_log(member.voice)
if log["type"] == "JOINED":
if activity_log[str(member)][-1]["type"] == "ONLINE":
activity_log[str(
member)][-1]["timestamp"] = time()
else:
log["type"] = "ONLINE"
activity_log[str(member)].append(log)
save_activity_log(guild.id, activity_log)
def app():
st.markdown("# Plots ")
workout = get_log()
exercises = st.multiselect("Exercise", list(workout))
if exercises:
#
for exercise in exercises:
fig = px.scatter(x=workout[exercise].index,
y=workout[exercise].values)
fig.update_layout(
# title="Plot Title",
xaxis_title="Date",
yaxis_title=exercise[0],
font=dict(family="Courier New, monospace",
size=18,
color="#7f7f7f"),
)
st.plotly_chart(fig)
def meta_validation(self, epoch):
self.model.to(self.device)
self.model.eval()
valoss = 0
self.mtrloader.dataset.set_mode('valid')
dlen = len(self.mtrloader.dataset)
y_all, y_pred_all = [], []
pbar = tqdm(self.mtrloader)
with torch.no_grad():
for batch in pbar:
batch_loss = 0
y_batch, y_pred_batch = [], []
for x, g, acc in batch:
y_pred = self.forward(x, decode_ofa_mbv3_to_igraph(g))
y = acc.to(self.device)
batch_loss += self.model.mseloss(y_pred, y)
y = y.squeeze().tolist()
y_pred = y_pred.squeeze().tolist()
y_batch.append(y)
y_pred_batch.append(y_pred)
y_all.append(y)
y_pred_all.append(y_pred)
valoss += float(batch_loss)
pbar.set_description(
get_log(epoch,
batch_loss,
y_pred_batch,
y_batch,
self.acc_std,
self.acc_mean,
tag='val'))
return valoss / dlen, pearsonr(np.array(y_all),
np.array(y_pred_all))[0]
def export_log(request):
"""
@summary:日志下载接口
"""
key = request.GET.get('key', '')
start = request.GET.get('start', '')
app_id = request.GET.get('app_id', '')
size = request.GET.get('size', '')
is_success, error_code, data = get_log(size, start, key,
request.COOKIES['bk_token'], app_id)
file_abs = "log_%s.txt" % app_id
def file_iterator(file_name, chunk_size=512):
with open(file_name) as f:
while True:
c = f.read(chunk_size)
if c:
yield c
else:
break
with open(file_abs, "wt") as f:
for k in data.get("log_list"):
log_k = [
k.get("moduleName"), " ",
str(k.get("tagTime")), " ",
k.get("log").encode("utf-8"), " ",
k.get("serverIp"), " ",
k.get("localTime"), "\n"
]
f.writelines(log_k)
f.writelines("Finish it")
f.close()
response = StreamingHttpResponse(file_iterator(file_abs),
content_type='text/plain')
response['Content-Type'] = 'application/octet-stream'
response['Content-Disposition'] = 'attachment;filename="{0}"'.format(
file_abs)
return response
def meta_train_epoch(self, epoch):
self.model.to(self.device)
self.model.train()
self.mtrloader.dataset.set_mode('train')
dlen = len(self.mtrloader.dataset)
trloss = 0
y_all, y_pred_all = [], []
pbar = tqdm(self.mtrloader)
for batch in pbar:
batch_loss = 0
y_batch, y_pred_batch = [], []
self.optimizer.zero_grad()
for x, g, acc in batch:
y_pred = self.forward(x, decode_ofa_mbv3_to_igraph(g))
y = acc.to(self.device)
batch_loss += self.model.mseloss(y_pred, y)
y = y.squeeze().tolist()
y_pred = y_pred.squeeze().tolist()
y_batch.append(y)
y_pred_batch.append(y_pred)
y_all.append(y)
y_pred_all.append(y_pred)
batch_loss.backward()
trloss += float(batch_loss)
self.optimizer.step()
pbar.set_description(
get_log(epoch, batch_loss, y_pred_batch, y_batch, self.acc_std,
self.acc_mean))
return trloss / dlen, pearsonr(np.array(y_all),
np.array(y_pred_all))[0]
def __init__(self, api_key=None, timeout=10, mixins=[], access_token=None,
refresh_token=None, client_id=None, password=None, username=None,
portal_id=None, **extra_options):
super(BaseClient, self).__init__()
# reverse so that the first one in the list because the first parent
mixins.reverse()
for mixin_class in mixins:
if mixin_class not in self.__class__.__bases__:
self.__class__.__bases__ = (mixin_class,) + self.__class__.__bases__
self.api_key = api_key or extra_options.get('api_key')
self.access_token = access_token or extra_options.get('access_token')
self.refresh_token = refresh_token or extra_options.get('refresh_token')
self.client_id = client_id or extra_options.get('client_id')
self.portal_id = portal_id or extra_options.get('portal_id')
self.password = password or extra_options.get('password')
self.username = username or extra_options.get('username')
self.log = utils.get_log('happy')
if not ((self.api_key or self.access_token or self.refresh_token) and self.portal_id):
raise Exception("Missing required credentials.")
self.options = {
'api_base': 'api.hubapi.com',
'portal_id': portal_id
}
if client_id and portal_id and password and username:
tokens = self._new_refresh_token(username, password, portal_id, client_id)
self.refresh_token = tokens['refresh_token']
self.access_token = tokens['access_token']
if not _PYTHON25:
self.options['timeout'] = timeout
self.options.update(extra_options)
self._prepare_connection_type()
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and#
# limitations under the License.
import re
import time
from utils import get_log
LOG = get_log(__name__)
# Maximum Bytes Per Packet
CHUNK_SIZE = 512 * 1024 # B
class FileLikeProxy:
def __init__(self, transfer_object, callback, speed_limit='1mb'):
self.__callback = callback if callback else lambda size, length, obj_id, name: True
self.resp = transfer_object['resource'].get_ref_image(
transfer_object['id'])
self.length = (self.resp.length
if self.resp.length else transfer_object['size'])
self.id = transfer_object['id']
self.name = transfer_object['name']
self.percent = self.length / 100
import torch.optim as optim
import torch.nn as nn
from utils import accuracy, writeOFFfile, get_log,
from models import GAE, GCNencoder, GCNdecoder
import visdom
from PairGraphDataset import GraphDataset
# ------------------------------------------------------------------------------------------
sample_dir = 'reconst'
check_point = 'check_points'
first_train_tag = True
first_val_tag = True
logger = get_log('log.txt')
# Create a directory if not exists
if not os.path.exists(sample_dir):
os.makedirs(sample_dir)
if not os.path.exists(check_point):
os.makedirs(check_point)
# Device configuration
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
# Training settings
parser = argparse.ArgumentParser()
parser.add_argument('--no-cuda',
action='store_true',
default=False,
def train_model(train_dataset, train_num_each, val_dataset, val_num_each):
if if_load_old == True:
pdb.set_trace()
print("please choose the previous one")
time_cur = '1586310709.4848218'
else:
time_cur = time.time()
writer = SummaryWriter(summary_dir + str(time_cur))
logger = utils.get_log('log/' + str(time_cur) + '.txt')
# num_train = len(train_dataset)
# num_val = len(val_dataset)
train_useful_start_idx = get_useful_start_idx(sequence_length,
train_num_each)
val_useful_start_idx = get_useful_start_idx(sequence_length, val_num_each)
# train_idx = []
# for i in range(num_train_we_use):
# for j in range(sequence_length):
# train_idx.append(train_we_use_start_idx[i] + j)
val_idx = []
for i in range(len(val_useful_start_idx)):
for j in range(sequence_length):
val_idx.append(val_useful_start_idx[i] + j)
# num_train_all = len(train_idx)
num_val_all = len(val_idx)
# print('num of train dataset: {:6d}'.format(num_train))
# print('num train start idx : {:6d}'.format(len(train_useful_start_idx)))
# print('last idx train start: {:6d}'.format(train_useful_start_idx[-1]))
# print('num of train we use : {:6d}'.format(num_train_we_use))
# print('num of all train use: {:6d}'.format(num_train_all))
# print('num of valid dataset: {:6d}'.format(num_val))
# print('num valid start idx : {:6d}'.format(len(val_useful_start_idx)))
# print('last idx valid start: {:6d}'.format(val_useful_start_idx[-1]))
# print('num of valid we use : {:6d}'.format(num_val_we_use))
# print('num of all valid use: {:6d}'.format(num_val_all))
val_loader = DataLoader(val_dataset,
batch_size=val_batch_size,
sampler=SeqSampler(val_dataset, val_idx),
num_workers=workers,
pin_memory=True)
#select data to train
X = train_useful_start_idx
select_num = math.floor(len(X) * quary_portion) #every time choose 10%
if is_first_selection is True:
pdb.set_trace()
print("this is first selectin!!!! please check your parameter in .sh")
import random
mask = [1 for n in range(0, len(X))]
selected = random.sample(X, select_num)
for i in range(len(X)):
if X[i] in selected:
mask[i] = 0
unselected = [X[i] for i in range(len(X)) if X[i] not in selected]
save_select_data(save_select_txt_path, selected, unselected, mask,
time_cur)
else:
# load_select_data return: data['selected'],data['unselected'],data['mask']
selected, unselected, mask = load_select_data(
os.path.join(save_select_txt_path, json_name))
if select_chose == 'non_local':
print("this is non_local select")
test_idx = []
for i in range(len(unselected)):
for j in range(sequence_length):
test_idx.append(unselected[i] + j)
num_test_all = len(test_idx)
subset = Subset(train_dataset, test_idx)
selected, unselected, mask = non_local_select(
val_model_path, subset, sequence_length, X, select_num,
selected, unselected, mask)
elif select_chose == 'DBN':
print("this is DBN select")
test_idx = []
for i in range(len(unselected)):
for j in range(sequence_length):
test_idx.append(unselected[i] + j)
num_test_all = len(test_idx)
subset = Subset(train_dataset, test_idx)
selected, unselected, mask = DBN_select(val_model_path, subset,
sequence_length, X,
select_num, selected,
unselected, mask)
elif select_chose == 'random':
print("this is random select")
test_idx = []
for i in range(len(unselected)):
for j in range(sequence_length):
test_idx.append(unselected[i] + j)
num_test_all = len(test_idx)
selected, unselected, mask = random_select_data(
X, select_num, selected, unselected, mask)
pdb.set_trace()
selected = [
selected[i] for i in range(len(selected))
if selected[i] in test_idx
]
else:
print(
"just using old load select data to train without select new data"
)
# pdb.set_trace()
if is_save_json is True:
save_select_data(save_select_txt_path, selected, unselected, mask,
time_cur)
pdb.set_trace()
# save_dir = save_dir_base + '/' + str(time_cur) + '_' + str(learning_rate) + '_tbs' + str(train_batch_size) \
# + '_seq' + str(sequence_length) + '_opt' + str(optimizer_choice) + '_crop' + str(crop_type) + '_adjlr' \
# + '_adamgamma' + str(adamgamma) + '_adamstep' + str(adam_step) + '_weight_decay' + str(adamweightdecay) + '_block_num' + str(block_num)
if train_mode == 'RESLSTM' or train_mode == 'RESLSTM_DBN':
save_dir = save_dir_base + '/' + str(train_mode) + '/' + str(time_cur) + 'txtname' + json_name + '_' + str(learning_rate) + '_tbs' + str(train_batch_size) \
+ '_seq' + str(sequence_length) + '_opt' + str(optimizer_choice) + '_crop' + str(crop_type) \
+ '_sgdstep' + str(sgd_step) + '_sgd_gamma' + str(sgd_gamma) + '_sgd_adjust_lr' + str(sgd_adjust_lr)+ '_weight_decay' + str(weight_decay)
elif train_mode == 'RESLSTM_NOLOCAL' or train_mode == 'RESLSTM_NOLOCAL_dropout0.2':
save_dir = save_dir_base + '/' + str(train_mode) + '/' + str(time_cur) + 'txtname' + json_name + '_' + str(learning_rate) + '_tbs' + str(train_batch_size) \
+ '_seq' + str(sequence_length) + '_opt' + str(optimizer_choice) + '_crop' + str(crop_type) \
+ '_adamgamma' + str(adamgamma) + '_adamstep' + str(adam_step) + '_adamweightdecay' + str(adamweightdecay) + '_block_num' + str(block_num)
if if_load_old == True:
# Check if a checkpoint is in there
if len([name for name in os.listdir(save_dir)]) > 0:
print("Loading old model")
else:
print("nothing to load")
pdb.set_trace()
else:
os.makedirs(save_dir)
if train_mode == 'RESLSTM':
model = resnet_lstm()
elif train_mode == 'RESLSTM_NOLOCAL':
model = resnet_lstm_nonlocal()
elif train_mode == 'RESLSTM_NOLOCAL_dropout0.2':
model = resnet_lstm_nonlocal()
chk = 'results_ResLSTM_Nolocal/RESLSTM_NOLOCAL/1572847215.642195txtname42974_1572767025.1601517.json_0.0005_tbs400_seq10_opt1_crop0_adamgamma0.1_adamstep3_adamweightdecay0.0001_block_num1/checkpoint_best-23.pt'
print("Restoring: ", chk)
# Load
state = torch.load(chk)
# newdict = {}
# for k,v in state['state_dict'].items():
# if k[0:7] != 'module.':
# name = 'module.' + k
# newdict[name] = v
# else:
# newdict[k] = v
model.load_state_dict(state['state_dict'])
elif train_mode == 'RESLSTM_DBN':
model = resnet_lstm_dropout()
else:
print("not implemented")
pdb.set_trace()
# print (model)
# pdb.set_trace()
if use_gpu:
model = DataParallel(model)
model.to(device)
criterion = nn.CrossEntropyLoss(size_average=False)
optimizer = None
exp_lr_scheduler = None
if multi_optim == 0:
if optimizer_choice == 0:
optimizer = optim.SGD(model.parameters(),
lr=learning_rate,
momentum=momentum,
dampening=dampening,
weight_decay=weight_decay,
nesterov=use_nesterov)
if sgd_adjust_lr == 0:
exp_lr_scheduler = lr_scheduler.StepLR(optimizer,
step_size=sgd_step,
gamma=sgd_gamma)
elif sgd_adjust_lr == 1:
exp_lr_scheduler = lr_scheduler.ReduceLROnPlateau(
optimizer, 'min')
elif optimizer_choice == 1:
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
elif multi_optim == 1:
if optimizer_choice == 0:
optimizer = optim.SGD([
{
'params': model.module.share.parameters()
},
{
'params': model.module.lstm.parameters(),
'lr': learning_rate
},
{
'params': model.module.fc.parameters(),
'lr': learning_rate
},
],
lr=learning_rate / 10,
momentum=momentum,
dampening=dampening,
weight_decay=weight_decay,
nesterov=use_nesterov)
if sgd_adjust_lr == 0:
exp_lr_scheduler = lr_scheduler.StepLR(optimizer,
step_size=sgd_step,
gamma=sgd_gamma)
elif sgd_adjust_lr == 1:
exp_lr_scheduler = lr_scheduler.ReduceLROnPlateau(
optimizer, 'min')
elif optimizer_choice == 1:
# optimizer = optim.Adam([
# {'params': model.module.share.parameters()},
# {'params': model.module.lstm.parameters(), 'lr': learning_rate},
# {'params': model.module.fc.parameters(), 'lr': learning_rate},
# ], lr=learning_rate / 10)
optim_params = list(
filter(lambda p: p.requires_grad, model.parameters()))
print('Optimizing %d paramters' % len(optim_params))
optimizer = optim.Adam(optim_params,
lr=learning_rate,
weight_decay=adamweightdecay)
exp_lr_scheduler = lr_scheduler.StepLR(optimizer,
step_size=adam_step,
gamma=adamgamma)
#check if need load old weigth, optimizer
if if_load_old:
# Find last, not last best checkpoint
files = glob(save_dir + '/*')
global_steps = np.zeros([len(files)])
for i in range(len(files)):
# Use meta files to find the highest index
if 'best' in files[i]:
continue
if 'checkpoint-' not in files[i]:
continue
# Extract global step
nums = [int(s) for s in re.findall(r'\d+', files[i])]
global_steps[i] = nums[-1]
# Create path with maximum global step found
chkPath = save_dir + '/checkpoint-' + str(int(
np.max(global_steps))) + '.pt'
print("Restoring: ", chkPath)
# Load
state = torch.load(chkPath)
# Initialize model and optimizer
newdict = {}
for k, v in state['state_dict'].items():
if k[0:7] != 'module.':
name = 'module.' + k
newdict[name] = v
else:
newdict[k] = v
model.load_state_dict(newdict)
# model.load_state_dict(state['state_dict'])
optimizer.load_state_dict(state['optimizer'])
# pdb.set_trace()
start_epoch = state['epoch']
best_epoch = int(np.max(global_steps))
best_val_accuracy = state['best_val_accuracy']
correspond_train_acc = state['correspond_train_acc']
else:
start_epoch = 1
best_epoch = -1
best_val_accuracy = 0.0
correspond_train_acc = 0.0
if sv_init_model is not None:
print("Restoring supervised model: ", sv_init_model)
# Load
state = torch.load(sv_init_model)
# Initialize model and optimizer
newdict = {}
for k, v in state['state_dict'].items():
if k[0:7] != 'module.':
name = 'module.' + k
newdict[name] = v
else:
newdict[k] = v
model.load_state_dict(newdict)
best_model_wts = copy.deepcopy(model.module.state_dict())
for epoch in range(start_epoch, epochs + 1):
np.random.shuffle(selected)
train_idx = []
for i in range(len(selected)):
for j in range(sequence_length):
train_idx.append(selected[i] + j)
num_train_all = len(train_idx)
# subset = Subset(train_dataset,train_idx)
# train_loader = DataLoader(
# subset,
# batch_size=train_batch_size,
# sampler=SeqSampler(subset, train_idx),
# num_workers=workers,
# pin_memory=True
# )
train_loader = DataLoader(train_dataset,
batch_size=train_batch_size,
sampler=SeqSampler(train_dataset, train_idx),
num_workers=workers,
pin_memory=True)
# pdb.set_trace()
# Sets the module in training mode.
model.train()
train_loss = 0.0
train_corrects = 0
batch_progress = 0.0
train_start_time = time.time()
for data in train_loader:
optimizer.zero_grad()
# torch.cuda.empty_cache()
with torch.set_grad_enabled(True):
if use_gpu:
inputs, labels = data[0].to(device), data[1].to(device)
labels = labels[(sequence_length - 1)::sequence_length]
else:
inputs, labels = data[0], data[1]
labels = labels[(sequence_length - 1)::sequence_length]
# pdb.set_trace()
inputs = inputs.view(-1, sequence_length, 3, 224, 224)
# pdb.set_trace()
outputs = model.forward(inputs)
# pdb.set_trace()
outputs = outputs[sequence_length - 1::sequence_length]
_, preds = torch.max(outputs.data, 1)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
train_loss += loss.data.item()
batch_corrects = torch.sum(preds == labels.data)
train_corrects += batch_corrects
batch_acc = float(
batch_corrects) / train_batch_size * sequence_length
batch_progress += 1
if batch_progress * train_batch_size >= num_train_all:
percent = 100.0
print('Batch progress: %s [%d/%d] Batch acc:%.2f' %
(str(percent) + '%', num_train_all, num_train_all,
batch_acc),
end='\n')
else:
percent = round(
batch_progress * train_batch_size / num_train_all *
100, 2)
print('Batch progress: %s [%d/%d] Batch acc:%.2f' %
(str(percent) + '%', batch_progress *
train_batch_size, num_train_all, batch_acc),
end='\r')
train_elapsed_time = time.time() - train_start_time
train_accuracy = float(train_corrects) / float(
num_train_all) * sequence_length
train_average_loss = train_loss / num_train_all * sequence_length
# Sets the module in evaluation mode.
model.eval()
val_loss = 0.0
val_corrects = 0
val_start_time = time.time()
val_progress = 0
with torch.no_grad():
for data in val_loader:
# torch.cuda.empty_cache()
if use_gpu:
inputs, labels = data[0].to(device), data[1].to(device)
labels = labels[(sequence_length - 1)::sequence_length]
else:
inputs, labels = data[0], data[1]
labels = labels[(sequence_length - 1)::sequence_length]
if crop_type == 0 or crop_type == 1:
inputs = inputs.view(-1, sequence_length, 3, 224, 224)
outputs = model.forward(inputs)
elif crop_type == 5:
inputs = inputs.permute(1, 0, 2, 3, 4).contiguous()
inputs = inputs.view(-1, 3, 224, 224)
outputs = model.forward(inputs)
outputs = outputs.view(5, -1, 7)
outputs = torch.mean(outputs, 0)
elif crop_type == 10:
inputs = inputs.permute(1, 0, 2, 3, 4).contiguous()
inputs = inputs.view(-1, 3, 224, 224)
outputs = model.forward(inputs)
outputs = outputs.view(10, -1, 7)
outputs = torch.mean(outputs, 0)
outputs = outputs[sequence_length - 1::sequence_length]
_, preds = torch.max(outputs.data, 1)
loss = criterion(outputs, labels)
val_loss += loss.data.item()
val_corrects += torch.sum(preds == labels.data)
val_progress += 1
if val_progress * val_batch_size >= num_val_all:
percent = 100.0
print('Val progress: %s [%d/%d]' %
(str(percent) + '%', num_val_all, num_val_all),
end='\n')
else:
percent = round(
val_progress * val_batch_size / num_val_all * 100, 2)
print('Val progress: %s [%d/%d]' %
(str(percent) + '%', val_progress * val_batch_size,
num_val_all),
end='\r')
val_elapsed_time = time.time() - val_start_time
val_accuracy = float(val_corrects) / float(
num_val_all) * sequence_length
val_average_loss = val_loss / num_val_all * sequence_length
write_dict = {
"train_loss": train_average_loss,
"val_loss": val_average_loss,
"train_accuracy": train_accuracy,
"val_accuracy": val_accuracy
}
writer.add_scalars('scalar', write_dict, epoch)
if optimizer_choice == 0:
if sgd_adjust_lr == 0:
exp_lr_scheduler.step()
elif sgd_adjust_lr == 1:
exp_lr_scheduler.step(val_average_loss)
if optimizer_choice == 1:
exp_lr_scheduler.step()
if val_accuracy >= best_val_accuracy:
if val_accuracy > best_val_accuracy:
best_val_accuracy = val_accuracy
correspond_train_acc = train_accuracy
best_model_wts = copy.deepcopy(model.module.state_dict())
oldBestInd = best_epoch
best_epoch = epoch
if val_accuracy == best_val_accuracy:
if train_accuracy > correspond_train_acc:
correspond_train_acc = train_accuracy
best_model_wts = copy.deepcopy(model.module.state_dict())
oldBestInd = best_epoch
best_epoch = epoch
# Delte previously best model
if os.path.isfile(save_dir + '/checkpoint_best-' +
str(oldBestInd) + '.pt'):
os.remove(save_dir + '/checkpoint_best-' + str(oldBestInd) +
'.pt')
# Save currently best model
state = {
'epoch': epoch,
'state_dict': best_model_wts,
'optimizer': optimizer.state_dict(),
'best_val_accuracy': best_val_accuracy,
'correspond_train_acc': correspond_train_acc
}
torch.save(state,
save_dir + '/checkpoint_best-' + str(epoch) + '.pt')
# If its not better, just save it delete the last checkpoint if it is not current best one
# Save current model
state = {
'epoch': epoch,
'state_dict': model.module.state_dict(),
'optimizer': optimizer.state_dict(),
'best_val_accuracy': best_val_accuracy,
'correspond_train_acc': correspond_train_acc
}
torch.save(state, save_dir + '/checkpoint-' + str(epoch) + '.pt')
# Delete last one
if os.path.isfile(save_dir + '/checkpoint-' + str(epoch - 1) + '.pt'):
os.remove(save_dir + '/checkpoint-' + str(epoch - 1) + '.pt')
logger.info("\n")
logger.info('Epoch: %d/%d (%d h %d m %d s)' %
(epoch, epochs, int(train_elapsed_time / 3600),
int(np.mod(train_elapsed_time, 3600) / 60),
int(np.mod(np.mod(train_elapsed_time, 3600), 60))) +
time.strftime("%d.%m.-%H:%M:%S", time.localtime()))
logger.info('validation time: %d h %d m %d s' %
(int(val_elapsed_time / 3600),
int(np.mod(val_elapsed_time, 3600) / 60),
int(np.mod(np.mod(val_elapsed_time, 3600), 60))) +
time.strftime("%d.%m.-%H:%M:%S", time.localtime()))
logger.info("training loss: %6f" % train_average_loss)
logger.info("validation loss: %6f" % val_average_loss)
logger.info("train accu: %6f" % train_accuracy)
logger.info("validation accu: %6f" % val_accuracy)
logger.info("best val accu: %6f at Epoch %d" %
(best_val_accuracy, best_epoch))
logger.info("best corresponding train accu: %6f" %
correspond_train_acc)
writer.close()
from lxml import etree
from pyquery import PyQuery as pq
from models import JsonEntity, Article, Account, Backpack, Ftp
from config import get_mysql_new, GETCAPTCHA_URL, mongo_conn, ADD_COLLECTION, GET_ACCOUNT_FROM_MYSQL, JUDEG
from utils import uploads_mysql, get_log, GetDrver, driver, get_captcha_path, time_strftime, save_name, abuyun_proxy
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import WebDriverException
from PIL import Image
from io import BytesIO
from verification_code import captch_upload_image
current_dir = os.getcwd()
log = get_log('daily_collect')
class AccountHttp(object):
def __init__(self):
self.url = 'https://weixin.sogou.com/weixin?type=1&s_from=input&query={}&ie=utf8&_sug_=n&_sug_type_='
self.account = ''
self.name = ''
self.search_name = ''
self.tags = ''
self.s = requests.Session()
self.s.keep_alive = False # 关闭多余连接
self.s.adapters.DEFAULT_RETRIES = 5 # 增加重连次数
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
nmp_layers = 4
elif args.dataset_name == 'hotel':
args.num_iterations = 7437
nmp_layers = 1
elif args.dataset_name == 'univ':
args.num_iterations = 5540
nmp_layers = 5
elif args.dataset_name == 'zara1':
args.num_iterations = 6318
nmp_layers = 3
elif args.dataset_name == 'zara2':
args.num_iterations = 5987
nmp_layers = 4
logger = get_log(
os.path.join(sys.path[0], 'logs', args.checkpoint_name + '.txt'),
__file__.split('/')[0])
def init_weights(m):
classname = m.__class__.__name__
if classname.find('Linear') != -1:
nn.init.kaiming_normal_(m.weight)
def get_dtypes(args):
long_dtype = torch.LongTensor
float_dtype = torch.FloatTensor
if args.use_gpu == 1:
long_dtype = torch.cuda.LongTensor
float_dtype = torch.cuda.FloatTensor
entry['it'] *= batch_size
if args.list_keys > 0:
keys = list_log_keys(hist[exp_name])
print 'keys in file %s (experiment "%s"):' % (fname, exp_name)
for k in keys:
print ' %s' % k
if args.list_keys > 0:
sys.exit(0)
# sanity check for what_dict
for k in what_dict:
found = False
for exp_name in hist:
try:
h = get_log(what_dict[k][0], hist[exp_name])
if len(h[0]) > 0:
found = True
break
except BaseException:
pass
if not found:
print 'WARNING: key %s=%s configured but not found in any history' % (
k, what_dict[k][0])
# prepare plot
if args.grad > 0:
fig, ax = plt.subplots(2, 1)
ax0 = ax[0]
else:
fig, ax = plt.subplots(1, 1)
import os
import random
import re
import time
from ftplib import FTP
import requests
from config import mongo_conn
from utils import get_log
from pyquery import PyQuery as pq
import datetime
import hashlib
import uuid
import zipfile
from utils import abuyun_proxy
log = get_log('models').info
class Article(object):
def __init__(self):
self.url = ''
self.account = ''
self.title = ''
self.content = ''
# 作者即公众号名称name
self.author = ''
self._from = ''
self.time = ''
self.image_url = ''
self.readnum = ''
self.likenum = ''
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and#
# limitations under the License.
import re
import time
from utils import get_log
LOG = get_log(__name__)
# Maximum Bytes Per Packet
CHUNK_SIZE = 512 * 1024 # B
class FileLikeProxy:
def __init__(self, transfer_object, callback, speed_limit='1mb'):
self.__callback = callback if callback else lambda size, length, obj_id, name: True
self.resp = transfer_object['resource'].get_ref_image(
transfer_object['id'])
self.length = (
self.resp.length if self.resp.length else transfer_object['size'])
self.id = transfer_object['id']
self.name = transfer_object['name']
self.percent = self.length / 100
def __init__(self):
self.config = yaml.load(open("config.yml"))
self.log = get_log(self.config['logfile'])
def info():
utils.log('Explorer up!!! Virtutoso ' + str(query.memoized_query_sparql.cache_info()))
return utils.get_log()
from utils import accuracy, writeOFFfile, get_log, reprocess, smootherror
from models import GCNencoder, GCNdecoder , Discriminator,LSDiscriminator,ZDiscriminator
from PairGraphDataset import GraphDataset, PairGLightColorDataset
# ------------------------------------------------------------------------------------------
path_ae = 'fixedmodel'
check_point = 'checkpoints/gan'
generate_dir = 'generate'
first_tag = True
first_val_tag = True
logger = get_log('train-GANlog.txt')
# Create a directory if not exists
if not os.path.exists(path_ae):
os.makedirs(path_ae)
if not os.path.exists(generate_dir):
os.makedirs(generate_dir)
# Device configuration
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
# visdom
viz = visdom.Visdom(env='train-gan')
# Training settings
parser = argparse.ArgumentParser()
