def signal_handler(signal, frame):
'''
Close socket before exitting
'''
s.close()
utils.print_log('Stopping discovery server')
sys.exit(0)
def do_dispatch(self, session, request):
""" dispatch request to the relevant processor """
method = request['method']
params = request.get('params', [])
suffix = method.split('.')[-1]
if session is not None:
if suffix == 'subscribe':
session.subscribe_to_service(method, params)
prefix = request['method'].split('.')[0]
try:
p = self.processors[prefix]
except:
print_log("error: no processor for", prefix)
return
p.add_request(session, request)
if method in ['server.version']:
try:
session.version = params[0]
session.protocol_version = float(params[1])
except:
pass
def get_bookid(filename, fb2):
global _connect
# search bookid in fb2
if options.search_id and fb2 is not None:
find = xpath('/m:FictionBook/m:description/m:custom-info')
bookid = None
for e in find(fb2):
bid = e.get('librusec-book-id')
if bid is not None:
try:
bookid = int(bid)
except:
pass
else:
return bookid
# search bookid by filename
try:
bookid = int(filename)
except ValueError:
curs = _connect.cursor()
curs.execute("SELECT BookId FROM libbookold WHERE FileName = ?",
(filename,))
res = curs.fetchone()
if res is None:
print_log('ERROR: file not found in db:', filename, level=3)
return None
return res[0]
return bookid
def run_trial(trial_num, object_id, force, mass, friction, verbose=True):
change_object_dynamics(object_id, mass, friction)
start_x = p.getBasePositionAndOrientation(object_id)[0][0]
if verbose:
print('\n*************TRIAL %d**************' % trial_num)
print_log(object_id, force)
# log1_id = p.startStateLogging(p.STATE_LOGGING_GENERIC_ROBOT,
# 'data/pybullet_logs/log_object_dynamics_%s.bin' % str(trial_num).zfill(5))
# log2_id = p.startStateLogging(p.STATE_LOGGING_CONTACT_POINTS,
# 'data/pybullet_logs/log_contact_dynamics_%s.bin' % str(trial_num).zfill(5))
p.applyExternalForce(object_id,
-1,
force,
posObj=[0, 0, 0],
flags=p.LINK_FRAME)
p.stepSimulation()
while is_object_moving(object_id):
p.stepSimulation()
time.sleep(1 / 240)
return p.getBasePositionAndOrientation(
object_id)[0][0] - start_x # return total x-displacement
def train(train_loader, model, criterion, optimizer, log=None):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
model.train()
end_time = time.time()
for i, (input, label) in enumerate(tqdm(train_loader)):
data_time.update(time.time() - end_time)
if args.use_cuda:
label = label.cuda()
input = input.cuda()
with torch.no_grad():
input_var = torch.autograd.Variable(input)
label_var = torch.autograd.Variable(label)
output = model(input_var)
loss = criterion(output, label_var)
prec1, prec5 = accuracy(output.data, label, topk=(1, 5))
losses.update(loss.data, input.size(0))
top1.update(prec1, input.size(0))
top5.update(prec5, input.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
print_log(
' **Train** Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'.format(
top1=top1, top5=top5), log)
return top1.avg, losses.avg
def train_step1(args, state_info, Train_loader, Test_loader, Memory, criterion,
epoch):
cuda = True if torch.cuda.is_available() else False
FloatTensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor
LongTensor = torch.cuda.LongTensor if cuda else torch.LongTensor
state_info.model.train()
utils.print_log('Type, Epoch, Batch, total, percentage')
correct = torch.tensor(0, dtype=torch.float32)
total_Size = torch.tensor(0, dtype=torch.float32)
for it, (x, y, label) in enumerate(Train_loader):
x, y, label = to_var(x, FloatTensor), to_var(y, LongTensor), to_var(
label, LongTensor)
state_info.optim_model.zero_grad()
out, z = state_info.forward(args, x)
Memory.Batch_Insert(z, y, torch.ones(z.size(0)).cuda())
loss = criterion(out, y)
loss.backward(retain_graph=True)
state_info.optim_model.step()
_, pred = torch.max(out.data, 1)
correct += float(pred.eq(y.data).cpu().sum())
total_Size += float(x.size(0))
if it % 10 == 0:
utils.print_log('Init, {}, {}, {:.6f}, {:.3f}'.format(
epoch, it, loss.item(), 100. * correct / total_Size))
print('Init, {}, {}, {:.6f}, {:.3f}'.format(
epoch, it, loss.item(), 100. * correct / total_Size))
epoch_result = test(args, state_info, Test_loader, epoch)
return epoch_result
def test(self):
test_data = data.Dataset(self.config['test_data_file'], shuffle=False)
test_graph = tf.Graph()
with tf.Session(graph=test_graph) as sess:
model = models.RumourDetectModel(
embed_dim=self.config['embed_dim'],
vocab_size=self.config['vocab_size'],
sent_hidden_dims=self.config['sent_hidden_dims'],
branch_hidden_dims=self.config['branch_hidden_dims'],
sdqc_attn_dim=self.config['sdqc_attn_dim'],
veracity_attn_dim=self.config['veracity_attn_dim'],
sdqc_hidden_dim=self.config['sdqc_hidden_dim'],
veracity_hidden_dim=self.config['veracity_hidden_dim'],
embed_pret_file=self.config['embed_pret_file'],
dicts_file=self.config['dicts_file'],
keep_prob=1.0,
reuse=None)
model(is_train=False)
sess.run(tf.global_variables_initializer())
if self.config['embed_pret_file']:
model.embedder.init_pretrained_emb(sess)
saver = tf.train.Saver(max_to_keep=self.config['max_ckpts'])
ckpt_dir = os.path.dirname(self.config['ckpt'])
ckpt = tf.train.latest_checkpoint(ckpt_dir)
saver.restore(sess, ckpt)
utils.print_log('Testing ...')
batch_num = int(
math.floor(len(test_data.records) / self.config['batch_size']))
sdqc_corr, sdqc_total, veracity_corr, veracity_total = 0, 0, 0, 0
for _ in range(batch_num):
X, X_pret, Y_sdqc, Y_veracity, sent_length, branch_length = \
test_data.get_next(self.config['batch_size'])
c1, t1, c2, t2 = sess.run(
[
model.sdqc_correct_count, model.sdqc_total_count,
model.veracity_correct_count,
model.veracity_total_count
],
feed_dict={
model.word_ids: X,
model.word_ids_pret: X_pret,
model.sdqc_labels: Y_sdqc,
model.veracity_labels: Y_veracity,
model.sent_length: sent_length,
model.branch_length: branch_length,
})
sdqc_corr += c1
sdqc_total += t1
veracity_corr += c2
veracity_total += t2
utils.print_log(
'SDQC Task Acc = {}, Veracity Task Acc = {}'.format(
float(sdqc_corr) / sdqc_total,
float(veracity_corr) / veracity_total))
def let_us_go(url):
try:
print '\n\n'
# 获得帖子源码
post_content = get_url_content(url)
if not post_content:
raise IOError('%s 内容获取失败，检查请求状态' % url)
# 获得帖子标题
post_image_dir_name = dry_nga_title(get_post_title(post_content))
print '标题 : %s' % post_image_dir_name
# 获得帖子页数
post_pages = get_post_total_pages(post_content)
print '页数 : %s' % post_pages
# 生成帖子图片文件夹
img_path = make_post_image_dir(post_image_dir_name)
print '图片存放目录名 : %s' % img_path
# 存放已下载链接的txt文件名
record_file_name = img_path + post_image_dir_name + '.txt'
# 获得帖子所有图片http路径
img_links = fetch_post_image_links(url, post_pages)
# 删除已经下载过的图片链接
print_log('开始删除重复链接')
new_img_links = remove_repeat_img_links(record_file_name, img_links)
print_log('删除重复链接完毕')
print '下载图片数: %s' % len(new_img_links)
# 下载图片
download_images_from_link_list(new_img_links, img_path, record_file_name)
except IOError, e:
print e
def metrics(loader, size = 128):
# import tensorflow.keras.backend as K
#
# def L1(y_true, y_pred):
# return K.sqrt(K.mean(K.square(y_pred - y_true)))
model = tf.keras.models.load_model("log/20191130-003024/model-snap70000.h5")
adam = optimizers.Nadam()
model.compile(adam, 'logcosh', metrics=[tf.keras.metrics.RootMeanSquaredError(), tf.keras.metrics.MeanAbsoluteError()])
length = 10000
LOSS = np.zeros(shape=(length))
RMSE = np.zeros(shape=(length))
MAE = np.zeros(shape=(length))
for i in range(length):
x, y = loader.get_validation_batch(size)
loss, rmse, mae = model.test_on_batch(x, y)
LOSS[i] = np.mean(loss)
RMSE[i] = np.mean(rmse)
MAE[i] = np.mean(mae)
print_log(f"running {i}, LOSS = {LOSS[i]}, RMSE = {RMSE[i]}, MAE = {MAE[i]}")
LOSS = np.mean(LOSS)
RMSE = np.mean(RMSE)
MAE = np.mean(MAE)
print_log(f"LOSS = {LOSS}, RMSE = {RMSE}, MAE = {MAE}")
return LOSS, RMSE, MAE
def get_market_info(cnx, exist_markets):
result = []
sql = '''
select result from cmc_markets
'''
with cnx.cursor() as cursor:
cursor.execute(sql)
markets = cursor.fetchall()
for m in markets:
try:
jm = json.loads(m[0])
name = jm['name']
website = ''
twitter = ''
if name not in exist_markets:
website = jm['media'][0][0]
twitter = ''
if len(jm['media']) > 1 and 'twitter' in jm['media'][1][0]:
twitter = jm['media'][1][0]
result.append({
'name': jm['name'],
'cmc_url': jm['url'],
'website': website,
'twitter': twitter
})
except Exception as e:
print_log(e)
return result
def do_dispatch(self, session, request):
""" dispatch request to the relevant processor """
method = request["method"]
params = request.get("params", [])
suffix = method.split(".")[-1]
if session is not None:
if suffix == "subscribe":
session.subscribe_to_service(method, params)
# store session and id locally
request["id"] = self.store_session_id(session, request["id"])
prefix = request["method"].split(".")[0]
try:
p = self.processors[prefix]
except:
print_log("error: no processor for", prefix)
return
p.add_request(request)
if method in ["server.version"]:
session.version = params[0]
try:
session.protocol_version = float(params[1])
except:
pass
def validate(val_loader,
model,
criterion,
log=None,
writer=None,
embedding=False):
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
model.eval()
for i, (input, label) in enumerate(tqdm(val_loader)):
if args.use_cuda:
label = label.cuda()
input = input.cuda()
with torch.no_grad():
input_var = torch.autograd.Variable(input)
label_var = torch.autograd.Variable(label)
output = model(input_var)
loss = criterion(output, label_var)
prec1, prec5 = accuracy(output.data, label, topk=(1, 5))
losses.update(loss.data, input.size(0))
top1.update(prec1, input.size(0))
top5.update(prec5, input.size(0))
if embedding and writer is not None and i == 0:
out = torch.cat((output.cpu().data, torch.ones(len(output), 1)), 1)
writer.add_embedding(out,
metadata=label.data,
label_img=input.data)
print_log(
' **Test** Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}'.format(
top1=top1, top5=top5), log)
return top1.avg, losses.avg
def process(arg):
global not_deleted_list, update_time
curs = _connect.cursor()
res = curs.execute("SELECT BookId FROM libbook WHERE NOT (Deleted&1) and FileType = 'fb2' ")
not_deleted_list = curs.fetchall()
not_deleted_list = set([i[0] for i in not_deleted_list])
curs.execute('SELECT * FROM librusec')
update_time = curs.fetchone()[0]
for fn in walk(arg):
for ftype, z_filename, data in read_file(fn, zip_charset='utf-8'):
process_file(fn, ftype, z_filename, data)
if options.search_deleted:
deleted = set()
for fn in walk(options.search_deleted):
bookid = base_name(fn, '.fb2.zip')
try:
bookid = int(bookid)
except ValueError:
continue
if bookid in not_deleted_list:
deleted.append(fn)
for fn in deleted:
for ftype, z_filename, data in read_file(fn, zip_charset='utf-8'):
ret = process_file(fn, ftype, z_filename, data)
if ret:
print_log('restore deleted:', bookid)
print
print 'processed:', stats.total
print 'passed:', stats.passed
print 'fixed:', stats.fixed
print 'errors:', stats.errors
if options.not_found:
fd = open(options.not_found, 'w')
for bookid in not_deleted_list:
print >> fd, bookid
def write_TFRecord(self, filename, flag=None):
trackProgress = ProgressBar()
writer = tf.python_io.TFRecordWriter(
os.path.join(setting.config["data"]["tfrecord"]["directory"],
filename))
image_addrs = self.return_addrs(self.image_dir)
label_addrs = self.return_addrs(self.label_dir)
utils.print_log("Proceeding with writing tfrecords for " + flag,
"INFO")
for i in trackProgress(range(len(image_addrs))):
# image, label = self.load_image(image_addrs[i], label_addrs[i])
image, label, height, width, one_hot_label = self.process_image(
image_addrs[i], label_addrs[i])
# label, _, __ = self.process_image(label_addrs[i])
feature = {
'height': self.int64_feature(height),
'width': self.int64_feature(width),
'label': self.bytes_feature(label),
'image': self.bytes_feature(image),
'one_hot_label': self.bytes_feature(one_hot_label)
}
example = tf.train.Example(features=tf.train.Features(
feature=feature))
writer.write(example.SerializeToString())
writer.close()
sys.stdout.flush()
utils.print_log("Writing tfrecords is completed", "INFO")
def train(model, optimizer, criterion, train_loader, epoch, is_main):
model.train()
train_loss = 0
total = 0
correct = 0
for batch_idx, (data, target) in enumerate(train_loader):
if torch.cuda.is_available():
data, target = Variable(data.cuda()), Variable(target.cuda())
else:
data, target = Variable(data), Variable(target)
optimizer.zero_grad()
output = model(data)
loss = criterion(output, target)
loss.backward()
optimizer.step()
train_loss += loss.data[0]
_, predicted = torch.max(output.data, 1)
total += target.size(0)
correct += predicted.eq(target.data).cpu().sum()
if batch_idx % 10 == 0 and is_main is True:
utils.print_log('Epoch: {} | Batch: {} | Loss: ({:.4f}) | Acc: ({:.2f}%) ({}/{})'
.format(epoch, batch_idx, train_loss / (batch_idx + 1), 100. * correct / total, correct, total))
print('Epoch: {} | Batch: {} | Loss: ({:.4f}) | Acc: ({:.2f}%) ({}/{})'
.format(epoch, batch_idx, train_loss / (batch_idx + 1), 100. * correct / total, correct, total))
elif batch_idx % 10 == 0 and is_main is False:
utils.print_log('SWICH: {} | Batch: {} | Loss: ({:.4f}) | Acc: ({:.2f}%) ({}/{})'
.format(epoch, batch_idx, train_loss / (batch_idx + 1), 100. * correct / total, correct, total))
print('SWICH: {} | Batch: {} | Loss: ({:.4f}) | Acc: ({:.2f}%) ({}/{})'
.format(epoch, batch_idx, train_loss / (batch_idx + 1), 100. * correct / total, correct, total))
def run(self):
print_log( ("SSL" if self.use_ssl else "TCP") + " server started on port %d"%self.port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((self.host, self.port))
sock.listen(5)
while not self.shared.stopped():
#if self.use_ssl: print_log("SSL: socket listening")
try:
connection, address = sock.accept()
except:
traceback.print_exc(file=sys.stdout)
time.sleep(0.1)
continue
#if self.use_ssl: print_log("SSL: new session", address)
try:
session = TcpSession(self.dispatcher, connection, address, use_ssl=self.use_ssl, ssl_certfile=self.ssl_certfile, ssl_keyfile=self.ssl_keyfile)
except BaseException, e:
error = str(e)
print_log("cannot start TCP session", error, address)
connection.close()
time.sleep(0.1)
continue
client_req = TcpClientRequestor(self.dispatcher, session)
client_req.start()
responder = TcpClientResponder(session)
responder.start()
def plog(msg, error_log):
log = []
for s in error_log:
log.append(' Line %s: Column %s: %s' % (s.line, s.column, s.message))
log = '\n'+'\n'.join(log)
#print_log('Schemas validity ERROR:', str(schema.error_log))
print_log(msg, log)
def update(self):
response = self.request_dispatcher.pop_response()
#print "pop response", response
internal_id = response.get('id')
method = response.get('method')
params = response.get('params')
# A notification
if internal_id is None: # and method is not None and params is not None:
found = self.notification(method, params, response)
if not found and method == 'blockchain.address.subscribe':
request = {
'id':
None,
'method':
method.replace('.subscribe', '.unsubscribe'),
'params':
[self.shared.config.get('server', 'password')] + params,
}
self.request_dispatcher.push_request(None, request)
# A response
elif internal_id is not None:
self.send_response(internal_id, response)
else:
print_log("no method", response)
def evaluate(val_loaders, fake_loader, net, criterion, args, log,
num_mc_samples, num_mc_samples2):
freeze(net)
if args.gpu == 0:
print("-----------------deterministic-----------------")
deter_rets = ens_validate(val_loaders, net, criterion, args, log, 1)
unfreeze(net)
if args.gpu == 0:
print("-----------------ensemble {} times-----------------".format(
num_mc_samples2))
rets = ens_validate(val_loaders, net, criterion, args, log,
num_mc_samples2)
ens_attack(val_loaders, net, criterion, args, log, num_mc_samples,
min(num_mc_samples, 8))
if args.gpu == 0:
for k in val_loaders:
print_log(
'{} vs. adversarial: AP {}'.format(
k[0], plot_mi(args.save_path, 'adv_' + k[0], k[0])), log)
ens_validate(fake_loader,
net,
criterion,
args,
log,
num_mc_samples,
suffix='fake')
if args.gpu == 0:
for k in val_loaders:
print_log(
'{} vs. DeepFake: AP {}'.format(
k[0], plot_mi(args.save_path, 'fake', k[0])), log)
def get_bookid(filename, fb2):
global _connect
# search bookid in fb2
if options.search_id and fb2 is not None:
find = xpath('/m:FictionBook/m:description/m:custom-info')
bookid = None
for e in find(fb2):
bid = e.get('librusec-book-id')
if bid is not None:
try:
bookid = int(bid)
except:
pass
else:
return bookid
# search bookid by filename
try:
bookid = int(filename)
except ValueError:
curs = _connect.cursor()
curs.execute("SELECT BookId FROM libbookold WHERE FileName = ?",
(filename, ))
res = curs.fetchone()
if res is None:
print_log('ERROR: file not found in db:', filename, level=3)
return None
return res[0]
return bookid
def refine_MALA(logp_net, g, x_g, sgld_lr_z):
# latent space sgld
eps_sgld = torch.randn_like(x_g)
z_sgld = torch.randn((eps_sgld.size(0), args.noise_dim)).to(eps_sgld.device)
vs = (z_sgld.requires_grad_(), eps_sgld.requires_grad_())
steps = [vs]
accepts = []
gfn = lambda z, e: g.g(z) + g.logsigma.exp() * e
efn = lambda z, e: logp_net(gfn(z, e)).squeeze()
with torch.no_grad():
x_init = gfn(z_sgld, eps_sgld)
# plot(("{}/{:0%d}_init.png" % niters_digs).format(gen_sgld_dir, itr),
# x_init.view(x_g.size(0), *args.data_size))
for k in range(args.n_sample_steps):
vs, a = utils.hmc.MALA(vs, efn, sgld_lr_z)
steps.append(vs)
accepts.append(a.item())
print('...', k)
ar = np.mean(accepts)
utils.print_log("latent eps accept rate: {}".format(ar), args)
sgld_lr_z = sgld_lr_z + args.mcmc_lr * (ar - .57) * sgld_lr_z
z_sgld, eps_sgld = steps[-1]
with torch.no_grad():
x_ref = gfn(z_sgld, eps_sgld)
# plot(("{}/{:0%d}_ref.png" % niters_digs).format(gen_sgld_dir, itr),
# x_sgld.view(x_g.size(0), *args.data_size))
return x_init, x_ref, sgld_lr_z
def convert(source_file, config: Config):
if not check_file(source_file):
return
temp_dir_name = str(uuid.uuid1())
temp_dir = os.path.join(config.tmp_dir, temp_dir_name)
if os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
os.makedirs(temp_dir)
print_log("转换[%s]到epub" % source_file)
epub_file = convert_kf8_to_epub(source_file, temp_dir, config)
is_azw3 = str(source_file).lower().endswith(".azw3")
file_source_suffix = ".azw3" if is_azw3 else source_file[source_file.rfind("."):]
if epub_file:
file_copy(epub_file, source_file.replace(file_source_suffix, ".epub"))
mobi_file = None
if epub_file and is_azw3:
print_log("转换epub到mobi [%s]" % epub_file)
mobi_file = convert_epub_to_mobi(epub_file, config)
if mobi_file:
file_copy(mobi_file, source_file.replace(file_source_suffix, ".mobi"))
def eval_tgt(encoder, classifier, data_loader):
"""Evaluation for target encoder by source classifier on target dataset."""
# set eval state for Dropout and BN layers
encoder.eval()
classifier.eval()
# init loss and accuracy
loss = 0
acc = 0
# set loss function
criterion = nn.CrossEntropyLoss()
# evaluate network
for (images, labels) in data_loader:
images = make_variable(images, volatile=True)
labels = make_variable(labels).squeeze_()
preds = classifier(encoder(images))
loss += criterion(preds, labels).item()
pred_cls = preds.data.max(1)[1]
acc += pred_cls.eq(labels.data).cpu().sum()
loss /= len(data_loader)
acc_per = float(acc) / len(data_loader.dataset)
print_log("Avg Loss = {}, Avg Accuracy = {:2%}".format(loss, acc_per))
def calculate_rel_blast_matrix(self):
file_name = '{}-{}-{}_scores.npy'.format(self.org1.org_id,
self.org2.org_id, 'rel_blast')
if utils.file_exists(file_name, path_name=cs.NP_PATH):
message = 'using saved relative blast from {}'.format(file_name)
utils.print_log(message)
self.blast_sim_n_rel = utils.load_np(self.np_file)
self.blast_sim = utils.load_np(self.raw_np_file)
# self.blast_sim = interface.blast_xml_to_matrix(self)
else:
# blast similarity measure
blast_sim = interface.blast_xml_to_matrix(self)
self.blast_sim = blast_sim
blast_1 = interface.self_blast_xml_to_vec(self.org1)
blast_1[blast_1 == 0] = 1
blast_1 = np.array([blast_1]).repeat(self.org2.node_count,
axis=0).T
blast_2 = interface.self_blast_xml_to_vec(self.org2)
blast_2[blast_2 == 0] = 1
blast_2 = np.array([blast_2]).repeat(self.org1.node_count, axis=0)
blast_sim = blast_sim.reshape(self.org1.node_count,
self.org2.node_count)
blast_sim = blast_sim / np.power((blast_1 * blast_2), 0.5)
blast_sim = blast_sim.reshape(self.dim_sim)
# normalize blast matrix
self.blast_sim_n_rel = utils.normalize(blast_sim)
np_file = utils.join_path(cs.NP_PATH, file_name)
utils.write_np(self.blast_sim_n_rel, np_file)
def do_dispatch(self, session, request):
""" dispatch request to the relevant processor """
method = request['method']
params = request.get('params', [])
suffix = method.split('.')[-1]
if session is not None:
if suffix == 'subscribe':
session.subscribe_to_service(method, params)
prefix = request['method'].split('.')[0]
try:
p = self.processors[prefix]
except:
print_log("error: no processor for", prefix)
return
p.add_request(session, request)
if method in ['server.version']:
session.version = params[0]
try:
session.protocol_version = float(params[1])
except:
pass
def handle_command(self, raw_command, session):
try:
command = json.loads(raw_command)
except:
session.send_response({"error": "bad JSON"})
return True
try:
# Try to load vital fields, and return an error if
# unsuccessful.
message_id = command['id']
method = command['method']
except:
# Return an error JSON in response.
session.send_response({
"error": "syntax error",
"request": raw_command
})
else:
#print_log("new request", command)
my_method = command['method']
if my_method == 'blockchain.transaction.broadcast':
print_log("new request", command, "-----", session,
" Address:--->", session.address)
self.dispatcher.push_request(session, command)
def format_com(com, currency, exist_markets):
market_id = None
must_keys = ('market', 'pair', 'price', 'volume')
for k in must_keys:
if k not in com.keys():
return None
# get symbol / anchor
(symbol, anchor) = com['pair'].split('/')
if not symbol or not anchor:
return None
# get market_id
market_id = exist_markets.get(com['market'], None)
if not market_id:
print_log('passed {}: not exist'.format(com['market']))
return None
volume = float(com['volume'])
return {
'name': currency,
'market_id': market_id,
'symbol': symbol,
'anchor': anchor,
'volume_24h_usd': volume,
}
def get(self, platform):
log_path = self.config['log']['callback_log']
utils.print_log('callback', log_path, self.request.uri)
params = {}
if platform == 'youmiios':
self.platform = 1
params['sign'] = self.get_argument('sign','')
sign = self.check_sign_ios()
elif platform == 'youmiaos':
self.platform = 2
params['sign'] = self.get_argument('sig','')
sign = self.check_sign_aos()
else:
self.write('what the f**k?')
keys = ['order', 'ad', 'adid', 'user', 'points', \
'price', 'time', 'device']
for key in keys:
params[key] = self.get_argument(key, '')
if not params:
self.write('arguments is required')
return
if sign != params['sign']:
self.write('invalid sign')
return
self.save_order(**params)
self.write('ok')
def analysis_currency(currency_data):
rd = []
for cd in currency_data:
jd = json.loads(cd[0].decode("utf-8"))
if 'name' in jd.keys() and 'symbol' in jd.keys()\
and jd['name'] and jd['symbol']:
# deal supplies
for s in ('total_supply', 'max_supply', 'circulating_supply'):
if s in jd.keys() and jd[s] != '-':
jd[s] = translate_sp_str_to_number(jd[s])
else:
jd[s] = 0
# deal media
media_types = ('Website', 'Explorer',
'Announcement', 'Message Board')
for m in jd['media']:
if m[1] in media_types:
key = str(m[1]).lower().replace(' ', '_')
jd[key] = m[0]
rd.append(jd)
else:
print_log('error found, no name or symbol:'
' name is {}, symbol is {}'.format(
jd.get('name', None), jd.get('symbol', None)))
print_log(jd['url'])
return rd
def test(model, criterion, test_loader, epoch, is_main):
model.eval()
test_loss = 0
correct = 0
total = 0
for batch_idx, (data, target) in enumerate(test_loader):
if torch.cuda.is_available():
data, target = Variable(data.cuda()), Variable(target.cuda())
else:
data, target = Variable(data), Variable(target)
outputs = model(data)
loss = criterion(outputs, target)
test_loss += loss.data[0]
_, predicted = torch.max(outputs.data, 1)
total += target.size(0)
correct += predicted.eq(target.data).cpu().sum()
max_result.append(correct)
if is_main is True:
utils.print_log('# TEST : Epoch : {} | Loss: ({:.4f}) | Acc: ({:.2f}%) ({}/{}) | Err: ({:.2f}%) | Max: ({})'
.format(epoch, test_loss/(batch_idx+1), 100.*correct/total, correct, total, 100-100.*correct/total, max(max_result)))
print('# TEST : Epoch : {} | Loss: ({:.4f}) | Acc: ({:.2f}%) ({}/{}) | Err: ({:.2f}% | Max: ({}))'
.format(epoch, test_loss/(batch_idx+1), 100.*correct/total, correct, total, 100-100.*correct/total, max(max_result)))
elif is_main is False:
utils.print_log('$ TEST_S : Epoch : {} | Loss: ({:.4f}) | Acc: ({:.2f}%) ({}/{}) | Err: ({:.2f}%) | Max: ({})'
.format(epoch, test_loss/(batch_idx+1), 100.*correct/total, correct, total, 100-100.*correct/total, max(max_result)))
print('$ TEST_S : Epoch : {} | Loss: ({:.4f}) | Acc: ({:.2f}%) ({}/{}) | Err: ({:.2f}% | Max: ({}))'
.format(epoch, test_loss/(batch_idx+1), 100.*correct/total, correct, total, 100-100.*correct/total, max(max_result)))
def two_step(driver: WebDriver) -> None:
"""
Performs the two-step verification, if the website asks for it. If no two step prompt is detected, then
the script will go on to the next login step.
:param driver: The WebDriver.
:return: None; modifies the website given by the WebDriver.
"""
try:
text_button = driver.find_element_by_xpath(
"//input[@type='radio' and @name='option' and @value='sms']")
text_button.click()
driver.find_element_by_id("continue").click()
verification = WebDriverWait(driver, 10).until(
expected_conditions.presence_of_element_located(
(By.XPATH, "//input[@type='text' and @name='code']")))
# need user input
code = input("Enter your 6 digit 2 Step Verification Code: ")
verification.send_keys(code)
submit = driver.find_element_by_xpath("//input[@type='submit']")
submit.click()
except NoSuchElementException:
print_log("No 2 Step Verification Required!")
def extract_uid_from_recordings(driver: WebDriver,
indices_to_download: List[int],
metadata: List[Dict[str, Any]]) -> None:
"""
Adds a new "audio_id" key to the metadata for each recording, which represents the audio_id of each recording.
This will be used to download the .wav files for each recording later on.
:param driver: The WebDriver.
:param indices_to_download: A list of which recordings to download (determined by their index in a list of
recordings where the most recent recording has index 0).
:param metadata: The metadata information for all of the recordings.
:return: None; modifies the metadata dictionaries.
"""
print_log("Extracting Audio ID.")
events = [
json.loads(entry["message"])["message"]
for entry in driver.get_log("performance")
]
response_events = list(filter(check_for_uid, events))
if len(response_events) != len(indices_to_download):
print_log(
f"WARNING: The number of network events to find the uid ({len(response_events)}) "
f"do not equal the number of recordings there are on the alexa website ({len(indices_to_download)}). "
"This could be because the site format has changed. The audio files will not be "
"downloaded in this run.")
else:
for idx, event in enumerate(response_events):
idx_to_update = indices_to_download[idx]
metadata[idx_to_update].update(
{"audio_id": get_uid_from_event(event)})
def get_article(url, subject, tears=15, verbose=False):
# Get a html
resp = urlopen(url)
soup = BeautifulSoup(resp, 'lxml')
articles = soup.find_all('tr')[1:]
# 게시글만 추출
articles = [a for a in articles if a.find('td', {'class': 't_left'}) is not None]
utils.print_log(verbose, "articles cnt", len(articles))
# 게시글이 없으면 리턴
if len(articles) == 0:
del resp
return pd.DataFrame()
a_list = []
for a in articles:
l = []
title = a.find('a')['alt']
user_id = a.find('span', {'class': 'nick'}).text
article_link = a.find('a')['href']
article_link = article_link.replace(re.search(r'&query=.+&', article_link).group(), '&')
article_id = re.search(r'(\d{18})', article_link).group()
reply_num = mod_reply(a)
view_num = a.find('span', {'class': 'viewV'}).text.replace(',', '')
if view_num == '':
continue
# Gathering the cotent of each article
con = BeautifulSoup(urlopen(article_link), 'html.parser')
# 감동 주제일 경우 Y값을 판단해서 Y가 아니면 next loop
if subject == 'touching':
yn = touch_article(con, tears)
if not yn:
continue
date = con.find_all('span', {'class': 'val'})[3].text
# date = mod_user_id(date)
content = ''
# Making the list
l.append(title)
l.append(date)
l.append(article_id)
l.append(user_id)
l.append(article_link)
l.append(content)
l.append(reply_num)
l.append(view_num)
a_list.append(l)
utils.print_log(verbose, "article line 1", l)
time.sleep(random.randint(2, 7) / 3)
if len(a_list) == 0: # 감동 주제일 경우 적합 게시물이 없을 경우 빈 DF 반환
del resp
return pd.DataFrame()
del resp
result = pd.DataFrame(a_list)
# munging of the dataframe
result.columns = ['title', 'date_time', 'article_id', 'member_id', 'article_link', 'content', 'reply_num', 'view_num']
result.loc[:, 'date_time'] = pd.to_datetime(result['date_time'])
result.set_index('article_id')
return result
def train(args, state_info, Train_loader, Test_loader, epoch):
cuda = True if torch.cuda.is_available() else False
device = 'cuda' if cuda else 'cpu'
FloatTensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor
LongTensor = torch.cuda.LongTensor if cuda else torch.LongTensor
criterion = torch.nn.CrossEntropyLoss().to(device)
train_Size = torch.tensor(0, dtype=torch.float32)
correct_Real = torch.tensor(0, dtype=torch.float32)
correct_Real2 = torch.tensor(0, dtype=torch.float32)
# train
state_info.model.train()
utils.print_log('Type, Epoch, Batch, loss')
tot_loss = 0
for it, (x, y) in enumerate(Train_loader):
x, y = to_var(x, FloatTensor), to_var(y, LongTensor)
state_info.optim_model.zero_grad()
rot_cls, logits = state_info.forward(x)
loss = criterion(logits, y)
loss.backward()
state_info.optim_model.step()
tot_loss += loss.item()
utils.print_log('Train, {}, {}, {:.6f}'.format(epoch, it, tot_loss))
print('Train, {}, {}, {:.6f}'.format(epoch, it, tot_loss))
epoch_result = test(args, state_info, Test_loader, epoch)
return epoch_result
def test(args, state_info, Test_loader, epoch):
cuda = True if torch.cuda.is_available() else False
FloatTensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor
LongTensor = torch.cuda.LongTensor if cuda else torch.LongTensor
testSize = torch.tensor(0, dtype=torch.float32)
correct_Test = torch.tensor(0, dtype=torch.float32)
# test
state_info.model.eval()
for it, (x, y) in enumerate(Test_loader):
x, y = to_var(x, FloatTensor), to_var(y, LongTensor)
rot_cls, logits = state_info.forward(x)
_, pred = torch.max(logits.softmax(dim=1), 1)
correct_Test += float(pred.eq(y.data).cpu().sum())
testSize += float(x.size(0))
utils.print_log('Type, Epoch, Batch, Percentage')
utils.print_log('Test, {}, {}, {:.3f}'.format(
epoch, it, 100. * correct_Test / testSize))
print('Test, {}, {}, {:.3f}'.format(epoch, it,
100. * correct_Test / testSize))
return 100. * correct_Test / testSize
def get(self):
sharepoint = self.db.get("SELECT * FROM `options` WHERE `key` = 'share'")
type = self.arguments.get('type')
print type
share = self.db.get(
"SELECT `share_fb`,`share_tw`,`share_gg`,`like` FROM `user` WHERE `uid`=%s", self.current_user['uid'])
if (share['share_fb']==0 and type=="1") or (share['share_tw']==0 and type=="2") or (share['share_gg']==0 and type=="3"):
prize = int(sharepoint['values'])
self.db.execute(
"UPDATE `user` SET `points` = `points` + %s, \
`total_points` = `total_points` + %s , \
`share_points` = `share_points`+ %s "
"WHERE `scopeid` = %s" %(prize, prize, prize, self.arguments.get('scopeid')))
if type=="1":
self.db.execute(
"UPDATE `user` SET `share_fb` = 1 WHERE `uid`=%s", self.current_user['uid'])
elif type=="2":
self.db.execute(
"UPDATE `user` SET `share_tw` = 1 WHERE `uid`=%s", self.current_user['uid'])
elif type=="3":
self.db.execute(
"UPDATE `user` SET `share_gg` = 1 WHERE `uid`=%s", self.current_user['uid'])
order="分享紅利"
ad="分享紅利"
self.db.execute(
"INSERT INTO `callback_order`(`order`,`ad`,`user`,`platform`,`ad_source`,`points`,`appType`)"
"VALUES(%s,%s,%s,%s,%s,%s,1)", order,ad,self.current_user['uid'],3,2,prize)
self.db.execute(
"INSERT INTO `global_orders` (uid, points, last, type, note,appType)"
"VALUES (%s, %s, %s, %s, %s, 1)",
self.current_user['uid'], prize, self.current_user['points'], 2, ad)
elif (share['like']==0 and type=="4"):
likepoint = self.db.get("SELECT * FROM `options` WHERE `key` ='like'")
prize = int(likepoint['values'])
self.db.execute(
"UPDATE `user` SET `points` = `points` + %s, \
`total_points` = `total_points` + %s , \
`share_points` = `share_points`+ %s , \
`like` = '1'"
"WHERE `scopeid` = %s" %(prize, prize, prize, self.arguments.get('scopeid')))
order="fb點贊紅利"
ad="fb點贊紅利"
self.db.execute(
"INSERT INTO `callback_order`(`order`,`ad`,`user`,`platform`,`ad_source`,`points`,`appType`)"
"VALUES(%s,%s,%s,%s,%s,%s,1)", order,ad,self.current_user['uid'],3,11,prize)
self.db.execute(
"INSERT INTO `global_orders` (uid, points, last, type, note,appType)"
"VALUES (%s, %s, %s, %s, %s, 1)",
self.current_user['uid'], prize, self.current_user['points'], 11, ad)
else:
prize = 0
user_info=self.db.get(
"SELECT `points` from `user` WHERE `uid` = %s", self.current_user['uid'])
log_path = self.config['log']['fblike_log']
utils.print_log('fblike', log_path, self.request.uri+"uid="+str(self.current_user['uid'])+"points="+str(prize))
self.return_result({
"points":user_info['points'],
"bonus":prize,
})
def val(args, model, teacher, epoch, val_loader, log):
model.eval()
teacher.eval()
losses = AverageMeter()
MSE_loss = nn.MSELoss(reduction='sum')
for (data, _, _) in tqdm(val_loader):
data = data.to(device)
with torch.no_grad():
z, output, mu, log_var = model(data)
# get model's intermediate outputs
s_activations, _ = feature_extractor(
z, model.decode, target_layers=['12', '18', '24'])
t_activations, _ = feature_extractor(
data, teacher.features, target_layers=['8', '15', '22'])
mse_loss = MSE_loss(output, data)
kld_loss = 0.5 * torch.sum(-1 - log_var + torch.exp(log_var) +
mu**2)
for i in range(len(s_activations)):
s_act = model.adapter[i](s_activations[-(i + 1)])
mse_loss += MSE_loss(s_act, t_activations[i])
loss = mse_loss + args.kld_weight * kld_loss
losses.update(loss.item(), data.size(0))
print_log(('Val Epoch: {} loss: {:.6f}'.format(epoch, losses.avg)), log)
return losses.avg
def train(args, model, teacher, epoch, train_loader, optimizer, log):
model.train()
teacher.eval()
losses = AverageMeter()
MSE_loss = nn.MSELoss(reduction='sum')
for (data, _, _) in tqdm(train_loader):
data = data.to(device)
z, output, mu, log_var = model(data)
# get model's intermediate outputs
s_activations, _ = feature_extractor(z,
model.decode,
target_layers=['10', '16', '22'])
t_activations, _ = feature_extractor(data,
teacher.features,
target_layers=['7', '14', '21'])
optimizer.zero_grad()
mse_loss = MSE_loss(output, data)
kld_loss = 0.5 * torch.sum(-1 - log_var + torch.exp(log_var) + mu**2)
for i in range(len(s_activations)):
s_act = model.adapter[i](s_activations[-(i + 1)])
mse_loss += MSE_loss(s_act, t_activations[i])
loss = mse_loss + args.kld_weight * kld_loss
losses.update(loss.sum().item(), data.size(0))
loss.backward()
optimizer.step()
print_log(('Train Epoch: {} Loss: {:.6f}'.format(epoch, losses.avg)), log)
def get(self):
signInfo = self.db.get(
"SELECT * FROM `user_sign` WHERE `uid` = %s",
self.current_user['uid'])
log_path = self.config['log']['signinfo_log']
day1 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize1'")
day2 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize2'")
day3 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize3'")
day4 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize4'")
day5 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize5'")
day6 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize6'")
sign_prize = 5
if signInfo:
times = signInfo['time']
#print (datetime.datetime.now().date()-(times+datetime.timedelta(hours=8)).date()).days
#print (datetime.datetime.now().date()-times.date()).days
if 0<=(datetime.datetime.now().date()-times.date()).days<=1:
#print times
#print "days"
int = (signInfo['day']+1)%6
#print str(int)+"int"
if int==2:
sign_prize=day2['values']
if int==3:
sign_prize=day3['values']
if int==4:
sign_prize=day4['values']
if int==5:
sign_prize=day5['values']
if int==6:
sign_prize=day6['values']
if int==0:
sign_prize=day6['values']
times = signInfo['time']-datetime.timedelta(hours=8)
else:
sign_prize=day1['values']
times =0
utils.print_log('signinfo', log_path, self.request.uri+"&lastSignTime:"+str(times)+"&user:"******"&NextSignPoint:"+str(sign_prize))
#print str(times)+"times"
#timeArray = time.localtime(time.time())
#otherStyleTime = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
#print time.time()
#print otherStyleTime
taskInfo = self.db.get("SELECT * FROM `global_orders` WHERE `type`=6 AND `uid`=%s \
AND DATE_SUB(CURDATE(), INTERVAL 5 DAY) <= date(record_time) LIMIT 1", self.current_user['uid'])
if taskInfo :
showInterstitial = 0
signable = True
else:
showInterstitial = 1
signable = False
self.return_result({
"sign_time":times,
"current_time":long(time.time()),
"reward_points":sign_prize,
"showInterstitial":showInterstitial,
"signable":signable,
#"user":self.current_user['uid'],
})
def process(self, session, request):
method = request['method']
params = request['params']
result = None
if method in ['server.stop', 'server.info', 'server.debug']:
try:
password = request['params'][0]
except:
password = None
if password != self.password:
self.push_response(session,
{'id': request['id'],
'result': None,
'error': 'incorrect password'})
return
if method == 'server.banner':
result = self.banner.replace('\\n', '\n')
elif method == 'server.peers.subscribe':
result = self.get_peers()
elif method == 'server.version':
result = VERSION
elif method == 'server.stop':
self.shared.stop()
result = 'stopping, please wait until all threads terminate.'
elif method == 'server.info':
result = map(lambda s: {"time": s.time,
"name": s.name,
"address": s.address,
"version": s.version,
"subscriptions": len(s.subscriptions)},
self.dispatcher.request_dispatcher.get_sessions())
elif method == 'server.debug':
try:
s = request['params'][1]
except:
s = None
if s:
from guppy import hpy
h = hpy()
bp = self.dispatcher.request_dispatcher.processors['blockchain']
try:
result = str(eval(s))
except:
result = "error"
else:
print_log("unknown method", method)
if result != '':
self.push_response(session, {'id': request['id'], 'result': result})
def run(self):
while not self.shared.stopped():
request, session = self.queue.get(10000000000)
try:
self.process(request, session)
except:
traceback.print_exc(file=sys.stdout)
print_log("processor terminating")
def unknown_endtag(self, name):
if name == 'fictionbook':
# sgmllib workaround
name = 'FictionBook'
if name not in self.NESTABLE_TAGS:
# unknown tag
print_log('unknown end tag:', name, level=2)
self.handle_data('</%s>' % name)
return
BeautifulSoup.BeautifulStoneSoup.unknown_endtag(self, name)
def run(self):
ircname = self.getname()
while not self.processor.shared.stopped():
try:
s = socket.socket()
s.connect(("irc.freenode.net", 6667))
s.settimeout(300)
except:
s.close()
time.sleep(10)
continue
try:
s.send("USER electrum 0 * :" + self.host + " " + ircname + "\n")
s.send("NICK " + self.nick + "\n")
s.send("JOIN #electrum\n")
sf = s.makefile("r", 0)
t = 0
while not self.processor.shared.stopped():
line = sf.readline().rstrip("\r\n").split()
if not line:
continue
if line[0] == "PING":
s.send("PONG " + line[1] + "\n")
elif "353" in line: # answer to /names
k = line.index("353")
for item in line[k + 1 :]:
if item.startswith(self.prepend):
s.send("WHO %s\n" % item)
elif "352" in line: # answer to /who
# warning: this is a horrible hack which apparently works
k = line.index("352")
try:
ip = socket.gethostbyname(line[k + 4])
except:
print_log("gethostbyname error", line[k + 4])
continue
name = line[k + 6]
host = line[k + 9]
ports = line[k + 10 :]
self.peers[name] = (ip, host, ports)
if time.time() - t > 5 * 60:
self.processor.push_response({"method": "server.peers", "params": [self.get_peers()]})
s.send("NAMES #electrum\n")
t = time.time()
self.peers = {}
except:
traceback.print_exc(file=sys.stdout)
finally:
sf.close()
s.close()
print_log("quitting IRC")
def stop_session(fd):
try:
# unregister before we close s
poller.unregister(fd)
except:
print_log("unregister error", fd)
traceback.print_exc(file=sys.stdout)
session = self.fd_to_session.pop(fd)
# this will close the socket
session.stop()
def copy_fb2(filename, data, to_dir=None, msg='save bad fb2 file:'):
if to_dir is None:
if not options.save_bad:
return
to_dir = options.save_bad
filename = str(filename)+'.fb2'
fn = os.path.join(to_dir, filename)
print_log(msg, fn)
if options.nozip:
open(fn).write(data)
else:
save_zip(fn, filename, data)
def check_tags(soup):
errors = 0
for t in soup.findAll(True):
if t.name not in FB2Parser.NESTABLE_TAGS:
print_log('FATAL: unknown tag:', t.name, level=3)
return -1
#print t.name, t.parent.name
if t.parent.name not in FB2Parser.NESTABLE_TAGS[t.name]:
print_log('WARNING: nestable tag: <%s> in <%s>'
% (t.name, t.parent.name), level=1)
errors += 1
return errors
def nova_aggregate_add_host(self, aggregate, host):
result = False
try:
add_result = self.nova.aggregates.add_host(aggregate, host)
print_log('Add host<%s> to aggregate<%s>, result : %s ' % (host, aggregate, add_result), logging.INFO)
result = True
except:
print_log('Exception when add host<%s> to aggregate<%s>, Exception : %s ' %
(host, aggregate, traceback.format_exc()), logging.ERROR)
return result
def process(self, request):
method = request["method"]
params = request["params"]
result = None
if method in ["server.stop", "server.info"]:
try:
password = request["params"][0]
except:
password = None
if password != self.password:
self.push_response({"id": request["id"], "result": None, "error": "incorrect password"})
return
if method == "server.banner":
result = self.banner.replace("\\n", "\n")
elif method == "server.peers.subscribe":
result = self.get_peers()
elif method == "server.version":
result = VERSION
elif method == "server.stop":
self.shared.stop()
result = "stopping, please wait until all threads terminate."
elif method == "server.info":
result = map(
lambda s: {
"time": s.time,
"name": s.name,
"address": s.address,
"version": s.version,
"subscriptions": len(s.subscriptions),
},
self.dispatcher.request_dispatcher.get_sessions(),
)
elif method == "server.cache":
p = self.dispatcher.request_dispatcher.processors["blockchain"]
result = len(repr(p.store.tx_cache))
elif method == "server.load":
p = self.dispatcher.request_dispatcher.processors["blockchain"]
result = p.queue.qsize()
else:
print_log("unknown method", request)
if result != "":
self.push_response({"id": request["id"], "result": result})
def check_empty_tags(xml):
errs = 0
find = etree.XPath('//*')
for e in find(xml):
if (not e.getchildren() and
not e.tag.endswith('empty-line') and
not e.tag.endswith('image') and
not e.tag.endswith('sequence')):
if e.text is None or not e.text.strip():
print_log('WARNING: Line %s: empty tag: %s' %
(e.sourceline, e.tag), level=1)
errs += 1
return errs
def process(self, request):
method = request['method']
params = request['params']
result = None
if method in ['server.stop', 'server.info']:
try:
password = request['params'][0]
except:
password = None
if password != self.password:
self.push_response({'id': request['id'],
'result': None,
'error': 'incorrect password'})
return
if method == 'server.banner':
result = self.banner.replace('\\n', '\n')
elif method == 'server.peers.subscribe':
result = self.get_peers()
elif method == 'server.version':
result = VERSION
elif method == 'server.stop':
self.shared.stop()
result = 'stopping, please wait until all threads terminate.'
elif method == 'server.info':
result = map(lambda s: {"time": s.time,
"name": s.name,
"address": s.address,
"version": s.version,
"subscriptions": len(s.subscriptions)},
self.dispatcher.request_dispatcher.get_sessions())
elif method == 'server.cache':
p = self.dispatcher.request_dispatcher.processors['blockchain']
result = len(repr(p.history_cache))
elif method == 'server.load':
p = self.dispatcher.request_dispatcher.processors['blockchain']
result = p.queue.qsize()
else:
print_log("unknown method", request)
if result != '':
self.push_response({'id': request['id'], 'result': result})
def process_file(filename):
# process one file
LogOptions.filename = os.path.abspath(filename)
for file_format, z_filename, data in read_file(filename):
LogOptions.z_filename = z_filename
Stat.total += 1
if file_format == 'error':
Stat.not_xml += 1
print_log('FATAL: read file error', level=3)
continue
if not check_xml(data):
Stat.not_xml += 1
continue
calc_statistics(data)
def validate(data, type='xml', log_msg='', quiet=False):
if log_msg:
log_msg = log_msg+':'
# sax validation
if type == 'sax':
try:
sax_lint(data)
except Exception, err:
#traceback.print_exc()
print_log(log_msg, 'sax validity check failed')
return None
if not quiet:
print_log(log_msg, 'sax validity check passed')
return True
def run(self):
ircname = self.getname()
while not self.processor.shared.stopped():
try:
s = socket.socket()
s.connect(('irc.freenode.net', 6667))
s.settimeout(300)
except:
s.close()
time.sleep(10)
continue
try:
s.send('USER electrum 0 * :' + self.host + ' ' + ircname + '\n')
s.send('NICK ' + self.nick + '\n')
s.send('JOIN #electrum\n')
sf = s.makefile('r', 0)
t = 0
while not self.processor.shared.stopped():
line = sf.readline().rstrip('\r\n').split()
if not line:
continue
if line[0] == 'PING':
s.send('PONG ' + line[1] + '\n')
elif '353' in line: # answer to /names
k = line.index('353')
for item in line[k+1:]:
if item.startswith(self.prepend):
s.send('WHO %s\n' % item)
elif '352' in line: # answer to /who
# warning: this is a horrible hack which apparently works
k = line.index('352')
ip = socket.gethostbyname(line[k+4])
name = line[k+6]
host = line[k+9]
ports = line[k+10:]
self.peers[name] = (ip, host, ports)
if time.time() - t > 5*60:
self.processor.push_response({'method': 'server.peers', 'params': [self.get_peers()]})
s.send('NAMES #electrum\n')
t = time.time()
self.peers = {}
except:
traceback.print_exc(file=sys.stdout)
finally:
sf.close()
s.close()
print_log("quitting IRC")
def subscribe_to_service(self, method, params):
if self.stopped():
return False
if len(self.subscriptions) > self.max_subscriptions:
print_log("max subscriptions reached", self.address)
return False
# append to self.subscriptions only if this does not raise
self.bp.do_subscribe(method, params, self)
with self.lock:
if (method, params) not in self.subscriptions:
self.subscriptions.append((method,params))
return True
def info(self):
for sub in self.subscriptions:
# print sub
method = sub[0]
if method == "blockchain.address.subscribe":
addr = sub[1]
break
else:
addr = None
if self.subscriptions:
print_log(
"%4s" % self.name, "%15s" % self.address, "%35s" % addr, "%3d" % len(self.subscriptions), self.version
)
def nova_aggregate_create(self, name, availability_zone):
result = None
try:
aggregate_result = self.nova.aggregates.create(name, availability_zone)
print_log('created Aggregate result is : %s ' % aggregate_result, logging.INFO)
if aggregate_result.name == name:
result = aggregate_result
except:
print_log('Exception when create AG for %s, Exception: %s' % (name, traceback.format_exc()), logging.ERROR)
return result
def unknown_starttag(self, name, attrs, selfClosing=0):
#print 'unknown_starttag:', repr(name)
if name == 'fictionbook':
# sgmllib workaround
name = 'FictionBook'
if name not in self.NESTABLE_TAGS:
# unknown tag
print_log('unknown start tag:', name, level=2)
#attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs))
attrs = ' '.join(y for x, y in attrs)
self.handle_data('<%s %s>' % (name, attrs))
return
BeautifulSoup.BeautifulStoneSoup.unknown_starttag(self, name, attrs,
selfClosing)
def get(self):
signInfo = self.db.get(
"SELECT * FROM `user_sign` WHERE `uid` = %s and `appType`=1 LIMIT 1",
self.current_user['uid'])
log_path = self.config['log']['signinfo_log']
day1 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize1'")
day2 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize2'")
day3 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize3'")
day4 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize4'")
day5 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize5'")
day6 = self.db.get("SELECT * FROM `options` WHERE `key` = 'sign_prize6'")
sign_prize = 5
if signInfo:
times = signInfo['time']
#print (datetime.datetime.now().date()-(times+datetime.timedelta(hours=8)).date()).days
#print (datetime.datetime.now().date()-times.date()).days
if 0<=(datetime.datetime.now().date()-times.date()).days<=1:
#print times
#print "days"
int = (signInfo['day']+1)%6
#print str(int)+"int"
if int==2:
sign_prize=day2['values']
if int==3:
sign_prize=day3['values']
if int==4:
sign_prize=day4['values']
if int==5:
sign_prize=day5['values']
if int==6:
sign_prize=day6['values']
if int==0:
sign_prize=day6['values']
times = signInfo['time']-datetime.timedelta(hours=8)
else:
sign_prize=day1['values']
times =0
utils.print_log('signinfo', log_path, self.request.uri+"kaka&lastSignTime:"+str(times)+"&user:"******"&NextSignPoint:"+str(sign_prize))
# print str(times)+"times"
# timeArray = time.localtime(time.time())
# otherStyleTime = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
#print time.time()
#print otherStyleTime
self.return_result({
"sign_time":times,
"current_time":long(time.time()),
"reward_points":sign_prize,
#"user":self.current_user['uid'],
})
def get(self, platform):
log_path = self.config['log']['adyoumi_log']
utils.print_log('adyoumi', log_path, self.request.uri)
if platform == 'adyoumi':
self.__save_adyoumi_order()
elif platform == 'iosadyoumi':
self.__save_iosadyoumi_order()
elif platform == 'tapjoy':
self.__save_tapjoy_order()
elif platform == 'adxmi':
self.__save_adxmi_order()
elif platform == 'fb':
self.__save_fb_order()
elif platform == 'fyber':
self.__save_fyber_order()
def main():
global db, config
try:
config = yaml.load(file(SETTINGS_FILE, 'r'), YamlLoader)
except yaml.YAMLError as e:
print "Error in configuration file: %s" % e
return
log_path = config['log']['autoban_log']
db = torndb.Connection(**config['mysql'])
while True:
params, succ, msg = duiba()
info = '%s %s' % (params['rejectOrderNums'], msg)
utils.print_log('autoban', log_path, info)
time.sleep(SLEEP_TIME)
