def send_print_job(redis_connection, filename):
client_logger = Logger.client_logger()
logger = client_logger.genLogger("print_client")
URL = json.loads(open("config/config.json", "r").read())["UPLOAD_URL"]
file_object = {"file": open(PRINT_PATH + filename, 'rb')}
req = requests.post(URL, files=file_object)
if req.status_code == 200:
if req.json()["Status"] == "Success":
#print "Send File %s for printing" % filename
logger.info("Send File %s for printing" % filename)
redis_connection.sadd("File_Queue", filename)
elif req.json()["Status"] == "Error":
#print "Upload Failed: %s" % req.json()["Error_Msg"]
logger.error("Upload Failed: %s" % req.json()["Error_Msg"])
else:
#print "Error from Webserver!"
#print req.text
logger.error("Error from Webserver!")
logger.error(req.text)
from gensim.models import Word2Vec
from sklearn import metrics
from print_log import Logger
from gensim.models import FastText
import sys
import itertools
from tqdm import tqdm
def mean(z): # used for FastText and Word2Vec
return sum(itertools.chain(z)) / len(z)
# save training log
logdir = 'log/'
sys.stdout = Logger(logdir + "thunderbird.log")
PAD = '<PAD>'
# sentence max length
pad_size = 80
tokenizer = lambda x: x.split(' ') # word-level
# loading FastText word embedding
fasttext_model = FastText.load('archi_fasttext.model')
# dataset file path
file_path = 'data/thunderbird_report.txt'
all_data = []
data_label = []
if args.embedding == 'random':
embedding = 'random'
model_name = args.model # TextRNN_Att
if model_name == 'FastText':
from utils_fasttext import build_dataset, build_iterator, get_time_dif
embedding = 'random'
else:
from utils import build_dataset, build_iterator, get_time_dif
# import_module()
x = import_module('models.' + model_name)
config = x.Config(dataset, embedding)
# save train log
logdir = config.log_path + '/'
sys.stdout = Logger(logdir + "{}.log".format(args.model))
# set train random seed
np.random.seed(1)
torch.manual_seed(1)
torch.cuda.manual_seed_all(1)
torch.backends.cudnn.deterministic = True
start_time = time.time()
print("Loading data...")
vocab, train_data, dev_data, test_data = build_dataset(config, args.word)
train_iter = build_iterator(train_data, config)
dev_iter = build_iterator(dev_data, config)
test_iter = build_iterator(test_data, config)
if isinstance(output, list):
loss_test = LOSS(output[0][idx_out_test], labels[idx_test])
print(' | loss: {:.4f}'.format(loss_test.item()), end='')
acc_test, f1_test = evaluate(output[0][idx_out_test], labels[idx_test])
else:
loss_test = LOSS(output[idx_out_test], labels[idx_test])
print(' | loss: {:.4f}'.format(loss_test.item()), end='')
acc_test, f1_test = evaluate(output[idx_out_test], labels[idx_test])
print(' | time: {:.4f}s'.format(time.time() - t))
loss_list[epoch] += [loss_test.item()]
return float(acc_test.item()), float(f1_test.item())
if __name__ == '__main__':
sys.stdout = Logger("{}.log".format(dataset))
experimentType = "labeled"
supPara = "0.01"
write_embeddings = False
path = '../data/' + dataset + '/'
adj, features, labels, idx_train_ori, idx_val_ori, idx_test_ori, idx_map = load_data(
path=path, dataset=dataset)
# Transductive的数据集变换
if args.inductive:
print("Transfer to be inductive.")
# resample
# 之前的数据集划分： 训练集20 * class 验证集1000 其他的测试集
def parse_arguments():
parser = argparse.ArgumentParser(
description='Argument parser for Fake News Detection')
# Data Related
# data_root_path = '/home/{}/GCN/FND/data/fakeNews/'.format(USERNAME)
data_root_path = './data/fakeNews/'
parser.add_argument("--root", type=str, default=data_root_path)
parser.add_argument("--train",
type=str,
default=data_root_path + 'fulltrain.csv')
parser.add_argument("--dev",
type=str,
default=data_root_path + 'balancedtest.csv')
parser.add_argument("--test",
type=str,
default=data_root_path + 'test.xlsx',
help='Out of domain test set')
parser.add_argument("--pte",
type=str,
default='',
help='Pre-trained embeds')
parser.add_argument("--entity_desc",
type=str,
help='entity description path.',
default=data_root_path + 'entityDescCorpus.pkl')
parser.add_argument("--entity_tran",
type=str,
help='entity transE embedding path.',
default=data_root_path + 'entity_feature_transE.pkl')
parser.add_argument("--adjs", type=str, default=data_root_path + 'adjs/')
# Hyper-parameters
parser.add_argument("--emb_dim", type=int, default=100)
parser.add_argument("--hidden_dim", type=int, default=100)
parser.add_argument("--node_emb_dim", type=int, default=32)
parser.add_argument("--max_epochs", type=int, default=15)
parser.add_argument("--max_sent_len", type=int, default=50)
parser.add_argument("--max_sents_in_a_doc", type=int, default=10000)
parser.add_argument("--batch_size", type=int, default=32)
parser.add_argument("--lr", type=float, default=1e-3)
parser.add_argument("--dropout", type=float, default=0.5)
parser.add_argument("--ntags", type=int, default=4) # 4 or 2
parser.add_argument("--weight_decay", type=float, default=1e-6)
parser.add_argument("--pooling",
type=str,
default='max',
help='Pooling type: "max", "mean", "sum", "att". ')
# parser.add_argument("--config", type=str, default='config_default',
# help='Name for saving plots')
parser.add_argument("--model_file",
type=str,
default='model_default.t7',
help='For evaluating a saved model')
parser.add_argument("--plot", type=int, default=0, help='set to plot attn')
parser.add_argument("--mode",
type=int,
default=0,
help='0: train&test, 1:test')
# parser.add_argument("--cuda", type=bool, default=True, help='use gpu to speed up or not')
parser.add_argument("--cuda",
type=bool,
default=True,
help='use gpu to speed up or not')
parser.add_argument("--device", type=int, default=0, help='GPU ID. ')
parser.add_argument("--HALF",
type=bool,
default=True,
help='Use half tensor to save memory')
parser.add_argument("--DEBUG", action='store_true', default=False, help='')
parser.add_argument(
"--node_type",
type=int,
default=3,
help='3 represents three types: Document&Entity&Topic; \n'
'2 represents two types: Document&Entiy; \n'
'1 represents two types: Document&Topic; \n'
'0 represents only one type: Document. ')
parser.add_argument('-r', "--repeat", type=int, default=1, help='')
parser.add_argument('-s', "--seed", type=list, default=[5], help='')
for dir in ["models/", "ckpt/", "plots/", "result/", "log/"]:
if not os.path.exists(dir): os.makedirs(dir)
args = parser.parse_args()
TIMENOW = (datetime.datetime.utcnow() +
datetime.timedelta(hours=8)).strftime("%m%d_%H%M")
NODETYPE = {0: "D", 1: "DT", 2: "DE", 3: "DET"}[args.node_type]
if args.mode == 0:
MODELNAME = 'CompareNet_{}_{}_{}'.format(args.pooling.capitalize(),
NODETYPE, TIMENOW)
args.model_file = 'model_{}.t7'.format(MODELNAME)
args.config = MODELNAME
sys.stdout = Logger("./log/{}_{}.log".format(MODELNAME, TIMENOW))
else:
MODELNAME = args.model_file.split(".")[0]
args.config = MODELNAME
sys.stdout = Logger("./log/{}_{}.log".format(MODELNAME, TIMENOW))
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.device)
args.cuda = args.cuda and torch.cuda.is_available()
args.repeat = len(args.seed) if isinstance(args.seed,
list) else args.repeat
print("TimeStamp: {}\n".format(TIMENOW), json.dumps(vars(args), indent=2))
return args
def print_file(redis_conn):
"""
This function fetches file queue from redis server and
print the file using win32 print API
Args:
redis_conn(redis.Redis)
Returns:
void
Example:
redis_connection = StrictRedis()
print_file(redis_connection)
"""
win_logger = Logger.win_logger()
logger = win_logger.genLogger("win_print")
if redis_conn.ping():
filename_list = list(redis_conn.smembers("File_Queue"))
while len(filename_list) != 0:
file_name = ""
received_time = int(time.time())
filename = filename_list[0].decode('utf-8')
printed_file = [
i.decode('utf-8')
for i in list(redis_conn.smembers("Printed_File"))
]
if filename in printed_file:
file_name = filename[:-4] + "_" + str(
received_time) + "_" + filename[-4:]
else:
file_name = filename
if not redis_conn.hgetall(file_name):
redis_conn.hmset(
file_name, {
"filename": filename,
"queue_name": file_name,
"received_time": received_time,
"printed_status": "NO"
})
logger.info("Currently Printing %s" % file_name)
win32api.ShellExecute(0, "print", PRINT_PATH + filename,
win32print.GetDefaultPrinter(), ".", 0)
redis_conn.sadd("Printed_File", file_name)
redis_conn.hset(file_name, "printed_status", "YES")
redis_conn.srem("File_Queue", filename)
filename_list = list(redis_conn.smembers("File_Queue"))
else:
#print("Queue Server cannot be located")
logger.error("Queue Server cannot be located!")
default=1,
help='Number of repeated trials')
parser.add_argument('--node',
action='store_false',
default=True,
help='Use node-level attention or not. ')
parser.add_argument('--type',
action='store_false',
default=True,
help='Use type-level attention or not. ')
args = parser.parse_args()
dataset = args.dataset
args.cuda = not args.no_cuda and torch.cuda.is_available()
sys.stdout = Logger(logdir + "{}.log".format(dataset))
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
loss_list = dict()
def margin_loss(preds, y, weighted_sample=False):
nclass = y.shape[1]
preds = preds[:, :nclass]
y = y.float()
lam = 0.25
m = nn.Threshold(0., 0.)