def main():
from Args import Args
print("OutcomeStats.main()")
# print(type(self.outcomeFormats()))
args=Args('occurrence_qc.json', 'outcomeStats.xlsx', 'stats.ini')
workbook = xlsxwriter.Workbook(args.getOutfile())
worksheet = workbook.add_worksheet()
origin1 = [0,0]
origin2 = [5,0]
stats=OutcomeStats(workbook, worksheet,args, origin1, origin2)
def main():
from Args import Args
print("OutcomeStats.main()")
# print(type(self.outcomeFormats()))
args = Args('occurrence_qc.json', 'outcomeStats.xlsx', 'stats.ini')
workbook = xlsxwriter.Workbook(args.getOutfile())
worksheet = workbook.add_worksheet()
origin1 = [0, 0]
origin2 = [5, 0]
stats = OutcomeStats(workbook, worksheet, args, origin1, origin2)
def main(argv):
# Setup logging
setup_log()
logging.debug('Hello!')
# Print usage
if len(argv) <= 1:
abort('USAGE: jm module [module args]')
# Parse the arguments
args = Args.Args(argv[1:])
parse_global_config(args.args)
# Load the module
module = args.margs[0]
job = JobBinary(module)
# Remove JM arguments when passing to the module
margv = args.margs
# Wrapper to include job module
def run_wrapper(argv):
return run(argv, job)
# Run the module
logging.info('Running module')
r = job.spits_main(margv, run_wrapper)
# Finalize
logging.debug('Bye!')
def main(argv):
# Print usage
if len(argv) <= 1:
abort('USAGE: jm module [module args]')
# Parse the arguments
args = Args.Args(argv[1:])
parse_global_config(args.args)
# Setup logging
setup_log()
logging.debug('Hello!')
# Enable memory debugging
if jm_memstat == 1:
memstat.enable()
memstat.stats()
# Load the module
module = args.margs[0]
job = JobBinary(module)
# Remove JM arguments when passing to the module
margv = args.margs
# Keep a run identifier
runid = [0]
# Wrapper to include job module
def run_wrapper(argv, jobinfo):
runid[0] = runid[0] + 1
return run(argv, jobinfo, job, runid[0])
# Wrapper for the heartbeat
finished = [False]
def heartbeat_wrapper():
heartbeat(finished)
# Start the heartbeat
threading.Thread(target=heartbeat_wrapper).start()
# Run the module
logging.info('Running module')
memstat.stats()
r = job.spits_main(margv, run_wrapper)
memstat.stats()
# Stop the heartbeat thread
finished[0] = True
# Kill the workers
if jm_killtms:
killtms()
# Print final memory report
memstat.stats()
# Finalize
logging.debug('Bye!')
def main(argv):
if len(argv) <= 1:
abort('USAGE: tm [args] module [module args]')
args = Args.Args(argv[1:])
parse_global_config(args.args)
setup_log()
logging.debug('Hello!')
App(args).run()
logging.debug('Bye!')
def main():
args = Args({'-H': 'tgtHosts', '-r': 'redirectPage', '-f': 'userPassFile' })
options = args.getArgs()
tgtHosts = str(options.tgtHosts).split(', ')
passwordFile = options.userPassFile
redirect = options.redirectPage
for tgtHost in tgtHosts:
username = None
password = None
if anonLogin(tgtHost) == True:
username = '******'
password = '******'
print '[+] Using Anonymous Creds to Attack'
attack(username, password, tgtHost, redirect)
elif passwordFile != None:
(username, password) = bruteLogin(tgtHost, passwordFile)
if password != None:
print '[+] Using Creds: ' + username + '/' + 'password to attack'
attack(username, password, tgtHost, redirect)
def _cli_callable_from_params(params: List[str] = None) -> callable:
args: Args = Args(params)
if args.version:
return api.version_method
if args.clear_log:
return api.clear_log_method
if args.dump_log:
return partial(api.dump_log_method, args.dump_log)
return lambda: None
def get_argument():
""" Read the args and return a date object from the date=
the worksheets= and the optional bonus """
args = Args.Parse(date=str, worksheets=str, bonus=False)
try:
date = args.date
date = [int(x) for x in date.split('-')]
date = datetime(date[0], date[1], date[2]).date()
except:
print(f"Date {args.date} should be in format yyyy-mm-dd")
raise
try:
worksheets = args.worksheets.split(',')
except:
print(f"Worksheets {args.worksheets} should be in the form of "
"\"worksheet-demo,worksheet00-pseudo\"")
print("No blanks or quoates allowed by the parser Comma separated!")
raise
bonus = bool(args.bonus) # Force to True/Fals
return date, worksheets, bonus
from inputs import inputs
from glob import glob
import Args
import sys
from evalInContext import evalInContext
import os.path as osp
from datetime import datetime
import json
try:
import db
except ModuleNotFoundError:
print("no db module")
sys.exit(0)
args = Args.Parse(force=0, local=0, verbose=0)
bottle.SimpleTemplate.defaults["get_url"] = lambda x: x
bottle.SimpleTemplate.defaults["static"] = lambda x: x
def getAnswers(path):
"""Save the answer from a post"""
key = path.split("/")[-1].split(".")[0]
inp = inputs(key)
pages = None
section = '003'
team_number = 0
member_number = 0
rid = 1
# Render assuming no pages, but re-render w/ pages to get all the info
if os.name == 'nt':
defaultFontSize = 16
else:
defaultFontSize = 12
commandLineArgSpecs = (
('fontscheme', 0, 'scheme', 'fonts to use [eg pmw2] (Tk defaults)'),
('fontsize', 0, 'num', 'size of fonts to use with fontscheme',
defaultFontSize),
('stdout', 0, Args.Bool, 'print messages rather than display in label'),
)
program = 'All.py'
msg = Args.parseArgs(program, sys.argv, commandLineArgSpecs, 0)
if msg is not None:
print msg
sys.exit()
size = Args.get('fontsize')
fontScheme = Args.get('fontscheme')
Pmw.initialise(root, size=size, fontScheme=fontScheme, useTkOptionDb=1)
root.title('Pmw ' + Pmw.version() + ' megawidget demonstration')
if size < 18:
geometry = '800x550'
else:
geometry = '1000x700'
root.geometry(geometry)
def test_args_version_is_false_when_version_is_not_in_args_list(self):
assert not Args([]).version
def test_args_dump_log_defaults_to_cwd(self):
cwd: Path = Path()
params = ['--dump-log']
assert Args(params).dump_log == cwd
def test_unrecognized_arguments_raises_system_exit(self):
with pytest.raises(SystemExit), mock.patch('sys.stderr'):
Args(['--foo'])
def test_expected_params(self, params):
assert Args(params)
-xyz (if at least x,y have defaults)
abreviations allowed (arguments = 'name', 'territorial') (acceptable, -na -ter) etc... unless ambiguious
group = argparse.add_mutually_exclusive_group()
group.add_argument...
any arguments added, only 1 can be used or error
.set_defaults(**kwargs) --> will become default arguments even if the arguments were not created with .add_argument
.get_default('key') --> return the default of the key
--- useage ---
(above class saved in Args.py modual)
import Args
arg_parser = Args.Args()
inputArgs = arg_parser.parse()
inputArgs.ARGNAME #ie inputArgs.sample
--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
Looping
for x,y in zip(l1,l2): --> loop through two items at once
for k,v in dictionary.iteriteritems(): --> loop through keys and values in a dictionary
xrange(0,10,2) --> range of 0 to 10, by 2s
'''
Label the blobs using a previously trained model
Gary Bishop July 2018
'''
import pandas as pd
import Args
import pickle
from features import getFeatures
args = Args.Parse(inblobs='output.blobs.bz2',
outblobs='output.labeled.bz2',
model='models/LR1.pkl')
data = pd.read_pickle(args.inblobs)
model = pickle.load(open(args.model, 'rb'))
features = getFeatures(data)
labels = model.predict(features)
data.isdot = labels
data.to_pickle(args.outblobs)
def main():
# Parse arguments
Args.parse()
from bottle import route, request, static_file, run
import Args
import pandas as pd
import numpy as np
from PIL import Image
import base64
from io import BytesIO
args = Args.Parse(
blobs=str,
outblobs=str
)
data = pd.read_pickle(args.blobs)
data['colFromIndex'] = data.index
data = data.sort_values(['isdot', 'colFromIndex'])
perRow = 10
NROWS = 100
def toImage(img, title):
tpl = '<image title="{title}" src="data:image/png;base64,{data}" />'
pil_img = Image.fromarray(img)
buff = BytesIO()
pil_img.save(buff, format="png")
data = base64.b64encode(buff.getvalue()).decode("utf-8")
t = tpl.format(data=data, title=title)
return t
# import OutcomeFormats
from OutcomeStats import *
from OutcomeFormats import *
from Args import *
import argparse
# import unittest
# parser = argparse.ArgumentParser()
# parser.add_argument('--i',default='occurrence_qc.json', help="Defaults to occurrence_qc.json if '--i' absent")
# parser.add_argument('--o',default='outcomeStats.xlsx', help="Defaults to outcomeStats.xlsx if '--o' absent")
# parser.add_argument('--c',default='stats.ini', help="Defaults to stats.ini if --c absent")
# args = parser.parse_args()
# outfile = args.o
# args = parser.parse_args()
args = Args("occurrence_qc.json", "outcomeStats.xlsx", "stats.ini")
# Supply your favorite JSON output of FP-Akka as input. Do python3 statstest.py --help for help
# tested against FP-Akka 1.5.2 JSON output with python3
if __name__ == "__main__":
###################################################
############# First initialize resources ##########
###################################################
# set input jason file from FPAkka or elsewhere
# set output xlsx file
# set stats.ini configurations
args = Args("occurrence_qc.json", "outcomeStats.xlsx", "stats.ini")
# load entire jason file. (Note: syntactically it is a Dictionary !!! )
with open(args.getInfile()) as data_file:
fpAkkaOutput = json.load(data_file)
for f in os.listdir(kron_dir):
filename = os.fsdecode(f)
if filename.endswith(".txt"):
txt_files.append(filename)
elif filename.endswith(".dat"):
return utils.load_graph_list(os.path.join(kron_dir, filename))
G_list = []
for filename in txt_files:
G_list.append(
utils.snap_txt_output_to_nx(os.path.join(kron_dir, filename)))
return G_list
if __name__ == "__main__":
args = Args()
args_evaluate = Args_evaluate()
parser = argparse.ArgumentParser(description="Evaluation arguments.")
feature_parser = parser.add_mutually_exclusive_group(required=False)
feature_parser.add_argument("--export-real",
dest="export",
action="store_true")
feature_parser.add_argument("--no-export-real",
dest="export",
action="store_false")
feature_parser.add_argument(
"--kron-dir",
dest="kron_dir",
help="Directory where graphs generated by kronecker method is stored.",
)
#
# The idea is to create a new project on Transifex to get humans to help but to use the
# old THR translations and Google until then.
import Args # my args module
import polib
import re
import os.path as osp
import requests
import json
from apiclient.discovery import build
args = Args.Parse(
podir='', # path to dir with po files
src='src/Messages.ts', # where to find the needed strings
langs=str, # comma separataed list of 2-letter iso language codes
msgdir=str, # directory for the json files for translations
key=str, # Google API key
_config='scripts/translatekey.json')
# extract strings from my Messages.ts file
msgpat = re.compile(r"\s+(\w+):\s*'(.*)',")
enMessages = []
for line in open(args.src, 'r'):
m = msgpat.match(line)
if m:
enMessages.append(m.groups())
API_KEY = args.key
text = self.text
self.displayCommand(text)
if os.name == 'nt':
defaultFontSize = 16
else:
defaultFontSize = 12
commandLineArgSpecs = (
('fontscheme', 0, 'scheme', 'fonts to use [eg pmw2] (Tk defaults)'),
('fontsize', 0, 'num', 'size of fonts to use with fontscheme', defaultFontSize),
('stdout', 0, Args.Bool, 'print messages rather than display in label'),
)
program = 'All.py'
msg = Args.parseArgs(program, sys.argv, commandLineArgSpecs, 0)
if msg is not None:
print msg
sys.exit()
size = Args.get('fontsize')
fontScheme = Args.get('fontscheme')
Pmw.initialise(root, size = size, fontScheme = fontScheme, useTkOptionDb = 1)
root.title('Pmw ' + Pmw.version() + ' megawidget demonstration')
if size < 18:
geometry = '800x550'
else:
geometry = '1000x700'
root.geometry(geometry)
def test_args(self):
args = Args.Args(['--timeout=10', 'module', 'arg1', 'arg2'])
self.assertIn('timeout', args.args)
self.assertEqual(args.args['timeout'], '10')
self.assertSequenceEqual(['module', 'arg1', 'arg2'], args.margs)
def test_args_param_property_is_true_when_param_in_args_list(self, params, param_property):
args: Any = Args(params)
assert param_property.getter(args)
def test_parse_global_config(self):
args = Args.Args(['--timeout=10'])
tm.parse_global_config(args.args)
self.assertEqual(tm.tm_timeout, 10)
def test_args_dump_log_is_not_none_when_dump_log_dir_in_args_list(self):
params = ['--dump-log', '.']
assert Args(params).dump_log is not None
tf = item[1]
dl = file[item[0]][1]
TF = ((k1 + 1) * tf) / (k1 * (0.25 + 0.75 * dl / avdl) + tf)
### Add into Rank List
if item[0] not in rank: rank[item[0]] = [0] * len(term)
rank[item[0]][indx] = float(DF * TF * QF)
### Add into Vect List
vect.append(time)
indx += 1
return vect, rank
args = Args.Args()
vocb = Parse.ParseVocb(args.m)
file, avdl = Parse.ParseFile(args.m, args.d)
invt = Parse.ParseInvt(args.m)
qury = Parse.ParseQury(args.i)
print("")
N = len(file)
text = "query_id,retrieved_docs\n"
for temp in qury:
accm = {}
### Consider label k
for k in range(1, 5):
def test_non_existing_dir_raises_argument_type_error(self, tmp_path_as_cwd):
with pytest.raises(SystemExit), pytest.raises(argparse.ArgumentTypeError), mock.patch('sys.stderr'):
params = ['--dump-log', 'non_existing_dir']
Args(params)
elen = len(expected)
# we could parameterize these per question if desired.
dmin = max(1, 0.1 * elen)
dmax = max(2, 0.5 * elen)
p = (distance - dmin) / (dmax - dmin)
p = max(0, min(1, p))
factor = 0.75 * (1 - p) + 0.25 * p
return distance, factor * answer["points"]
if __name__ == "__main__":
args = Args.Parse(
testing=0,
verbose=0,
validate=0,
list=0, # set to non-zero to get a list of items to grade
force="", # set to force grading an item
user="", # set to select only one user
maxpenalty=0.50, # set to reduce max penalty
)
# ## Connect to the class db
db.init()
db = db.open_db()
cursor = db.cursor()
# ## Get the classroll
roll = pd.read_sql("""select * from roll""", db, index_col="onyen")
# find those who need grading
if args.force:
import xlsxwriter
#import OutcomeFormats
from OutcomeStats import *
from OutcomeFormats import *
from Args import *
import argparse
#import unittest
#parser = argparse.ArgumentParser()
#parser.add_argument('--i',default='occurrence_qc.json', help="Defaults to occurrence_qc.json if '--i' absent")
#parser.add_argument('--o',default='outcomeStats.xlsx', help="Defaults to outcomeStats.xlsx if '--o' absent")
#parser.add_argument('--c',default='stats.ini', help="Defaults to stats.ini if --c absent")
#args = parser.parse_args()
#outfile = args.o
#args = parser.parse_args()
args=Args('occurrence_qc.json', 'outcomeStats.xlsx', 'stats.ini')
#Supply your favorite JSON output of FP-Akka as input. Do python3 statstest.py --help for help
#tested against FP-Akka 1.5.2 JSON output with python3
if __name__=="__main__":
###################################################
############# First initialize resources ##########
###################################################
# set input jason file from FPAkka or elsewhere
# set output xlsx file
# set stats.ini configurations
args=Args('occurrence_qc.json', 'outcomeStats.xlsx', 'stats.ini')
#load entire jason file. (Note: syntactically it is a Dictionary !!! )
with open(args.getInfile()) as data_file:
fpAkkaOutput=json.load(data_file)
def get_parser():
main_parser = argparse.ArgumentParser()
main_parser.add_argument('--version',
action='version',
version=version(),
help='display version information and exit')
subparsers = main_parser.add_subparsers(title='modules', dest='which')
global_parser = argparse.ArgumentParser(add_help=False)
settings_parser = Args.settings_args(subparsers.add_parser('settings', help='user definable settings', parents=[global_parser]))
snv_parser = Args.snv_args(subparsers.add_parser('snv', help='run single nucleotide variant association models', parents=[global_parser]))
snvgroup_parser = Args.snvgroup_args(subparsers.add_parser('snvgroup', help='run variant group (ie. gene) based association models', parents=[global_parser]))
meta_parser = Args.meta_args(subparsers.add_parser('meta', help='run meta analysis', parents=[global_parser]))
compile_parser = Args.compile_args(subparsers.add_parser('compile', help='verify and compile results files', parents=[global_parser]))
resubmit_parser = Args.resubmit_args(subparsers.add_parser('resubmit', help='resubmit failed results', parents=[global_parser]))
snvplot_parser = Args.snvplot_args(subparsers.add_parser('snvplot', help='generate qq and manhattan plots for single variant results', parents=[global_parser]))
snvgroupplot_parser = Args.snvgroupplot_args(subparsers.add_parser('snvgroupplot', help='generate qq and manhattan plots for grouped variant results', parents=[global_parser]))
filter_parser = Args.filter_args(subparsers.add_parser('filter', help='filter results / correct single variant results for genomic inflation', parents=[global_parser]))
merge_parser = Args.merge_args(subparsers.add_parser('merge', help='merge and annotate results with external files / snpEff', parents=[global_parser]))
tools_parser = Args.tools_args(subparsers.add_parser('tools', help='run any command line tool with ability to include genomic region automatically', parents=[global_parser]))
return main_parser
'''
Extract fixations from videos by finding the blue dots
Gary Bishop July 2018
'''
import cv2
import numpy as np
import pandas as pd
import Args
sampleVideo = ('/home/gb/Dropbox/Karen and Gary Shared Files/'
'Videos & Transcripts/MSB/MSB_Video 1 (09-30-17).mp4')
args = Args.Parse(video=sampleVideo,
start=100,
end=3680,
blobs='output.blobs.bz2')
vc = cv2.VideoCapture(args.video)
def grabFrame(fn):
'''
Extract a frame and convert it to LAB float32 format
'''
vc.set(cv2.CAP_PROP_POS_FRAMES, fn)
rval, im = vc.read()
im = cv2.cvtColor(im.astype(np.float32) / 255.0, cv2.COLOR_BGR2LAB)
return im
def main():
parser = Args.Parser().getParser()
args = parser.parse_args()
processors = {
'zy': ZYProcessor,
}
device = torch.device("cuda:3" if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO)
if not args.do_train and not args.do_eval and not args.do_predict:
raise ValueError(
"At least one of `do_train` or `do_eval` or `do_predict` must be True."
)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]()
label_list = processor.get_labels()
num_labels = len(label_list)
tokenizer = BertTokenizer.from_pretrained(args.vocab_file,
do_lower_case=args.do_lower_case)
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_optimization_steps = int(
len(train_examples) /
args.train_batch_size) * args.num_train_epochs
model = BertForMultiSequenceClassification.from_pretrained(
args.model_dir, num_labels=num_labels)
model.to(device)
# Prepare optimizer
if args.do_train:
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in param_optimizer
if not any(nd in n for nd in no_decay)
],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_optimization_steps)
if args.do_train:
train_features = convert_examples_to_features(train_examples,
label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_ids for f in train_features],
dtype=torch.float32)
train_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data,
sampler=train_sampler,
batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
for step, batch in enumerate(
tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
logits, _ = model(input_ids,
segment_ids,
input_mask,
labels=None)
loss_fct = torch.nn.BCEWithLogitsLoss()
loss = loss_fct(logits, label_ids)
loss.backward()
optimizer.step()
optimizer.zero_grad()
logger.info('')
logger.info('step: %d' % (step))
logger.info('loss: %f' % (loss.item()))
model_to_save = model.module if hasattr(
model, 'module') else model # Only save the model it-self
# If we save using the predefined names, we can load using `from_pretrained`
output_model_file = os.path.join(args.output_dir, WEIGHTS_NAME)
output_config_file = os.path.join(args.output_dir, CONFIG_NAME)
torch.save(model_to_save.state_dict(), output_model_file)
model_to_save.config.to_json_file(output_config_file)
tokenizer.save_vocabulary(args.output_dir)
if args.do_eval:
model = BertForMultiSequenceClassification.from_pretrained(
args.output_dir, num_labels=num_labels)
tokenizer = BertTokenizer.from_pretrained(
args.output_dir, do_lower_case=args.do_lower_case)
model.to(device)
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(eval_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_ids for f in eval_features],
dtype=torch.float32)
eval_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=args.eval_batch_size)
model.eval()
recall = 0
precision = 0
F1 = 0
steps = 0
for input_ids, input_mask, segment_ids, label_ids in tqdm(
eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
logits, probilities = model(input_ids,
segment_ids,
input_mask,
labels=None)
loss_fct = torch.nn.BCEWithLogitsLoss()
loss = loss_fct(logits, label_ids)
pred = probilities.detach().cpu().numpy()
label = label_ids.detach().cpu().numpy()
pred[pred > 0.5] = 1.0
pred[pred <= 0.5] = 0.0
steps += 1
F1 += f1_score(y_true=label, y_pred=pred, average='micro')
precision += precision_score(y_true=label,
y_pred=pred,
average='micro')
recall += recall_score(y_true=label, y_pred=pred, average='micro')
logger.info('')
logger.info('loss: %f' % (loss.item()))
logger.info('recall: %f' % (recall / steps))
logger.info('precision %f' % (precision / steps))
logger.info('f1: %f' % (F1 / steps))
if args.do_predict:
model = BertForMultiSequenceClassification.from_pretrained(
args.output_dir, num_labels=num_labels)
tokenizer = BertTokenizer.from_pretrained(
args.output_dir, do_lower_case=args.do_lower_case)
model.to(device)
pred_examples = processor.get_test_examples(args.data_dir)
pred_features = convert_examples_to_features(pred_examples, label_list,
args.max_seq_length,
tokenizer)
logger.info("***** Running prediction *****")
logger.info(" Num examples = %d", len(pred_examples))
logger.info(" Batch size = %d", args.predict_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in pred_features],
dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in pred_features],
dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in pred_features],
dtype=torch.long)
all_label_ids = torch.tensor([f.label_ids for f in pred_features],
dtype=torch.float32)
pred_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, all_label_ids)
# Run prediction for full data
pred_sampler = SequentialSampler(pred_data)
pred_dataloader = DataLoader(pred_data,
sampler=pred_sampler,
batch_size=args.predict_batch_size)
model.eval()
preds = []
for input_ids, input_mask, segment_ids, label_ids in tqdm(
pred_dataloader, desc="predicting"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
_, probilities = model(input_ids,
segment_ids,
input_mask,
labels=None)
pred = probilities.detach().cpu().numpy()
pred = pred.tolist()
preds.append(pred)
write_path = "./output/predicate_infer_out/"
if not os.path.exists(write_path):
os.makedirs(write_path)
score_file = "./output/predicate_infer_out/probility.txt"
predict_file = "./output/predicate_infer_out/predicate_predict.txt"
logger.info('')
logger.info('***********writing predict result**********')
with open(score_file, 'w', encoding='utf-8') as score_writer:
with open(predict_file, 'w', encoding='utf-8') as predict_writer:
num_total_lines = 0
for batch in preds:
for lines in batch:
score = ' '.join(str(number)
for number in lines) + '\n'
score_writer.write(score)
predict_relation = []
for idx, prob in enumerate(lines):
if prob > 0.5:
predict_relation.append(label_list[idx])
predict = ' '.join(predict_relation) + '\n'
predict_writer.write(predict)
num_total_lines += 1
assert num_total_lines == len(pred_examples)
