def main(args):
set_seed(args.seed)
Path(args.model_dir).mkdir(parents=True, exist_ok=True)
tk = Tokenizer(args.tokenizer)
model = TransformerModel(d_model=32,
d_ff=64,
dropout=.0,
layers=3,
heads=4,
d_emb=-1,
pad_token_id=tk.pad_id,
vocab_size=tk.vocab_size)
ds = Arithmetic(args.data)
print(
f'model size = {sum(p.numel() for p in model.parameters() if p.requires_grad)/1024/1024:.2f} M trainable parameters'
)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train(model=model,
dataset=ds,
batch_size=args.batch_size,
device=device,
tokenizer=tk,
epochs=args.epochs,
model_dir=args.model_dir,
save_epoch=args.save_epoch,
summary_step=args.summary_step,
lr=args.lr)
def __init__(self,
tokenizer_path: str,
model: UnifiedTransformer,
max_turns: int = 64,
device: Optional[str] = None):
self.tokenizer = Tokenizer(tokenizer_path=tokenizer_path)
self.batch_preparer = BatchPreparing(
sep_index=self.tokenizer.sep_index,
context_index=self.tokenizer.context_index,
pad_index=self.tokenizer.pad_index)
self.model = model
self.max_turns = max_turns
if device is None:
self.device = next(self.model.parameters()).device
else:
self.device = torch.device(device)
self.model.to(self.device)
self.model.eval()
self.model.set_seq2seq()
def endElement(self, tag):
"""
Signals the end of an element in non-namespace mode.
"""
if tag == "title":
# initialize a new document with title ==self.title
self.tokenizer = Tokenizer(self.title)
self.tokenizer.set_title(self.title)
elif tag == "text":
# By now the document title and id fields must have been extracted
Helpers.docid_docname_map[
self.tokenizer.get_doc_id()] = self.tokenizer.get_title()
# add text body to that document # TODO: use append
termid_freq_map = self.tokenizer.tokenize(self.text)
# print("term_termid_map", Helpers.term_termid_map)
for term in termid_freq_map:
# accumulate (termid: docid) pairs
Indexer.termid_docid_list.append(
(term, self.tokenizer.get_doc_id()))
elif tag == "id" and not self.insideRevision:
# DoNOT set id if inside <revision> <id>XXX</id>
self.tokenizer.set_doc_id(self.id)
elif tag == "revision":
self.insideRevision = False # </revision> encountered
self.tag = None
def __init__(self, text):
"""
:param text: client input
"""
self.text = text
self.tokenizer = Tokenizer(self.text)
self.parser = Parser(self.tokenizer.create_tokens())
self.GLOBAL_VARS = dict()
def test_throws_if_next_invoked_while_status_is_FINISHED(self):
mock_func = Mock()
tokenizer = Tokenizer('')
tokenizer._generator = mock_func
tokenizer._status = Tokenizer.EStatus.FINISHED
with self.assertRaisesRegex(TokenizerError, '(f|F)inished'):
tokenizer.next()
self.assertFalse(mock_func.called)
def process(filename):
with open(filename) as file:
text = file.read()
if not text:
raise Exception("Cannot read text from file")
tokenizer = Tokenizer(text)
parser = Parser(tokenizer.create_tokens())
tree = parser.parse()
symbol_table_builder = SemanticAnalyzer()
symbol_table_builder.visit(tree)
def test_returns_tokens_until_exhausted(self):
expected = [
Token('class', 'keyword', 1, 1),
Token('CorrectSyntax', 'identifier', 1, 7),
Token('{', 'symbol', 1, 21),
Token('field', 'keyword', 2, 3),
Token('String', 'identifier', 2, 9),
Token('bar', 'identifier', 2, 16),
Token(';', 'symbol', 2, 19),
Token('constructor', 'keyword', 4, 3),
Token('CorrectSyntax', 'identifier', 4, 15),
Token('new', 'identifier', 4, 29),
Token('(', 'symbol', 4, 32),
Token(')', 'symbol', 4, 33),
Token('{', 'symbol', 4, 35),
Token('let', 'keyword', 5, 5),
Token('bar', 'identifier', 5, 9),
Token('=', 'symbol', 5, 13),
Token('Hello world!', 'stringConst', 5, 15),
Token(';', 'symbol', 5, 29),
Token('return', 'keyword', 6, 5),
Token('this', 'keyword', 6, 12),
Token(';', 'symbol', 6, 16),
Token('}', 'symbol', 7, 3),
Token('method', 'keyword', 9, 3),
Token('void', 'keyword', 9, 10),
Token('greetings', 'identifier', 9, 15),
Token('(', 'symbol', 9, 24),
Token(')', 'symbol', 9, 25),
Token('{', 'symbol', 9, 27),
Token('do', 'keyword', 10, 5),
Token('Output', 'identifier', 10, 8),
Token('.', 'symbol', 10, 14),
Token('printString', 'identifier', 10, 15),
Token('(', 'symbol', 10, 26),
Token('bar', 'identifier', 10, 27),
Token(')', 'symbol', 10, 30),
Token(';', 'symbol', 10, 31),
Token('return', 'keyword', 11, 5),
Token(';', 'symbol', 11, 11),
Token('}', 'symbol', 12, 3),
Token('}', 'symbol', 13, 1)
]
filename = TEST_FILES / 'CorrectSyntax.jack'
tokenizer = Tokenizer(filename)
tokens = []
while not tokenizer.finished():
token = tokenizer.next()
if not token:
break
tokens.append(token)
self.assertEqual(tokens, expected)
def test_throws_if_string_constant_has_unescaped_double_quote_within(self):
filename = TEST_FILES / 'UnescapedDoubleQuotesInString.jack'
tokenizer = Tokenizer(filename)
line = 5
character = 15
for i in range(16):
tokenizer.next()
with self.assertRaisesRegex(
TokenizerError,
f'(L|l)ine.*{line}.*(C|c)haracter.*{character}'):
tokenizer.next()
def test_throws_if_string_constant_has_missing_end_double_quote(self):
filename = TEST_FILES / 'MalformedString.jack'
tokenizer = Tokenizer(filename)
line = 5
character = 15
for i in range(16):
tokenizer.next()
with self.assertRaisesRegex(
TokenizerError,
f'(L|l)ine.*{line}.*(C|c)haracter.*{character}'):
tokenizer.next()
def __init__(self, trec_dir, stopword_file):
parser = Parser()
for filename in os.listdir(trec_dir):
try:
with open(os.path.join(trec_dir, filename), 'r') as f:
parser.load(f.read())
except Exception as e:
pass
self.docs = parser.docs
with open(stopword_file, 'r') as f:
stopwords = f.readlines()
self.tokenizer = Tokenizer(stopwords)
def compile_all_jack_files(jack_files):
try:
os.mkdir("xml-export")
except:
pass
for jack_file in jack_files:
with open(jack_file) as file:
code = remove_all_comments(file.read())
tokenizer = Tokenizer(code)
tokenizer.export_xml(jack_file.replace(".jack", ""))
try:
parser = Parser(tokenizer)
parser.export_xml(jack_file.replace(".jack", ""))
except Exception as e:
print(e)
exit()
def parse(self):
"""
Replace sentiment with 2: Positive, 1: Neutral, 0: Negative
"""
tk = Tokenizer(preserve_case=False)
with open(self.trainingPath) as training:
tsvRead = csv.reader(training, delimiter="\t")
enum = {'positive': 2, 'neutral': 1, 'negative': 0, 'unknown':3}
tweet_dict = {}
for line in tsvRead:
if tk.tokenize(line[1]):
phrase = tk.tokenize(line[1])
for i,word in enumerate(phrase):
if i>50 and word in ["neutral","positive","negative","unknown"]:
phrase = phrase[:i]
break
self.data.append({'Sentiment' : enum[line[0]], 'Tweet' : phrase})
def test_skips_everything_inbetween_multiline_comment(self):
expected = [
Token('let', 'keyword', 5, 5),
Token('foo', 'identifier', 5, 9),
Token('=', 'symbol', 5, 13),
Token('5', 'intConst', 5, 15),
Token(';', 'symbol', 5, 16)
]
filename = TEST_FILES / 'IgnoreMultilineComment.jack'
tokenizer = Tokenizer(filename)
tokens = []
while not tokenizer.finished():
token = tokenizer.next()
if not token:
break
tokens.append(token)
self.assertEqual(tokens, expected)
def test_skips_rest_of_line_after_line_comment(self):
expected = [
Token('let', 'keyword', 1, 1),
Token('foo', 'identifier', 1, 5),
Token('=', 'symbol', 1, 9),
Token('5', 'intConst', 1, 11),
Token(';', 'symbol', 1, 12)
]
filename = TEST_FILES / 'IgnoreLineComment.jack'
tokenizer = Tokenizer(filename)
tokens = []
while not tokenizer.finished():
token = tokenizer.next()
if not token:
break
tokens.append(token)
self.assertEqual(tokens, expected)
def run(self):
if self.config.download:
logger.info('Download')
collector.download()
if self.config.train_bpe:
logger.info('Train BPE')
collector.train_bpe()
self.tokenizer = Tokenizer(tokenizer_path=self.bpe_model_path,
need_bos=True,
need_eos=True,
sep_token=self.config.sep_token,
context_token=self.config.context_token)
if self.config.collect_data:
logger.info('Parse data')
self.make_dir(self.train_dir, override=True)
self.make_dir(self.validation_dir, override=True)
collector.collect()
def build_index():
global Index, Header
tokenizer = Tokenizer()
for subdir in os.listdir(config.RAW_WEBPAGES):
full_subdir = os.path.join(config.RAW_WEBPAGES, subdir)
if os.path.isdir(full_subdir):
to_parse = read_directory(full_subdir)
print("Subdirectory: ", subdir)
for _file in tqdm(to_parse):
filename = "/".join(_file.split("/")[1:])
header, txt = parse(_file)
Header[filename] = header
token_counter = tokenizer.counter_tokenize(txt)
for tok in token_counter:
if tok not in Index:
Index[tok] = {filename: token_counter[tok]}
else:
Index[tok][filename] = token_counter[tok]
save_index()
save_header()
def main(args):
set_seed(args.seed)
Path(args.model_dir).mkdir(parents=True, exist_ok=True)
tk = Tokenizer(args.tokenizer)
model = TransformerModel(
d_model=768,
d_ff=1024,
dropout=args.dropout,
layers=args.layer,
heads=args.heads,
d_emb=-1,
pad_token_id=tk.pad_id,
vocab_size=tk.vocab_size
)
ds = NewsDataset(args.data, args.alpha, args.beta, inplace=args.inplace,
sample=args.sample, seed=args.seed)
print(
f'model size = {sum(p.numel() for p in model.parameters() if p.requires_grad)/1024/1024:.2f} M trainable parameters')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
train(
model=model,
dataset=ds,
batch_size=args.batch_size,
device=device,
tokenizer=tk,
epochs=args.epochs,
model_dir=args.model_dir,
save_epoch=args.save_epoch,
summary_step=args.summary_step,
lr=args.lr,
warnup=args.warnup)
torch.cuda.empty_cache()
def main(args):
set_seed(args.seed)
tk = Tokenizer(args.tokenizer)
ds = Arithmetic(args.data)
dl = torch.utils.data.DataLoader(dataset=ds,
batch_size=args.batch_size,
shuffle=False,
collate_fn=ds.get_collate_fn(tk))
model = TransformerModel(d_model=32,
d_ff=64,
dropout=.0,
layers=3,
heads=4,
d_emb=-1,
pad_token_id=tk.pad_id,
vocab_size=tk.vocab_size)
device = torch.device(args.device)
model.load_state_dict(torch.load(args.ckpt, map_location=device)['model'])
model.to(device)
start = timeit.default_timer()
total = 0
tp = 0
for x, y in tqdm(dl):
p = beam_search_v2(model, x, tk, lambda b, nx, ny:
(nx + ny) * b > 4096 * 6 * 64, 1, device, 10)
r = list(map(lambda i: i[0] == i[1], zip(p, y)))
total += len(r)
tp += np.count_nonzero(r)
print((timeit.default_timer() - start))
print(tp / total)
def main():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("prop", help="Propositional statement")
arg_parser.add_argument("--tokens", "-t", action="store_true")
arg_parser.add_argument("--ast", "-a", action="store_true")
arg_parser.add_argument("--symbols", "-s", action="store_true")
arg_parser.add_argument("--truthtable", "-tt", nargs='?', const=1, type=int)
args = arg_parser.parse_args()
if not args.prop:
args.prop = " "
error_collector = ErrorCollector(args.prop)
symbol_table = SymbolTable()
tokenizer = Tokenizer(error_collector, symbol_table)
parser = Parser(tokenizer, error_collector)
ast = parser.parse()
# Para hacer debug jeje
if args.symbols:
symbol_table.show_symbols()
if error_collector.has_errors():
error_collector.show_errors()
return
if args.tokens:
tokenizer.show_tokens()
if args.ast:
print(ast)
if args.truthtable:
tt = TruthTable(ast, symbol_table, args.truthtable)
tt.show()
def run(src: str) -> None:
global env
#tokenization
tkz = Tokenizer()
tokens, err = tkz.tokenize(src)
if tok_debug:
for i in tokens:
print(i)
if display_errors(err, "LOX: SYNTAX ERROR"):
return
#don't send single EOF token to parser
#this allows parser to make stricter assertions while generating the AST
if tokens[0].type == TokenType.EOF:
return
#parsing
prs = Parser()
program, err = prs.parse(tokens)
if parse_debug:
for tree in program:
print(tree)
if display_errors(err, "LOX: GRAMMAR ERROR"):
return
#interpretation
itr = Interpreter(env)
exit_status, err, env = itr.interpret(program)
display_errors(err, "LOX: RUNTIME ERROR")
if env_debug:
print(env.map)
def main(args):
set_seed(args.seed) if args.seed is not None else None
tk = Tokenizer(args.tokenizer)
model = TransformerModel(d_model=768,
d_ff=1024,
dropout=0,
layers=args.layer,
heads=args.heads,
d_emb=-1,
pad_token_id=tk.pad_id,
vocab_size=tk.vocab_size)
ds = NewsDataset(args.data,
args.a,
args.b,
inplace=args.inplace,
sample=args.sample,
seed=args.seed)
device = torch.device(args.device)
@torch.no_grad()
def find_dist(model):
model.eval()
probs = [[] for i in range(tk.vocab_size)]
dl = torch.utils.data.DataLoader(
dataset=ds,
batch_size=args.batch_size,
shuffle=False,
collate_fn=ds.get_collate_fn(tk),
num_workers=2,
)
def pad(inseq, max_len):
return torch.nn.utils.rnn.pad_sequence(list(
map(lambda x: torch.LongTensor(x[:max_len]), inseq)),
padding_value=0,
batch_first=True).to(device)
tkcount = 0
for x, y in tqdm(dl, smoothing=0):
tkcount += sum([len(i) for i in y])
x = pad(x, 512)
y = pad(y, 513)
# B S V
p = torch.nn.functional.softmax(model(x, y[:, :-1]), dim=-1)
# B S
r = torch.gather(p, -1, y[:, 1:].unsqueeze(-1)).squeeze(-1).cpu()
for vids, ps in zip(y[:, 1:], r):
for vid, p in zip(vids, ps):
probs[vid].append(float(p))
return probs
if args.ckpt == 'latest':
args.ckpt = find_latest_ckpt(args.model_dir, args.ckpt_pattern)
model.load_state_dict(torch.load(args.ckpt, map_location=device)['model'])
model.to(device)
results = find_dist(model)
pickle.dump(results, open(args.output, 'wb'))
def __init__(self):
super().__init__()
self.tokenizer = Tokenizer()
self.index = defaultdict(list)
lambda x: (x[0], float(x[1])),
map(lambda x: x.strip().split('\t'),
filter(len,
open('data/t2.1.vocab').readlines()))))
w, f = zip(*d)
f = np.array(f)
print(np.exp(f[39:]).sum())
l = list(map(len, w[39:]))
freq = sorted(dict(Counter(l)).items(), key=lambda x: x[0])
print('\n'.join(f'|{i}|{j}|' for i, j in freq))
# In[]
ds = NewsDataset('data/news_dataset_tag10_v2.1.db')
# ds = NewsDataset('data/wiki.db')
tk = Tokenizer('data/t2.1_c1')
# In[]
from src.utils import peek
d = peek(ds.data, 1)
print(d[0][2])
print(tk.detokenize(tk.tokenize(d[0][2])))
# In[]
ll = list(map(lambda x: len(x[2]), ds.data))
sl = sorted(ll)
print(sl[0])
print(sl[int(len(sl) * 0.25)])
print(sl[int(len(sl) * 0.5)])
print(sl[int(len(sl) * 0.75)])
print(sl[-1])
def parse(code):
return Parser(Tokenizer(code).run()).run()
arg_parser.add_argument('--prefix', type=str, default='char_level_gru')
arg_parser.add_argument('--metric_name', type=str, default='val_ppl')
arg_parser.add_argument('--device', choices=['cuda', 'cpu'], default='cpu')
arg_parser.add_argument('--seed', type=int, default=42)
args = arg_parser.parse_args()
seed_all(args.seed)
print('########################################')
print('Load data')
train_data = WordsDataset._read_data(args.train_path)
valid_data = WordsDataset._read_data(args.valid_path)
print(f'Train size: {len(train_data)}\nValid size: {len(valid_data)}')
print('########################################')
tokenizer = Tokenizer(train_data)
print('########################################')
print('Build datasets')
train_dataset = WordsDataset(train_data,
tokenizer,
max_length=args.max_length)
valid_dataset = WordsDataset(valid_data,
tokenizer,
max_length=args.max_length)
print('########################################')
print('Build dataloaders')
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True)
def setUp(self):
self.text_input = [
'Hi! , How are you?', 'How everything is going?', 'how old are you'
]
self.tokenizer = Tokenizer()
config.read(user_config, encoding=encoding)
for section, options in sections.items():
if not config.has_section(section):
raise ValueError(
"Config is not valid! Section '{}' is missing!".format(section))
for option in options:
if not config.has_option(section, option):
raise ValueError(
"Config is not valid! Option '{}' in section '{}' is missing!".
format(option, section))
# IOC
from src.redis_c import Redis
redis = Redis(config)
from src.tokenizer import Tokenizer
tokenizer = Tokenizer()
from src.repository import *
trigram_repository = TrigramRepository()
chance_repository = ChanceRepository()
media_repository = MediaRepository()
job_repository = JobRepository()
from src.service import *
data_learner = DataLearner()
reply_generator = ReplyGenerator()
media_checker = MediaUniquenessChecker()
chat_purge_queue = ChatPurgeQueue()
def __init__(self, dict_path):
super().__init__()
self.dict_path = dict_path
self.dictionary = set()
self.matrix = {}
self.tokenizer = Tokenizer()
def __init__(self, text, source_file=None):
self.tokenizer = Tokenizer(text, source_file)
def __init__(self):
super().__init__()
self.dictionary = set()
self.count = 0
self.tokenizer = Tokenizer()