def second_pass(self, file_lines):
memory_address = self.MEM_START_ADDR
for line in file_lines:
parser = Parser(instruction=line)
encoder = Encoder(instruction_type=parser.instruction_type)
if parser.instruction_type == InstructionType.c_instruction:
hack_line = encoder.encode(dest=parser.dest,
comp=parser.comp,
jump=parser.jump)
elif parser.instruction_type == InstructionType.a_instruction:
try:
integer_address = int(parser.address)
except ValueError:
if self.symbol_table.get(parser.address) is None:
self.symbol_table[parser.address] = memory_address
memory_address += 1
integer_address = self.symbol_table.get(parser.address)
hack_line = encoder.encode(address=integer_address)
else:
continue
self.hack_file.write(hack_line + '\r\n')
def run_tests():
e = Encoder()
d = Decoder(coding_polynomial=e.coding_polynomial,
k=e.k,
t=e.r,
gf_index=e.gf.index)
message = "zaqwsxcderfvbgtyhnmjuik,ol.p;/zaqwsxedcrfvtgbyhnujmzaqwsxcderf"
codeword = e.encode(message)
decoded_message = d.decode(codeword, 'basic')
for i in range(2, 28):
codeword.elements[i] = codeword.elements[i].multiplicative_inversion()
print('27 errors occurred...')
print(codeword)
decoded_message = d.decode(codeword, 'basic')
print('Decoded message: ' + decoded_message[:len(message)])
def train_model(dpath, ppath, epoch, version):
if dpath.endswith(".csv"):
d = pd.read_csv(dpath)
else:
raise ValueError("data format is not supported")
pipe = joblib.load(ppath)
encoder = Encoder(pipe)
x = encoder.encode(d.iloc[:, 1:-1])
m = create_model(
[
x.shape[1],
]
)
m.fit(x, d.iloc[:, -1], batch_size=1000, epochs=epoch)
m.save(f"model/{version}")
def test_integration(model_service, xy):
p = Path(curdir / "saved_model")
assert "http://127.0.0.1:8501" == model_service
fname = curdir / ".tmp.joblib"
train_sk_pipe(fname, xy[0])
assert os.path.exists(fname)
pipe = load(fname)
encoder = Encoder(pipe)
matrixs = encoder.encode(xy[0][:100]).tolist()
res = requests.post(
model_service + "/v1/models/tp_pred:predict",
data=json.dumps({"instances": matrixs}),
)
assert len(res.json()["predictions"]) == 100
def run_example_program():
e = Encoder()
d = Decoder(coding_polynomial=e.coding_polynomial,
k=e.k,
t=e.r,
gf_index=e.gf.index)
message = "zaqwsxcderfvbgtyhnmjuik,ol.p;/zaqwsxedcrf"
print('Message: ' + message)
codeword = e.encode(message)
print('Codeword: ' + str(codeword))
decoded_message = d.decode(codeword, 'basic')
print('Decoded message: ' + decoded_message[:len(message)])
for i in range(1, 28):
codeword.elements[i] = codeword.elements[i].multiplicative_inversion()
print('27 errors occurred...')
print(codeword)
decoded_message = d.decode(codeword, 'basic')
print('Decoded message: ' + decoded_message[:len(message)])
def test_encoder_fix_errors(ii, k, test_type, message):
e = Encoder()
d = Decoder(coding_polynomial=e.coding_polynomial, k=e.k, t=e.r, gf_index=e.gf.index)
encoded_message = e.encode(message)
if test_type == 'multiple':
random_indexes = random.sample(range(0, len(encoded_message)), k)
else:
random_start = random.randint(0, len(encoded_message)-k-1)
random_indexes = [i for i in range(random_start, random_start + k)]
# print("{}): {}".format(k, random_indexes))
for i in random_indexes:
encoded_message.elements[i] = encoded_message.elements[i].multiplicative_inversion()
try:
start = time.time()
decoded_message = d.decode(encoded_message, 'basic')
stop = time.time()
passed.write("{}, {}, {}, {}, {}\n".format(k, test_type, message, random_indexes, stop-start))
except CannotDetectErrorException as c:
failed.write("{}, {}, {}, {}\n".format(k, test_type, message, random_indexes))
assert False
assert message in decoded_message
class Guesser():
def __init__(self,
model="gpt2",
interact=False,
score=False,
topK=10,
Log=None,
Scorer=None):
self.model = model
self.topK = topK
self._build(Log, Scorer)
def _build(self, Log, Scorer):
if Log == None:
Log = Logger()
if Scorer == None:
Scorer = Score()
self.Log = Log
self.Scorer = Scorer
self.Encoder = Encoder()
self.GPT = GPT2LanguageModel(model_name=self.model)
def _getBestWords(self, text):
''' Creates finds best words and calculates their propbablity of occuring '''
logits = self.GPT.predict(text, "")
best_logits, best_indices = logits.topk(self.topK)
# converts best indicie list into a list of words
best_words = [self.GPT[idx.item()] for idx in best_indices]
# calculates probabilities
probabilities = torch.nn.functional.softmax(logits)
# creates a list of probabilites based on best_indicies. This is a parallel array to best_words
best_probabilities = self._getPropability(
probabilities[best_indices].tolist())
# returns a list of tuples. each tuple contains the world in position 0 and the probability in position 1
return [(best_words[i], best_probabilities[i])
for i in range(self.topK)]
def _getPropability(self, probabilities):
''' returns top-k Propabilities from GPT-2 '''
return [round(p * 100, 2) for p in probabilities]
def _run(self, text, guess):
''' scores inputted text and logs it '''
# gives a list of the best words that GPT predicts
guessList = self._getBestWords(text)
self.Log.Info(("Answer List : {}".format(guessList)))
# scores guess word with the list of preditions
score = self.Scorer.score(guessList, guess)
self.Log.Info(score)
def start(self, text=""):
'''
Starts The Program
text: Text to be fed into GPT. If left blank initiate interactive mode
'''
if text != "":
encoded_text = self.Encoder.encode(text=text)
for text, guess in encoded_text[0]:
if text == '':
continue
self._run(text, guess)
else:
while True:
text = self.Log.Input("Input Text >> ")
if text == "":
self.Log.Info("Please provide a valid input")
continue
if text == "#?":
self.Log.Info(
"Available Commands: \n#?: Shows available commands\n#end: Ends Execution"
)
continue
if text == "#end":
self.Log.Info("Ending Program")
break
guess = self.Log.Input("What will the next word be >> ")
self._run(text, guess)
self.Log.Info("Score: {}".format(self.Scorer.calcScore()))
class pyReadability():
'''
Main class for the applicaiton. Loads the data, encodes it and runs scoring algortim.
'''
def __init__(self,
model,
interact,
topK,
seed,
mod,
probability,
Log=None):
''' Take in variables and starts the program. See readme for inputs'''
self.model = model
self.topK = topK
self.interact = interact
self._seed = seed
self._probability = probability
self._score = -1
self._build(mod, Log)
def _build(self, mod, Log):
''' Builds application using variables provided by user '''
if Log == None:
Log = Log()
if (self._seed < 1):
random.seed(time.time())
self._seed = random.random()
self.Log = Log
self.Scorer = Score(mod, self.Log)
self.Encoder = Encoder(seed=self._seed, probability=self._probability)
self.GPT = GPT2LanguageModel(model_name=self.model)
def _run(self, text):
''' Runs GTP and gets results '''
logits = self.GPT.predict(text, "")
probabilities = torch.nn.functional.softmax(logits)
best_indices = logits.topk(self.topK)[1]
self.best_words = [self.GPT[idx.item()] for idx in best_indices]
self.best_probabilities = probabilities[best_indices].tolist()
def _getWords(self):
''' returns Top-K Words from GPT-2 '''
return self.best_words
def _getPropability(self):
''' returns top-k Propabilities from GPT-2 '''
return [round(p * 100, 2) for p in self.best_probabilities]
def _process(self, text, guess):
''' scores inputted text and logs it '''
self._run(text)
outputLst = self._output()
self.Log.Trace(("Answer List : {}".format(outputLst)))
score = self.Scorer.score(outputLst, guess)
self.Log.Trace(score)
self.Log.Info("Score of \'{}\': {}".format(score[0], score[1]))
def start(self, text=""):
'''
starts program
text = Text to be inputted
'''
if text == "" and not self.interact:
raise EnvironmentError(
"Please input valid text or use the --interact flag")
if text != "":
encoded = self.Encoder.encode(text=text)
for item in encoded:
if item[0] == '':
continue
self._process(item[0], item[1])
# Code for Manual Input, meant for debugging not for production use
else:
while self.interact:
text = self.Log.Input("Input Text >> ")
if text == "":
self.Log.Info("Please provide a valid input")
continue
if text == "#?":
self.Log.Info(
"Available Commands: \n#?: Shows available commands\n#end: Ends Execution"
)
continue
if text == "#end":
self.Log.Info("Ending Program")
break
guess = self.Log.Input("What will the next word be >> ")
self._process(text, guess)
self._score = self.Scorer.calcScore()
self.Log.Info("Normalized Score: {} | Unnormalized Score: {}".format(
self.getNormScore(), self.getUnNormScore()))
def getNormScore(self):
''' returns the normalized score '''
return self._score[0]
def getUnNormScore(self):
''' returns the unormalized score '''
return self._score[1]
def getSeed(self):
''' returns the seed used '''
return self._seed
def getEncoder(self):
''' returns the encoder object '''
return self.Encoder
def _output(self):
''' returns top-k words and propabilities '''
return [(self._getWords()[i], self._getPropability()[i])
for i in range(self.topK)]