def RESARE_no_continuation():
return LookupCollection([
Lookup(rules=MSStrCompare.includes('RESTRN', 9, 10, 11, 12, 15, 18, 19,
20, 21, 22),
instruction=SY('INFARE51')),
Lookup(rules=MSNoRules(), instruction=SY('RSRDEF51'))
]) @ LookupCollection([
Lookup(instruction=LS('DASH', 2, 'CHMGD'), table='Plain'),
Lookup(instruction=LC('CTYARE51'), table='Symbolized')
])
def RESARE_continuation_D():
includes = MSStrCompare.includes
return LookupCollection([
Lookup(rules=(includes('RESTRN', 9, 10, 11, 12, 15, 18, 19, 20, 21, 22)
| includes('CATREA', 4, 5, 6, 7, 10, 20, 22, 23)),
instruction=SY('CTYARE71')),
Lookup(instruction=SY('CTYARE51'))
]) @ LookupCollection([
Lookup(instruction=LS('DASH', 2, 'CHMGD'), table='Plain'),
Lookup(instruction=LC('CTYARE51'), table='Symbolized')
])
def RESARE_continuation_C():
includes = MSStrCompare.includes
return LookupCollection([
Lookup(rules=(includes('RESTRN', 13, 16, 17, 23, 24, 25, 26, 27)
| includes('CATREA', 1, 8, 9, 12, 14, 18, 19, 21, 24, 25,
26)),
instruction=SY('FSHRES61')),
Lookup(rules=(includes('RESTRN', 9, 10, 11, 12, 15, 18, 19, 20, 21, 22)
| includes('CATREA', 4, 5, 6, 7, 10, 20, 22, 23)),
instruction=SY('FSHRES71')),
Lookup(instruction=SY('FSHRES51'))
]) @ LookupCollection([
Lookup(instruction=LS('DASH', 2, 'CHMGD'), table='Plain'),
Lookup(instruction=LC('FSHRES51'), table='Symbolized')
])
class Combination(dict):
distance = Lookup(lambda x, y: 1.0 - sum(
[min(value, y[key]) for key, value in x.items() if key in y]))
def __init__(self, *keyValuePairs):
valueTotal = float(
sum([value * float(key[1]) for key, value in keyValuePairs]))
dict.__init__(self, [(key, float(key[1]) * value / valueTotal)
for key, value in keyValuePairs])
@Lookup
def dist(x1, x2, y1, y2):
x, y = x1 - x2, y1 - y2
return math.sqrt(x * x + y * y)
@staticmethod
def combDist(comb1, comb2):
return Combination.dist(comb1[0], comb2[0], comb1[1], comb2[1])
@staticmethod
def getVar(comb1, comb2, name1, name2, guess):
squareroot = Combination.distance(comb1, comb2) - Combination.combDist(
guess[name1], guess[name2])
return squareroot * squareroot
@staticmethod
def getPosition(targetkey, coords1, coords2, firstkey, secondkey, guess):
if firstkey == targetkey:
return coords1
if secondkey == targetkey:
return coords2
return guess[targetkey]
def __init__(self, host, port, start_mongo=True):
"""
Initializes a resource directory and creates registration, lookup resources.
:param host: the host where the resource directory is.
:param port: the port where the resource directory listens.
"""
CoAP.__init__(self, (host, port))
self.add_resource('rd/', Registration())
self.add_resource('rd-lookup/', Lookup())
self.add_resource('rd-lookup/res', LookupRes())
self.add_resource('rd-lookup/ep', LookupEp())
self.start_mongo = start_mongo
if self.start_mongo:
self.mongodb = Popen(['mongod', '--config', MONGO_CONFIG_FILE, '--auth'])
def RESARE(lookup_type, name):
includes = MSStrCompare.includes
return (Lookup(id='-CS(RESARE)', rules=includes(
'RESTRN', 7, 8, 14)) @ RESARE_continuation_A() +
Lookup(id='-CS(RESARE)', rules=includes(
'RESTRN', 1, 2)) @ RESARE_continuation_B() +
Lookup(id='-CS(RESARE)', rules=includes(
'RESTRN', 3, 4, 5, 6, 24)) @ RESARE_continuation_C() +
Lookup(id='-CS(RESARE)',
rules=includes('RESTRN', 13, 16, 17, 23, 25, 26,
27)) @ RESARE_continuation_D() +
Lookup(id='-CS(RESARE)',
rules=MSHasValue('RESTRN')) @ RESARE_no_continuation() +
Lookup(id='-CS(RESARE)') @ RESARE_continuation_E())
def RESARE_continuation_E():
rule_a = MSStrCompare.includes('CATREA', 1, 8, 9, 12, 14, 18, 19, 21, 24,
25, 26)
rule_b = MSStrCompare.includes('CATREA', 4, 5, 6, 7, 10, 20, 22, 23)
return LookupCollection([
Lookup(rules=rule_a & rule_b, instruction=SY('CTYARE71')),
Lookup(rules=rule_a, instruction=SY('CTYARE51')),
Lookup(rules=MSHasValue('CATREA') & rule_b,
instruction=SY('INFARE51')),
Lookup(instruction=SY('RSRDEF51'))
]) @ LookupCollection([
Lookup(instruction=LS('DASH', 2, 'CHMGD'), table='Plain'),
Lookup(instruction=LC('CTYARE51'), table='Symbolized')
])
def lookup():
# co-ordinates and dimesnsions for lookup interface
# ypos always 1 below current window
ypos = int((activewin / 5) + 13)
xpos = int(widths[2] + 1)
lookupwidth = int(widths[3] + widths[4])
lookupheight = 15
# create lookup interface
lookupwin = Swindow(ypos, xpos, lookupheight, lookupwidth, 4)
# return products matching user input
lookup = Lookup()
userinput = wins[activewin].sgetcontents()
if userinput:
lookup.readinput(userinput)
lookupresults = lookup.returnvalues()
# enter lookup results into lookup interface
lookupwin.sadditems(lookupresults)
wins[activewin].smove()
wins[activewin].srefresh()
def test_lookup(lookup, text="A test."):
print(lookup)
print("Testing with: [{}]".format(text))
id_of_bos = lookup.convert_tokens_to_ids(lookup.bos_token)
id_of_eos = lookup.convert_tokens_to_ids(lookup.eos_token)
id_of_pad = lookup.convert_tokens_to_ids(lookup.pad_token)
converted_bos_token = lookup.convert_ids_to_tokens(id_of_bos)
converted_eos_token = lookup.convert_ids_to_tokens(id_of_eos)
converted_pad_token = lookup.convert_ids_to_tokens(id_of_pad)
print("bos_token {} = {} and converted back to token = {}".format(
lookup.bos_token, id_of_bos, converted_bos_token))
print("eos_token {} = {} and converted back to token = {}".format(
lookup.eos_token, id_of_eos, converted_eos_token))
print("pad_token {} = {} and converted back to token = {}".format(
lookup.pad_token, id_of_pad, converted_pad_token))
#print(lookup._tokenizer.all_special_ids)
#print(lookup._tokenizer.all_special_tokens)
#print(lookup._tokenizer.special_tokens_map)
print("\n0. Save/load lookup object:")
if not os.path.exists(lookup.type):
os.makedirs(lookup.type)
lookup.save_special_tokens(
file_prefix=os.path.join(lookup.type, lookup.type))
lookup = Lookup(type=lookup.type) # recreate object
lookup.load(file_prefix=os.path.join(lookup.type, lookup.type))
print(lookup)
print("\n1. String to tokens (tokenize):")
tokens = lookup.tokenize(text)
print(tokens)
print("\n2. Tokens to ints (convert_tokens_to_ids):")
ids = lookup.convert_tokens_to_ids(tokens)
print(ids)
print("\n2.5 Token to int (convert_tokens_to_ids with a single str):")
id = lookup.convert_tokens_to_ids(tokens[0])
print(id)
print("\n3. Ints to tokens (convert_ids_to_tokens):")
tokens = lookup.convert_ids_to_tokens(ids)
print(tokens)
print("\n3.5 Int to token (convert_ids_to_tokens with a single int):")
token = lookup.convert_ids_to_tokens(id)
print(token)
print("\n4. Tokens to string (convert_tokens_to_string):")
recreated_text = lookup.convert_tokens_to_string(tokens)
print(recreated_text)
print("\n5. String to ints (encode):")
ids = lookup.encode(text)
print(ids)
print("\n6. Ints to string (decode):")
recreated_text = lookup.decode(ids)
print(recreated_text)
print("\n7. Encode adding special tokens:")
ids = lookup.encode(text, add_bos_eos_tokens=True)
print(ids)
print("How it looks like with tokens: {}".format(
lookup.convert_ids_to_tokens(ids)))
print("\n8. Decode skipping special tokens:")
recreated_text = lookup.decode(ids, skip_bos_eos_tokens=True)
print(recreated_text)
print(recreated_text)
print("\n7. Encode adding special tokens:")
ids = lookup.encode(text, add_bos_eos_tokens=True)
print(ids)
print("How it looks like with tokens: {}".format(
lookup.convert_ids_to_tokens(ids)))
print("\n8. Decode skipping special tokens:")
recreated_text = lookup.decode(ids, skip_bos_eos_tokens=True)
print(recreated_text)
if __name__ == "__main__":
# gpt2
lookup = Lookup(type="gpt2")
test_lookup(lookup)
# bpe
print("Create BPE model ...")
lookup = Lookup(type="bpe")
if not os.path.exists(lookup.type):
os.makedirs(lookup.type)
import sentencepiece as spm
spm.SentencePieceTrainer.Train(
'--input=dummy_corpus.txt --model_prefix=bpe/bpe --character_coverage=1.0 --model_type=bpe --num_threads=8 --split_by_whitespace=true --shuffle_input_sentence=true --max_sentence_length=8000 --vocab_size=1024'
)
lookup.load("bpe/bpe")
test_lookup(lookup)
def big_lookup(self, tmpdir):
lookup = Lookup(10, 3, 0.99, 10, 0.000001, str(tmpdir), None)
yield lookup
lookup.table.shutdown()
def lookup(self, tmpdir):
lookup = Lookup(2, 3, 0.99, 4, 0.000001, str(tmpdir), None)
yield lookup
lookup.table.shutdown()
def spellcheck():
handler = Lookup()
corrections = []
words_processed = 0
text = request.form['text']
start_time = 0
end_time = 0
if (type(text) is str and len(text.strip()) > 0):
start_time = time.time()
fw = open('log.txt', 'a+', encoding='utf-8')
fw.write("Proccessing Strated: {}\n".format(start_time))
lines = handler.load_raw_text(passage=text.strip())
fw.write("Lines fetched: {}\n".format(lines))
for line_of_words in lines:
prev_word = (None, None)
prev_status = None
words_processed += len(line_of_words)
for word in line_of_words:
word_result = handler.validate_word(word, prev_word)
fw.write("Validation Result: {}\n".format(word_result))
if (word_result['status'] != handler.CODE_CORRECT):
#If previous word is non-word ignore real-word error
if (prev_status == handler.CODE_NON_WORD_ERR
and word_result['status'] in [
handler.CODE_REAL_WORD_ERR,
handler.CODE_RWE_IGNORED
]):
fw.write(
"Included Status: Ignored (previous non-word)\n")
pass
#If previous word has real-word issue and also the current word has the same issue
#ignore it to prevent chain errors
elif (prev_status in [
handler.CODE_REAL_WORD_ERR,
handler.CODE_RWE_IGNORED
] and word_result['status'] in [
handler.CODE_REAL_WORD_ERR,
handler.CODE_RWE_IGNORED
]):
fw.write(
"Included Status: Ignored (previous real-word/rwe)\n"
)
pass
else:
fw.write("Included Status: Included\n")
corrections.append(word_result)
prev_word = word
prev_status = word_result['status']
end_time = time.time()
fw.write("Proccessing Ended at {} in {} seconds\n".format(
end_time, (end_time - start_time)))
fw.write("----------------------------------------\n\n")
fw.close()
output = {
'processed_words': words_processed,
'duration': int(end_time) - int(start_time),
'corrections': corrections
}
return jsonify(output)
import os
import sys
import random
import json
sys.path.insert(0, "../code_repo")
from lookup import Lookup
source_file = 'resources/corpus_cleansed.txt'
destination_validation = 'resources/validation/'
options_tuple = ('NW', 'RW', 'BO')
validations_set = []
maximum_limit = 100
lookup = Lookup()
def is_init():
for counter in range(1, 4):
for opt in options_tuple:
if (not os.path.exists("{}{}_{}.txt".format(
destination_validation, opt, counter))
and not os.path.exists("{}{}_{}.ready.txt".format(
destination_validation, opt, counter))):
return True
return False
if (not os.path.exists(source_file)):
print("Cleaned corpus does not exist")
sys.exit()
def load_lookups(self,
root,
point_style,
area_style,
displaycategory=None):
if displaycategory is None:
class AllInclusive:
def __contains__(self, val):
return True
displaycategory = AllInclusive()
for lookup in root.iter('lookup'):
try:
name = lookup.get('name')
id = lookup.get('id')
lookup_type = lookup.find('type').text
table_name = lookup.find('table-name').text
display = lookup.find('display-cat').text
comment = lookup.find('comment').text
str_instruction = lookup.find('instruction').text or ''
rules = MSAnd(*(MSFilter.from_attrcode(attr.text)
for attr in lookup.findall('attrib-code')))
except (KeyError, AttributeError):
continue
if name in self.excluded_lookups:
continue
if table_name in (point_style, area_style, 'Lines') and \
display in displaycategory:
if lookup_type == 'Point':
lookup_table = self.point_lookups.setdefault(name, [])
elif lookup_type == 'Line':
lookup_table = self.line_lookups.setdefault(name, [])
elif lookup_type == 'Area':
lookup_table = self.polygon_lookups.setdefault(name, [])
else:
lookup_table = []
lookups = Lookup(
id=id,
table=table_name,
display=display,
comment=comment,
instruction=[],
rules=rules,
)
# If we have a CS instruction, explode it in many lookups
parts = str_instruction.split(';')
for part in parts:
if not part:
continue
command = get_command(part)
if isinstance(command, CS):
details = command.proc
lookups @= lookups_from_cs(details, lookup_type, name)
else:
lookups.add_instruction(command)
for lookup in lookups:
lookup_table.append(lookup)
def add_book():
add_option = input('''Let's add a book. How would you like to add it?\n
1) Manually\n
2) Search by ISBN\n
3) Search by Title: ''')
add_option = int(add_option)
if add_option == 1:
manual_add = {}
print(
"If you have multiple authors or tags, please separate them with a comma."
)
print("e.g. author1,author2,author3")
print("To autogenerate a barcode, enter -1 for it.")
print()
for item in terms:
manual_add[item] = input("Please enter the " + item + ": ")
for item in manual_add:
if item in substitutions:
manual_add[substitutions[item]] = manual_add.pop(item)
manual_book = create_book_from_dict(manual_add)
book_db = Bdb(dbLocation)
book_db.store(manual_book)
elif add_option == 2:
isbn = input("Please enter the 10 or 13 digit ISBN: ")
lookup = Lookup()
book = lookup.by_isbn(isbn)
bc = input(
'''Please enter a unique barcode, or -1 to autogenerate: ''')
bc = int(bc)
book.bc = bc
location = input(
'''Please enter the location of the book, default blank: ''')
book.location = location
call_num = input('''Please enter the call number of the book: ''')
book.call_num = call_num
tags = input('''Please enter any tags, separated by a comma: ''')
tags = tags.strip()
book.tags = tags
print('''Ok, everything should be set. I'll show you what I've got,
and if it looks good, just press enter, otherwise type something in
and I'll return you to the beginning.''')
book.print_check()
is_ok = input("")
if is_ok != "": # not 100% sure this will work
raise ValueError
else:
book_db = Bdb(dbLocation)
book_db.store(book)
elif add_option == 3:
title = input("Please enter the title you'd like to search for:")
lookup = Lookup()
input(
"The following are the results. Please enter the number of the " +
"result you'd like. Press any key to display them.")
books = []
for index, book in enumerate(lookup.by_title(title), start=1):
if index == 11:
break # only print first 10 results?? Not very elegant.
print('%i) Title: %s, Author(s): %s' %
(index, book.title, str(book.authors).strip('[]')))
books.append(book)
user_choice = input(
"Which result would you like? Or hit enter for none.")
if user_choice == '':
return 2
user_choice = int(user_choice)
user_choice -= 1 # need to compensate for off-by-one.
book = books[user_choice]
bc = input(
'''Please enter a unique barcode, or -1 to autogenerate: ''')
bc = int(bc)
book.bc = bc
location = input(
'''Please enter the location of the book, default blank: ''')
book.location = location
call_num = input('''Please enter the call number of the book: ''')
book.call_num = call_num
tags = input('''Please enter any tags, separated by a comma: ''')
tags = tags.strip()
book.tags = tags
assert isinstance(book, Book)
print('''Ok, everything should be set. I'll show you what I've got,
and if it looks good, just press enter, otherwise type something in
and I'll return you to the beginning.''')
book.print_check()
is_ok = input("")
if is_ok != "":
raise ValueError # should consider changing to a UserQuit exception.
else:
book_db = Bdb(dbLocation)
book_db.store(book)
def lookup_movie(movie_name):
look = Lookup()
look.set_movie_query(movie_name[0])
look.check_availability()
def __init__(self,
pad,
board_number=0,
name='',
sad=0,
timeout=13,
send_eoi=1,
eos_mode=0):
'''
Must provide the pad (primary GPIB address) - older instrument so some standard commands do not apply
'''
super(Hp3562a, self).__init__(board_number, name, pad, sad, timeout,
send_eoi, eos_mode)
# GPIB identity string of the instrument
self.id_string = "LSCI,MODEL370,370447,09272005"
self.manufacturer = 'HP'
self.model_number = '3562A'
self.description = 'Dynamic Signal Analyzer'
#self.compare_identity()
self.numpts = 801 # fixed by the instrument
self.freq = numpy.zeros(self.numpts, dtype=numpy.float)
self.realspectrum = numpy.zeros(self.numpts, dtype=numpy.float)
self.complexspectrum = numpy.zeros(self.numpts, dtype=numpy.complex)
self.function_enum = ('No data', 'Frequency Response',
'Power Spectrum 1', 'Power Spectrum 2',
'Coherence', 'Cross Spectrum', 'Input time 1',
'Input time 2', 'Input linear spectrum1',
'Input linear spectrum 2', 'Impulse Response',
'Cross Correlation', 'Auto correlation 1',
'Auto correlation 2', 'Histogram 1',
'Histogram 2')
self.channel_enum = ('Channel 1', 'Channel 2', 'Channels 1 and 2',
'No channel')
self.domain_enum = ('Time', 'Frequency', 'Voltage (amplitude)')
self.peak_enum = ('Peak', 'RMS', 'Volts (indicates peak only)')
self.amplitude_enum = ('Volts', 'Volts^2', 'V^2/Hz', 'V^2s/Hz',
'V/sqrtHz', 'No units', 'Unit volts',
'Units volts^^2')
self.xaxis_enum = ('No Units', 'Hz', 'RPM', 'Orders', 'Seconds',
'Revs', 'Degrees', 'dB', 'dBV', 'Volts', 'V/sqrtHz',
'Hertz/second', 'Volts/EU', 'Vrms', 'V^2/Hz',
'Percent', 'Points', 'Records', 'Ohms', 'Hz/octave',
'Pulse/rev', 'Decades', 'Minutes', 'V^2s/Hz',
'Octave', 'Seconds/Decade', 'Seconds/Octace',
'Hz/point', 'Points/Sweep', 'Points/Decade',
'V/Vrms', 'V^2', 'EU', 'EU', 'EU')
self.measmode_enum = ('Linear Resolution', 'Log Resolution',
'Swept Sine', 'Time Capture',
'Linear Resolution Throughput')
self.sweepmode_enum = ('Linear Sweep', 'Log Sweep')
self.demod_enum = ('AM', 'FM', 'PM')
self.average_enum = ('No data', 'Not averaged', 'Averaged')
self.window_enum = ('N/A', 'Hanning', 'Flat Top', 'Uniform',
'Exponential', 'Force', 'Force1 Exp2',
'Exp1 Force2', 'User')
self.control_mode_switch = Lookup({
'closed': '1',
'zone': '2',
'open': '3',
'off': '4'
})
self.on_off_switch = Lookup({'off': '0', 'on': '1'})
def lookup_movie_specified_regions(movie_name, regions):
look = Lookup()
regions = parse_regions(regions)
utils.check_region(regions)
look.set_movie_query_spec_regions(movie_name[0], regions)
look.check_availability()
