def testCountryFromIPName(self):
(name, code) = Lookup.countryFromIP("101.98.154.154")
self.assertEqual(name, "New Zealand")
self.assertEqual(code, "NZ")
(name, code) = Lookup.countryFromName("duckduckgo.com")
self.assertEqual(name, "Singapore")
self.assertEqual(code, "SG")
def testCountryFromIPName(self):
(name, code) = Lookup.countryFromIP('101.98.154.154')
self.assertEqual(name, 'New Zealand')
self.assertEqual(code, 'NZ')
(name, code) = Lookup.countryFromName('duckduckgo.com')
self.assertEqual(name, "Singapore")
self.assertEqual(code, 'SG')
def run_generator(generator):
#Load offline info for Lookup class
try:
generator.run()
Lookup.writeStatistics()
except:
traceback.print_exc()
raise RuntimeError("thread raised an error")
def run_generator(generator):
#Load offline info for Lookup class
try:
generator.run()
Lookup.writeStatistics()
except:
traceback.print_exc()
raise RuntimeError("thread raised an error")
def add_new_rout_values(key, value):
global rout
look = Lookup(rout)
links = look.get_value(key)
if value not in links:
rout[key].append(value)
return
def RESARE_no_continuation():
return LookupCollection([
Lookup(rules=MSStrCompare.includes('RESTRN', 9, 10, 11, 12, 15, 18, 19,
20, 21, 22),
instruction=SY('INFARE51')),
Lookup(rules=MSNoRules(), instruction=SY('RSRDEF51'))
]) @ LookupCollection([
Lookup(instruction=LS('DASH', 2, 'CHMGD'), table='Plain'),
Lookup(instruction=LC('CTYARE51'), table='Symbolized')
])
def parse_config(self):
"""
Parses configuration file. Assumes configuration is in same directory as this script.
"""
try:
stream = file(self.config_path, 'r')
except IOError:
progress_logger.exception(
"Could not find yaml configuration file.")
raise
config = yaml.load(stream)
expected_config_keys = [
'root_dir', 'regular_filename', 'phish_filename', 'use_percentage',
'sender_profile_start_time', 'sender_profile_end_time',
'train_start_time', 'train_end_time', 'test_start_time',
'test_end_time', 'sender_profile_percentage',
'data_matrix_percentage', 'test_matrix_percentage',
'use_name_in_from', 'model_path_out', 'result_path_out', 'weights',
'detectors', 'emails_threshold', 'batch_threading_size', 'offline',
'results_size', 'parallel', 'num_threads', 'logging_interval',
'memlog_gen_features_frequency', 'memlog_classify_frequency',
'senders', 'recipients'
]
try:
for key in expected_config_keys:
setattr(self, key, config[key])
except KeyError:
progress_logger.exception("Configuration file missing entry")
raise
detectors = []
for detector, val in self.detectors.items():
if val == 1:
detectors.append(getattr(globals()['fc'], detector))
self.detectors = detectors
self.root_dir = os.path.abspath(self.root_dir)
Lookup.initialize(offline=self.offline)
if not self.use_percentage:
self.sender_profile_start_time = calendar.timegm(
time.strptime(self.sender_profile_start_time, "%B %d %Y"))
self.sender_profile_end_time = calendar.timegm(
time.strptime(self.sender_profile_end_time, "%B %d %Y"))
self.train_start_time = calendar.timegm(
time.strptime(self.train_start_time, "%B %d %Y"))
self.train_end_time = calendar.timegm(
time.strptime(self.train_end_time, "%B %d %Y"))
self.test_start_time = calendar.timegm(
time.strptime(self.test_start_time, "%B %d %Y"))
self.test_end_time = calendar.timegm(
time.strptime(self.test_end_time, "%B %d %Y"))
def orgGroups(self, sender, mID):
try:
newmID = mID
afterAT = mID
if "@" in sender:
afterAT = sender[sender.index("@") + 1:]
else:
self.domain2domainPairing[(newmID, afterAT)] = False
return False
# if I have seen this pairing before, do not flag as FP
if (newmID, afterAT) in self.domain2domainPairing:
return True
# check domain org match if I can find both domains in my pairings
if newmID in Lookup.seen_domain_org and afterAT in Lookup.seen_domain_org:
if Lookup.seen_domain_org[newmID] == Lookup.seen_domain_org[
afterAT]:
self.domain2domainPairing[(newmID, afterAT)] = True
return True
else:
self.domain2domainPairing[(newmID, afterAT)] = False
return False
# if the domains are not in the file, then use CIDR blocks
if newmID in self.domainCompanyPairing:
res11 = self.domainCompanyPairing[newmID]
elif newmID in Lookup.seen_domain_ip:
ip1 = Lookup.seen_domain_ip[newmID]
res11 = Lookup.getCIDR(ip1)
self.domainCompanyPairing[newmID] = res11
else:
self.domain2domainPairing[(newmID, afterAT)] = False
return False
if afterAT in self.domainCompanyPairing:
res22 = self.domainCompanyPairing[afterAT]
elif afterAT in Lookup.seen_domain_ip:
ip2 = Lookup.seen_domain_ip[afterAT]
res22 = Lookup.getCIDR(ip2)
self.domainCompanyPairing[afterAT] = res22
else:
self.domain2domainPairing[(newmID, afterAT)] = False
return False
if res11 == res22:
self.domain2domainPairing[(newmID, afterAT)] = True
return True
self.domain2domainPairing[(newmID, afterAT)] = False
return False
except ValueError:
self.domain2domainPairing[(newmID, afterAT)] = False
return False
def RESARE_continuation_D():
includes = MSStrCompare.includes
return LookupCollection([
Lookup(rules=(includes('RESTRN', 9, 10, 11, 12, 15, 18, 19, 20, 21, 22)
| includes('CATREA', 4, 5, 6, 7, 10, 20, 22, 23)),
instruction=SY('CTYARE71')),
Lookup(instruction=SY('CTYARE51'))
]) @ LookupCollection([
Lookup(instruction=LS('DASH', 2, 'CHMGD'), table='Plain'),
Lookup(instruction=LC('CTYARE51'), table='Symbolized')
])
def parse_config(self):
"""
Parses configuration file. Assumes configuration is in same directory as this script.
"""
try:
stream = file(self.config_path, 'r')
except IOError:
progress_logger.exception("Could not find yaml configuration file.")
raise
config = yaml.load(stream)
expected_config_keys = [
'root_dir',
'regular_filename',
'phish_filename',
'sender_profile_percentage',
'data_matrix_percentage',
'test_matrix_percentage',
'use_name_in_from',
'model_path_out',
'result_path_out',
'weights',
'detectors',
'emails_threshold',
'batch_threading_size',
'offline',
'results_size',
'parallel',
'num_threads',
'logging_interval',
'memlog_gen_features_frequency',
'memlog_classify_frequency'
]
try:
for key in expected_config_keys:
setattr(self, key, config[key])
except KeyError:
progress_logger.exception("Configuration file missing entry")
raise
detectors = []
for detector, val in self.detectors.items():
if val == 1:
detectors.append(getattr(globals()['fc'], detector))
self.detectors = detectors
self.root_dir = os.path.abspath(self.root_dir)
Lookup.initialize(offline=self.offline)
def RESARE_continuation_C():
includes = MSStrCompare.includes
return LookupCollection([
Lookup(rules=(includes('RESTRN', 13, 16, 17, 23, 24, 25, 26, 27)
| includes('CATREA', 1, 8, 9, 12, 14, 18, 19, 21, 24, 25,
26)),
instruction=SY('FSHRES61')),
Lookup(rules=(includes('RESTRN', 9, 10, 11, 12, 15, 18, 19, 20, 21, 22)
| includes('CATREA', 4, 5, 6, 7, 10, 20, 22, 23)),
instruction=SY('FSHRES71')),
Lookup(instruction=SY('FSHRES51'))
]) @ LookupCollection([
Lookup(instruction=LS('DASH', 2, 'CHMGD'), table='Plain'),
Lookup(instruction=LC('FSHRES51'), table='Symbolized')
])
def match_against(self, record):
graph = record.graph_for(self.subject)
if graph is None:
return False
item_sparql = self.sparql_query_for_item_of_system()
if item_sparql is None:
raise Exception("I don't know how to extract items for '%s'" % self.system)
# extract the actual items that we want to test against
query = """
PREFIX sp:<http://smartplatforms.org/terms#>
SELECT ?item ?source
WHERE {
?source a %s .
%s
}
""" % (self.subject, item_sparql)
items = graph.query(query)
if len(items) < 1:
return False
# get our lookup system
lookup = Lookup.for_system(self.system)
if lookup is None:
raise Exception("We don't have a lookup system for '%s'" % self.system)
# test the quality of our items (remember "items" is full of tuples from the sparql query)
for item in items:
if lookup.has_relation(item[0], self.predicate, self.object):
record.did_match_item(item[1], self)
return True
return False
class Combination(dict):
distance = Lookup(lambda x, y: 1.0 - sum(
[min(value, y[key]) for key, value in x.items() if key in y]))
def __init__(self, *keyValuePairs):
valueTotal = float(
sum([value * float(key[1]) for key, value in keyValuePairs]))
dict.__init__(self, [(key, float(key[1]) * value / valueTotal)
for key, value in keyValuePairs])
@Lookup
def dist(x1, x2, y1, y2):
x, y = x1 - x2, y1 - y2
return math.sqrt(x * x + y * y)
@staticmethod
def combDist(comb1, comb2):
return Combination.dist(comb1[0], comb2[0], comb1[1], comb2[1])
@staticmethod
def getVar(comb1, comb2, name1, name2, guess):
squareroot = Combination.distance(comb1, comb2) - Combination.combDist(
guess[name1], guess[name2])
return squareroot * squareroot
@staticmethod
def getPosition(targetkey, coords1, coords2, firstkey, secondkey, guess):
if firstkey == targetkey:
return coords1
if secondkey == targetkey:
return coords2
return guess[targetkey]
def orgGroups(self, sender, mID):
try:
newmID = mID
afterAT = mID
afterAT = sender[sender.index("@")+1:]
# if I have seen this pairing before, do not flag as FP
if (newmID, afterAT) in self.domain2domainPairing:
return True
# check domain org match if I can find both domains in my pairings
if newmID in Lookup.seen_domain_org and afterAT in Lookup.seen_domain_org:
if Lookup.seen_domain_org[newmID] == Lookup.seen_domain_org[afterAT]:
self.domain2domainPairing[(newmID, afterAT)] = True
return True
else:
self.domain2domainPairing[(newmID, afterAT)] = False
return False
# if the domains are not in the file, then use CIDR blocks
if newmID in self.domainCompanyPairing:
res11 = self.domainCompanyPairing[newmID]
else:
# createDomain2OrgPair(newmID)
ip1 = Lookup.seen_domain_ip[newmID]
res11 = getBinaryRep(ip1, Lookup.getCIDR(ip1))
self.domainCompanyPairing[newmID] = res11
if afterAT in self.domainCompanyPairing:
res22 = self.domainCompanyPairing[afterAT]
else:
# createDomain2OrgPair[afterAT]
ip2 = Lookup.seen_domain_ip[afterAT]
res22 = getBinaryRep(ip2, Lookup.getCIDR(ip2))
self.domainCompanyPairing[afterAT] = res22
if res11 == res22:
self.domain2domainPairing[(newmID, afterAT)] = True
return True
self.domain2domainPairing[(newmID, afterAT)] = False
return False
except:
self.domain2domainPairing[(newmID, afterAT)] = False
return False
def lookup_id(cls, id):
try:
model = cls.validate_id(id)
except SpotifyIDError:
raise
search = cls.__dict__[model]
return Lookup.by_id(
id, search, self.cache_function, self.cache_arguments)
def lookup():
# co-ordinates and dimesnsions for lookup interface
# ypos always 1 below current window
ypos = int((activewin / 5) + 13)
xpos = int(widths[2] + 1)
lookupwidth = int(widths[3] + widths[4])
lookupheight = 15
# create lookup interface
lookupwin = Swindow(ypos, xpos, lookupheight, lookupwidth, 4)
# return products matching user input
lookup = Lookup()
userinput = wins[activewin].sgetcontents()
if userinput:
lookup.readinput(userinput)
lookupresults = lookup.returnvalues()
# enter lookup results into lookup interface
lookupwin.sadditems(lookupresults)
wins[activewin].smove()
wins[activewin].srefresh()
def __init__(self, host, port, start_mongo=True):
"""
Initializes a resource directory and creates registration, lookup resources.
:param host: the host where the resource directory is.
:param port: the port where the resource directory listens.
"""
CoAP.__init__(self, (host, port))
self.add_resource('rd/', Registration())
self.add_resource('rd-lookup/', Lookup())
self.add_resource('rd-lookup/res', LookupRes())
self.add_resource('rd-lookup/ep', LookupEp())
self.start_mongo = start_mongo
if self.start_mongo:
self.mongodb = Popen(['mongod', '--config', MONGO_CONFIG_FILE, '--auth'])
def RESARE(lookup_type, name):
includes = MSStrCompare.includes
return (Lookup(id='-CS(RESARE)', rules=includes(
'RESTRN', 7, 8, 14)) @ RESARE_continuation_A() +
Lookup(id='-CS(RESARE)', rules=includes(
'RESTRN', 1, 2)) @ RESARE_continuation_B() +
Lookup(id='-CS(RESARE)', rules=includes(
'RESTRN', 3, 4, 5, 6, 24)) @ RESARE_continuation_C() +
Lookup(id='-CS(RESARE)',
rules=includes('RESTRN', 13, 16, 17, 23, 25, 26,
27)) @ RESARE_continuation_D() +
Lookup(id='-CS(RESARE)',
rules=MSHasValue('RESTRN')) @ RESARE_no_continuation() +
Lookup(id='-CS(RESARE)') @ RESARE_continuation_E())
def RESARE_continuation_E():
rule_a = MSStrCompare.includes('CATREA', 1, 8, 9, 12, 14, 18, 19, 21, 24,
25, 26)
rule_b = MSStrCompare.includes('CATREA', 4, 5, 6, 7, 10, 20, 22, 23)
return LookupCollection([
Lookup(rules=rule_a & rule_b, instruction=SY('CTYARE71')),
Lookup(rules=rule_a, instruction=SY('CTYARE51')),
Lookup(rules=MSHasValue('CATREA') & rule_b,
instruction=SY('INFARE51')),
Lookup(instruction=SY('RSRDEF51'))
]) @ LookupCollection([
Lookup(instruction=LS('DASH', 2, 'CHMGD'), table='Plain'),
Lookup(instruction=LC('CTYARE51'), table='Symbolized')
])
def testWhoIs(self):
Lookup.whois('101.98.154.154')
def add_book():
add_option = input('''Let's add a book. How would you like to add it?\n
1) Manually\n
2) Search by ISBN\n
3) Search by Title: ''')
add_option = int(add_option)
if add_option == 1:
manual_add = {}
print("If you have multiple authors or tags, please separate them with a comma.")
print("e.g. author1,author2,author3")
print("To autogenerate a barcode, enter -1 for it.")
print()
for item in terms:
manual_add[item] = input("Please enter the " + item + ": ")
for item in manual_add:
if item in substitutions:
manual_add[substitutions[item]] = manual_add.pop(item)
manual_book = create_book_from_dict(manual_add)
book_db = Bdb(dbLocation)
book_db.store(manual_book)
elif add_option == 2:
isbn = input("Please enter the 10 or 13 digit ISBN: ")
lookup = Lookup()
book = lookup.by_isbn(isbn)
bc = input('''Please enter a unique barcode, or -1 to autogenerate: ''')
bc = int(bc)
book.bc = bc
location = input('''Please enter the location of the book, default blank: ''')
book.location = location
call_num = input('''Please enter the call number of the book: ''')
book.call_num = call_num
tags = input('''Please enter any tags, separated by a comma: ''')
tags = tags.strip()
book.tags = tags
print('''Ok, everything should be set. I'll show you what I've got,
and if it looks good, just press enter, otherwise type something in
and I'll return you to the beginning.''')
book.print_check()
is_ok = input("")
if is_ok != "": # not 100% sure this will work
raise ValueError
else:
book_db = Bdb(dbLocation)
book_db.store(book)
elif add_option == 3:
title = input("Please enter the title you'd like to search for:")
lookup = Lookup()
input("The following are the results. Please enter the number of the " +
"result you'd like. Press any key to display them.")
books = []
for index, book in enumerate(lookup.by_title(title), start=1):
if index == 11:
break # only print first 10 results?? Not very elegant.
print('%i) Title: %s, Author(s): %s' % (
index,
book.title,
str(book.authors).strip('[]')
))
books.append(book)
user_choice = input("Which result would you like? Or hit enter for none.")
if user_choice == '':
return 2
user_choice = int(user_choice)
user_choice -= 1 # need to compensate for off-by-one.
book = books[user_choice]
bc = input('''Please enter a unique barcode, or -1 to autogenerate: ''')
bc = int(bc)
book.bc = bc
location = input('''Please enter the location of the book, default blank: ''')
book.location = location
call_num = input('''Please enter the call number of the book: ''')
book.call_num = call_num
tags = input('''Please enter any tags, separated by a comma: ''')
tags = tags.strip()
book.tags = tags
assert isinstance(book, Book)
print('''Ok, everything should be set. I'll show you what I've got,
and if it looks good, just press enter, otherwise type something in
and I'll return you to the beginning.''')
book.print_check()
is_ok = input("")
if is_ok != "":
raise ValueError # should consider changing to a UserQuit exception.
else:
book_db = Bdb(dbLocation)
book_db.store(book)
def test_lookup(lookup, text="A test."):
print(lookup)
print("Testing with: [{}]".format(text))
id_of_bos = lookup.convert_tokens_to_ids(lookup.bos_token)
id_of_eos = lookup.convert_tokens_to_ids(lookup.eos_token)
id_of_pad = lookup.convert_tokens_to_ids(lookup.pad_token)
converted_bos_token = lookup.convert_ids_to_tokens(id_of_bos)
converted_eos_token = lookup.convert_ids_to_tokens(id_of_eos)
converted_pad_token = lookup.convert_ids_to_tokens(id_of_pad)
print("bos_token {} = {} and converted back to token = {}".format(
lookup.bos_token, id_of_bos, converted_bos_token))
print("eos_token {} = {} and converted back to token = {}".format(
lookup.eos_token, id_of_eos, converted_eos_token))
print("pad_token {} = {} and converted back to token = {}".format(
lookup.pad_token, id_of_pad, converted_pad_token))
#print(lookup._tokenizer.all_special_ids)
#print(lookup._tokenizer.all_special_tokens)
#print(lookup._tokenizer.special_tokens_map)
print("\n0. Save/load lookup object:")
if not os.path.exists(lookup.type):
os.makedirs(lookup.type)
lookup.save_special_tokens(
file_prefix=os.path.join(lookup.type, lookup.type))
lookup = Lookup(type=lookup.type) # recreate object
lookup.load(file_prefix=os.path.join(lookup.type, lookup.type))
print(lookup)
print("\n1. String to tokens (tokenize):")
tokens = lookup.tokenize(text)
print(tokens)
print("\n2. Tokens to ints (convert_tokens_to_ids):")
ids = lookup.convert_tokens_to_ids(tokens)
print(ids)
print("\n2.5 Token to int (convert_tokens_to_ids with a single str):")
id = lookup.convert_tokens_to_ids(tokens[0])
print(id)
print("\n3. Ints to tokens (convert_ids_to_tokens):")
tokens = lookup.convert_ids_to_tokens(ids)
print(tokens)
print("\n3.5 Int to token (convert_ids_to_tokens with a single int):")
token = lookup.convert_ids_to_tokens(id)
print(token)
print("\n4. Tokens to string (convert_tokens_to_string):")
recreated_text = lookup.convert_tokens_to_string(tokens)
print(recreated_text)
print("\n5. String to ints (encode):")
ids = lookup.encode(text)
print(ids)
print("\n6. Ints to string (decode):")
recreated_text = lookup.decode(ids)
print(recreated_text)
print("\n7. Encode adding special tokens:")
ids = lookup.encode(text, add_bos_eos_tokens=True)
print(ids)
print("How it looks like with tokens: {}".format(
lookup.convert_ids_to_tokens(ids)))
print("\n8. Decode skipping special tokens:")
recreated_text = lookup.decode(ids, skip_bos_eos_tokens=True)
print(recreated_text)
print(recreated_text)
print("\n7. Encode adding special tokens:")
ids = lookup.encode(text, add_bos_eos_tokens=True)
print(ids)
print("How it looks like with tokens: {}".format(
lookup.convert_ids_to_tokens(ids)))
print("\n8. Decode skipping special tokens:")
recreated_text = lookup.decode(ids, skip_bos_eos_tokens=True)
print(recreated_text)
if __name__ == "__main__":
# gpt2
lookup = Lookup(type="gpt2")
test_lookup(lookup)
# bpe
print("Create BPE model ...")
lookup = Lookup(type="bpe")
if not os.path.exists(lookup.type):
os.makedirs(lookup.type)
import sentencepiece as spm
spm.SentencePieceTrainer.Train(
'--input=dummy_corpus.txt --model_prefix=bpe/bpe --character_coverage=1.0 --model_type=bpe --num_threads=8 --split_by_whitespace=true --shuffle_input_sentence=true --max_sentence_length=8000 --vocab_size=1024'
)
lookup.load("bpe/bpe")
test_lookup(lookup)
def big_lookup(self, tmpdir):
lookup = Lookup(10, 3, 0.99, 10, 0.000001, str(tmpdir), None)
yield lookup
lookup.table.shutdown()
def lookup(self, tmpdir):
lookup = Lookup(2, 3, 0.99, 4, 0.000001, str(tmpdir), None)
yield lookup
lookup.table.shutdown()
def lookup_movie_specified_regions(movie_name, regions):
look = Lookup()
regions = parse_regions(regions)
utils.check_region(regions)
look.set_movie_query_spec_regions(movie_name[0], regions)
look.check_availability()
def __init__(self,
pad,
board_number=0,
name='',
sad=0,
timeout=13,
send_eoi=1,
eos_mode=0):
'''
Must provide the pad (primary GPIB address) - older instrument so some standard commands do not apply
'''
super(Hp3562a, self).__init__(board_number, name, pad, sad, timeout,
send_eoi, eos_mode)
# GPIB identity string of the instrument
self.id_string = "LSCI,MODEL370,370447,09272005"
self.manufacturer = 'HP'
self.model_number = '3562A'
self.description = 'Dynamic Signal Analyzer'
#self.compare_identity()
self.numpts = 801 # fixed by the instrument
self.freq = numpy.zeros(self.numpts, dtype=numpy.float)
self.realspectrum = numpy.zeros(self.numpts, dtype=numpy.float)
self.complexspectrum = numpy.zeros(self.numpts, dtype=numpy.complex)
self.function_enum = ('No data', 'Frequency Response',
'Power Spectrum 1', 'Power Spectrum 2',
'Coherence', 'Cross Spectrum', 'Input time 1',
'Input time 2', 'Input linear spectrum1',
'Input linear spectrum 2', 'Impulse Response',
'Cross Correlation', 'Auto correlation 1',
'Auto correlation 2', 'Histogram 1',
'Histogram 2')
self.channel_enum = ('Channel 1', 'Channel 2', 'Channels 1 and 2',
'No channel')
self.domain_enum = ('Time', 'Frequency', 'Voltage (amplitude)')
self.peak_enum = ('Peak', 'RMS', 'Volts (indicates peak only)')
self.amplitude_enum = ('Volts', 'Volts^2', 'V^2/Hz', 'V^2s/Hz',
'V/sqrtHz', 'No units', 'Unit volts',
'Units volts^^2')
self.xaxis_enum = ('No Units', 'Hz', 'RPM', 'Orders', 'Seconds',
'Revs', 'Degrees', 'dB', 'dBV', 'Volts', 'V/sqrtHz',
'Hertz/second', 'Volts/EU', 'Vrms', 'V^2/Hz',
'Percent', 'Points', 'Records', 'Ohms', 'Hz/octave',
'Pulse/rev', 'Decades', 'Minutes', 'V^2s/Hz',
'Octave', 'Seconds/Decade', 'Seconds/Octace',
'Hz/point', 'Points/Sweep', 'Points/Decade',
'V/Vrms', 'V^2', 'EU', 'EU', 'EU')
self.measmode_enum = ('Linear Resolution', 'Log Resolution',
'Swept Sine', 'Time Capture',
'Linear Resolution Throughput')
self.sweepmode_enum = ('Linear Sweep', 'Log Sweep')
self.demod_enum = ('AM', 'FM', 'PM')
self.average_enum = ('No data', 'Not averaged', 'Averaged')
self.window_enum = ('N/A', 'Hanning', 'Flat Top', 'Uniform',
'Exponential', 'Force', 'Force1 Exp2',
'Exp1 Force2', 'User')
self.control_mode_switch = Lookup({
'closed': '1',
'zone': '2',
'open': '3',
'off': '4'
})
self.on_off_switch = Lookup({'off': '0', 'on': '1'})
import os
import sys
import random
import json
sys.path.insert(0, "../code_repo")
from lookup import Lookup
source_file = 'resources/corpus_cleansed.txt'
destination_validation = 'resources/validation/'
options_tuple = ('NW', 'RW', 'BO')
validations_set = []
maximum_limit = 100
lookup = Lookup()
def is_init():
for counter in range(1, 4):
for opt in options_tuple:
if (not os.path.exists("{}{}_{}.txt".format(
destination_validation, opt, counter))
and not os.path.exists("{}{}_{}.ready.txt".format(
destination_validation, opt, counter))):
return True
return False
if (not os.path.exists(source_file)):
print("Cleaned corpus does not exist")
sys.exit()
def lookup_movie(movie_name):
look = Lookup()
look.set_movie_query(movie_name[0])
look.check_availability()
def load_lookups(self,
root,
point_style,
area_style,
displaycategory=None):
if displaycategory is None:
class AllInclusive:
def __contains__(self, val):
return True
displaycategory = AllInclusive()
for lookup in root.iter('lookup'):
try:
name = lookup.get('name')
id = lookup.get('id')
lookup_type = lookup.find('type').text
table_name = lookup.find('table-name').text
display = lookup.find('display-cat').text
comment = lookup.find('comment').text
str_instruction = lookup.find('instruction').text or ''
rules = MSAnd(*(MSFilter.from_attrcode(attr.text)
for attr in lookup.findall('attrib-code')))
except (KeyError, AttributeError):
continue
if name in self.excluded_lookups:
continue
if table_name in (point_style, area_style, 'Lines') and \
display in displaycategory:
if lookup_type == 'Point':
lookup_table = self.point_lookups.setdefault(name, [])
elif lookup_type == 'Line':
lookup_table = self.line_lookups.setdefault(name, [])
elif lookup_type == 'Area':
lookup_table = self.polygon_lookups.setdefault(name, [])
else:
lookup_table = []
lookups = Lookup(
id=id,
table=table_name,
display=display,
comment=comment,
instruction=[],
rules=rules,
)
# If we have a CS instruction, explode it in many lookups
parts = str_instruction.split(';')
for part in parts:
if not part:
continue
command = get_command(part)
if isinstance(command, CS):
details = command.proc
lookups @= lookups_from_cs(details, lookup_type, name)
else:
lookups.add_instruction(command)
for lookup in lookups:
lookup_table.append(lookup)
def add_book():
add_option = input('''Let's add a book. How would you like to add it?\n
1) Manually\n
2) Search by ISBN\n
3) Search by Title: ''')
add_option = int(add_option)
if add_option == 1:
manual_add = {}
print(
"If you have multiple authors or tags, please separate them with a comma."
)
print("e.g. author1,author2,author3")
print("To autogenerate a barcode, enter -1 for it.")
print()
for item in terms:
manual_add[item] = input("Please enter the " + item + ": ")
for item in manual_add:
if item in substitutions:
manual_add[substitutions[item]] = manual_add.pop(item)
manual_book = create_book_from_dict(manual_add)
book_db = Bdb(dbLocation)
book_db.store(manual_book)
elif add_option == 2:
isbn = input("Please enter the 10 or 13 digit ISBN: ")
lookup = Lookup()
book = lookup.by_isbn(isbn)
bc = input(
'''Please enter a unique barcode, or -1 to autogenerate: ''')
bc = int(bc)
book.bc = bc
location = input(
'''Please enter the location of the book, default blank: ''')
book.location = location
call_num = input('''Please enter the call number of the book: ''')
book.call_num = call_num
tags = input('''Please enter any tags, separated by a comma: ''')
tags = tags.strip()
book.tags = tags
print('''Ok, everything should be set. I'll show you what I've got,
and if it looks good, just press enter, otherwise type something in
and I'll return you to the beginning.''')
book.print_check()
is_ok = input("")
if is_ok != "": # not 100% sure this will work
raise ValueError
else:
book_db = Bdb(dbLocation)
book_db.store(book)
elif add_option == 3:
title = input("Please enter the title you'd like to search for:")
lookup = Lookup()
input(
"The following are the results. Please enter the number of the " +
"result you'd like. Press any key to display them.")
books = []
for index, book in enumerate(lookup.by_title(title), start=1):
if index == 11:
break # only print first 10 results?? Not very elegant.
print('%i) Title: %s, Author(s): %s' %
(index, book.title, str(book.authors).strip('[]')))
books.append(book)
user_choice = input(
"Which result would you like? Or hit enter for none.")
if user_choice == '':
return 2
user_choice = int(user_choice)
user_choice -= 1 # need to compensate for off-by-one.
book = books[user_choice]
bc = input(
'''Please enter a unique barcode, or -1 to autogenerate: ''')
bc = int(bc)
book.bc = bc
location = input(
'''Please enter the location of the book, default blank: ''')
book.location = location
call_num = input('''Please enter the call number of the book: ''')
book.call_num = call_num
tags = input('''Please enter any tags, separated by a comma: ''')
tags = tags.strip()
book.tags = tags
assert isinstance(book, Book)
print('''Ok, everything should be set. I'll show you what I've got,
and if it looks good, just press enter, otherwise type something in
and I'll return you to the beginning.''')
book.print_check()
is_ok = input("")
if is_ok != "":
raise ValueError # should consider changing to a UserQuit exception.
else:
book_db = Bdb(dbLocation)
book_db.store(book)
def testWhoIs(self):
Lookup.whois("101.98.154.154")
def spellcheck():
handler = Lookup()
corrections = []
words_processed = 0
text = request.form['text']
start_time = 0
end_time = 0
if (type(text) is str and len(text.strip()) > 0):
start_time = time.time()
fw = open('log.txt', 'a+', encoding='utf-8')
fw.write("Proccessing Strated: {}\n".format(start_time))
lines = handler.load_raw_text(passage=text.strip())
fw.write("Lines fetched: {}\n".format(lines))
for line_of_words in lines:
prev_word = (None, None)
prev_status = None
words_processed += len(line_of_words)
for word in line_of_words:
word_result = handler.validate_word(word, prev_word)
fw.write("Validation Result: {}\n".format(word_result))
if (word_result['status'] != handler.CODE_CORRECT):
#If previous word is non-word ignore real-word error
if (prev_status == handler.CODE_NON_WORD_ERR
and word_result['status'] in [
handler.CODE_REAL_WORD_ERR,
handler.CODE_RWE_IGNORED
]):
fw.write(
"Included Status: Ignored (previous non-word)\n")
pass
#If previous word has real-word issue and also the current word has the same issue
#ignore it to prevent chain errors
elif (prev_status in [
handler.CODE_REAL_WORD_ERR,
handler.CODE_RWE_IGNORED
] and word_result['status'] in [
handler.CODE_REAL_WORD_ERR,
handler.CODE_RWE_IGNORED
]):
fw.write(
"Included Status: Ignored (previous real-word/rwe)\n"
)
pass
else:
fw.write("Included Status: Included\n")
corrections.append(word_result)
prev_word = word
prev_status = word_result['status']
end_time = time.time()
fw.write("Proccessing Ended at {} in {} seconds\n".format(
end_time, (end_time - start_time)))
fw.write("----------------------------------------\n\n")
fw.close()
output = {
'processed_words': words_processed,
'duration': int(end_time) - int(start_time),
'corrections': corrections
}
return jsonify(output)
def lookup_cidr_from_hop(hop):
ip = Lookup.public_domain(hop)
if not ip:
ip = Lookup.public_IP(hop)
return Lookup.getCIDR(ip) if ip else None
def lookup_cidr_from_hop(hop):
ip = Lookup.public_domain(hop)
if not ip:
ip = Lookup.public_IP(hop)
return Lookup.getCIDR(ip) if ip else None
