def __init__(self, text, username, password, location):
self.ssh = paramiko.SSHClient()
self.db = Database(dbname='yago',
username='******',
password='******',
host='127.0.0.1')
self.username = username
self.password = password
self.location = location
self.hostname = 'gw.info.unicaen.fr'
self.text = text
self.entities = []
self.top_types = {
'person': {
'pattern': 'wordnet_person_',
'entities': {}
},
'organization': {
'pattern': 'wordnet_organization_',
'entities': {}
},
'event': {
'pattern': 'wordnet_event_',
'entities': {}
},
'artifact': {
'pattern': 'wordnet_artifact_',
'entities': {}
},
'yagogeoentity': {
'pattern': 'yagoGeoEntity',
'entities': {}
}
}
self.wikipedia = [] # contains {'word': str, 'wikipedia_id': str}
def runTestPlugin(plugin, domain, email=None):
"""
Function for testing plugins. A fake testing environment is setup to allow plugins to run
without the testrunner.
@param plugin: Plugin instance to test
@param domain: Domain invloved in the test
@param email: Email starting the test
@return: Plugin result
"""
import sys
from os.path import dirname
from engine.database import Database
from engine.plugins.Plugin import PluginConfig, PluginResult, Result, LiveFeedback, PluginSockets
d = Database(configFile=dirname(sys.path[0]) + "/../config/config.ini")
d.connect()
pc = PluginConfig(d)
pr = PluginResult(d)
lf = LiveFeedback(None)
ps = PluginSockets()
if email != None:
plugin.setInput('email', email)
plugin.setInput('domain', domain)
plugin.setInput('testId', 1337)
plugin.pluginConfig = pc
plugin.pluginResult = pr
plugin.livefeedback = lf
plugin.pluginSockets = ps
plugin.setResult( Result(plugin) )
plugin.run()
return plugin.getResult()
def __init__(self, logger, verbosity):
self.running = False
self.verbosity = verbosity
self.logger = logger
self.config = Config(logger=self.logger)
self.logger.info('Starting engine...')
try:
self.db = Database()
self.db.connect()
except DatabaseConnectionFailedException, e:
self.logger.error('Could not establish connection with the database (%s).' \
'Shutting down...' % e)
self.shutdown(1)
return
class TestEngine(object):
"""Handles running and saving of tests"""
def __init__(self, logger, verbosity):
self.running = False
self.verbosity = verbosity
self.logger = logger
self.config = Config(logger=self.logger)
self.logger.info('Starting engine...')
try:
self.db = Database()
self.db.connect()
except DatabaseConnectionFailedException, e:
self.logger.error('Could not establish connection with the database (%s).' \
'Shutting down...' % e)
self.shutdown(1)
return
self.debug = self.config.getboolean('general', 'debug')
self.livefeedback = LiveFeedback(self.db)
self.runningTests = []
# BSD 3-Clause License -> see /LICENSE
# Copyright (c) 2017-2020 by Ben de Waal, All rights reserved.
#
import datetime, re
from engine.database import Database
def clean(s):
return re.sub(' +', ' ', s.strip())
db = Database("numbers.sqlite")
with db as conn:
print("HISTOGRAM")
for row in conn.execute("SELECT number,name,COUNT(number) FROM history GROUP BY number ORDER BY COUNT(number) DESC"):
print("%s %s %s" % (row[0],row[1],row[2]))
print()
print("HISTORY")
for row in conn.execute("SELECT epoch,number,name FROM history"):
print("%s %s %s" % (datetime.datetime.fromtimestamp(int(row[0])).strftime('%Y-%m-%d %H:%M:%S'),row[1],row[2]))
class Engine:
def __init__(self, text, username, password, location):
self.ssh = paramiko.SSHClient()
self.db = Database(dbname='yago',
username='******',
password='******',
host='127.0.0.1')
self.username = username
self.password = password
self.location = location
self.hostname = 'gw.info.unicaen.fr'
self.text = text
self.entities = []
self.top_types = {
'person': {
'pattern': 'wordnet_person_',
'entities': {}
},
'organization': {
'pattern': 'wordnet_organization_',
'entities': {}
},
'event': {
'pattern': 'wordnet_event_',
'entities': {}
},
'artifact': {
'pattern': 'wordnet_artifact_',
'entities': {}
},
'yagogeoentity': {
'pattern': 'yagoGeoEntity',
'entities': {}
}
}
self.wikipedia = [] # contains {'word': str, 'wikipedia_id': str}
# Run engine
def run(self):
print('-- EXECUTING AIDA (SSH SERVER)')
aida_output = self.execute_AIDA()
if aida_output is not None:
print('-- CLEANING AIDA OUTPUT')
self.clean_output_AIDA(aida_output)
print('-- FINDING ENTITIES IN YAGO DATABASE')
self.execute_YAGO()
print('-- CLEANING DATA TO USE PURE FRAMEWORK')
self.prepare_top_type_PURE()
self.save_entities_types()
print('-- EXECUTING PURE')
self.execute_PURE()
print('-- GET PURE OUTPUT')
pure = self.read_pure_files()
print('-- CREATE WIKIPEDIA ITEMS')
text = self.process_text_wikipedia()
print('-- END')
return {'text': text, 'pure': pure, 'yago': self.entities}
return None
# Execute AIDA on distant server
def execute_AIDA(self):
command = f'cd {self.location} && java -cp ".:./bin:./lib/*" mpi.aidalight.rmi.AIDALight_client "{self.text}"'
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(hostname=self.hostname,
username=self.username,
password=self.password)
stdin, stdout, stderr = self.ssh.exec_command(command)
output = stdout.readlines()
stdin.flush()
self.ssh.close()
return output[1:] if len(output) > 1 else None
# Clean AIDA output
def clean_output_AIDA(self, output_aida):
for index in range(0, len(output_aida)):
output_split = output_aida[index].split(
'\t') # split the output (\t)
word = output_split[0] # get the name of entity
entity = output_split[1].split('/') # split the wikipedia URL
entity = entity[len(entity) - 1][:-1]
self.wikipedia.append({
'word': word,
'wikipedia_id': output_split[1][:-1]
})
entity = f'<{entity}>' # get the end of URL
if entity != '<--NME-->':
if not any(ent['word'] == word for ent in
self.entities): # check there isn't same word
self.entities.append({'word': word, 'entity': entity})
# Get YAGO from database
def execute_YAGO(self):
for index in range(0, len(self.entities)):
temporary_result = self.db.execute(self.entities[index]['entity'])
self.entities[index]['yago'] = []
for i in range(0, len(temporary_result)):
if re.match('<wordnet_', temporary_result[i][0]) or re.match(
'<yagoGeoEntity>', temporary_result[i][0]):
self.entities[index]['yago'].append(temporary_result[i][0])
# Prepare top type for PURE framework
def prepare_top_type_PURE(self):
for index in range(0, len(self.entities)):
top_type = None
self.entities[index][
'top-type'] = '' # add to entities the top type (maybe not utils)
for yago_type in self.entities[index]['yago']:
if top_type is None:
top_type = self.find_top_type(yago_type) # find top type
if top_type is not None and top_type in self.top_types:
# add to top_types the entity
self.top_types[top_type]['entities'][self.entities[index][
'word']] = self.entities[index]['yago']
self.entities[index]['top-type'] = top_type
# Save entities in files
def save_entities_types(self):
for top_type in self.top_types:
with open(f'tmp/{top_type}.json', 'w+') as file:
json.dump(self.top_types[top_type]['entities'], file)
# Execute PURE framework
def execute_PURE(self):
for top_type in self.top_types:
command = f'python3 -W ignore PURE/run.py {top_type} ../tmp/{top_type}.json 100 > tmp/pure_{top_type}.txt'
os.system(command)
# Read PURE files
def read_pure_files(self):
output = []
for index, top_type in enumerate(self.top_types):
file = open(f'tmp/pure_{top_type}.txt', 'r')
lines = file.readlines()
output.append({'top': top_type, 'content': {}})
for line in lines[1:]:
temp = line.split(
'>'
) # split line type : <wordnet_XXXX_YYYY>ZZ.ZZ where ZZ.ZZ is PURE score
wordnet = temp[0] + '>'
score = float(temp[1])
if score > 0:
for entity in self.top_types[top_type]['entities']:
if any(wordnet in entity_type for entity_type in
self.top_types[top_type]['entities'][entity]):
new_entry = {
'word': entity,
'score': round(score, 2)
}
if len(output[index]['content']) == 0:
# init the dict
output[index]['content'] = {
wordnet: [new_entry]
}
elif wordnet in output[index]['content'].keys():
# add new entity of wordnet (without occurrences)
if new_entry not in output[index]['content'][
wordnet]:
output[index]['content'][wordnet].append(
new_entry)
else:
# create new key in dict
output[index]['content'][wordnet] = [new_entry]
return output
# Process text with wikipedia link
def process_text_wikipedia(self):
text_clean = []
text_split = self.text.split(' ') # split text with space
for word in text_split:
if any(wiki['word'] == word for wiki in self.wikipedia):
if self.get_wikipedia_link(
word) is not None: # if the link is right
text_clean.append({
'mark': True,
'word': word,
'link': self.get_wikipedia_link(word)
})
else:
text_clean.append({'mark': False, 'word': word})
else:
text_clean.append({'mark': False, 'word': word})
return text_clean
# Get wikipedia link
def get_wikipedia_link(self, word):
wikipedia_img = ''
# get the good wikipedia ID
for item in self.wikipedia:
if item['word'] == word:
if item['wikipedia_id'] != 'http://en.wikipedia.org/wiki/--NME--':
wikipedia_img = item['wikipedia_id']
else:
wikipedia_img = None
break
return wikipedia_img
# Get top type of entity
def find_top_type(self, entity_type):
for top_type in self.top_types:
if self.top_types[top_type]['pattern'] in entity_type:
return top_type
return None
# BSD 3-Clause License -> see /LICENSE # Copyright (c) 2017-2020 by Ben de Waal, All rights reserved. # import sys, os from config import FAX_TIMEOUT, REDIAL_WINDOW from engine.modem import RealModem from engine.database import Database from engine.sm import StateMachine # # set up environment # path = os.path.join(sys.argv[1], "numbers.sqlite") port = sys.argv[2] # database & migrations db = Database(path) db.setup() # we use a fake modem modem = RealModem(port) # build state machine sm = StateMachine(db, modem, FAX_TIMEOUT, REDIAL_WINDOW) # run until completion sm.run()
# non-immediate redial test
{ "at": 4.0, "from":"1234567891", "expect":"pulse" },
{ "at": 6.0, "from":"1234567891", "expect":"pulse" },
# auto blacklist promotion
{"at": 7.0, "from": "1234567892", "expect": "pulse"},
{"at": 8.5, "from": "1234567892", "expect": "pulse"},
{"at": 10.0, "from": "1234567892", "expect": "pulse"},
{"at": 11.5, "from": "1234567892", "expect": "pulse"},
{"at": 13.0, "from": "1234567892", "expect": "block"},
{"at": 14.5, "from": "1234567892", "expect": "block"},
# done
{"at": 15.0, "from": None }
]
# database & migrations
db = Database(":memory:")
db.setup()
with db as conn:
conn.execute("INSERT INTO whitelist(number) VALUES (?)",("408--white",))
conn.execute("INSERT INTO blacklist(number) VALUES (?)",("408--black",))
# we use a fake modem
modem = MockModem(prog)
# build state machine
sm = StateMachine(db, modem, 0, REDIAL_WINDOW)
# wait until completion
try:
sm.run()
except CompletedException: