class InvertedIndex:
def __init__(self):
self._db = IndexDatabase()
self._tokenizer = Tokenizer()
self.__logger = Logger().get_logger(__name__)
def calculate_relevance(self, request):
scores = defaultdict(float)
total_docs_count = self._db.get_constant('total_docs_count')
for target_word in request:
word_meta = self._db.get(target_word)
target_docs = word_meta.keys()
for doc in target_docs:
idf = math.log(total_docs_count / len(target_docs))
scores[doc] += word_meta[doc] * abs(idf) / len(request)
return scores
def search(self, request, result_size):
parsed_request = self._tokenizer.tokenize(request)
if not parsed_request:
self.__logger.error('Empty search request.')
return None
target_request = [
word for word in parsed_request
if word in self._db.get_words_list()
]
relevance = self.calculate_relevance(target_request)
return sorted(relevance.items(),
key=lambda item: item[1],
reverse=True)[:result_size]
class HTTPClient(object):
def __init__(self, url, method='GET', headers=None, cookie=None):
"""
:param method:
:param headers: Must be a dict. Such as headers={'Content_Type':'text/html'}
"""
self.logger = Logger(__name__).get_logger()
self.url = url
self.session = requests.session()
self.method = method.upper()
self.headers = headers
self.cookie = cookie
self._set_header()
self._set_cookie()
def _set_header(self):
"""设置header"""
if self.headers:
self.session.headers.update(self.headers)
self.logger.info('Set headers: {0}'.format(self.headers))
def _set_cookie(self):
"""设置cookie"""
if self.cookie:
self.session.cookies.update(self.cookie)
self.logger.info('Set cookies: {0}'.format(self.cookie))
def _check_method(self):
"""检查传入的method是否可用。"""
if self.method not in METHODS:
self.logger.exception(
UnSupportMethod(u'不支持的method:{0}，请检查传入参数！'.format(
self.method)))
else:
return True
def send(self, params=None, data=None, **kwargs):
"""send request to url.If response 200,return response, else return None."""
if self._check_method():
response = self.session.request(method=self.method,
url=self.url,
params=params,
data=data,
**kwargs)
self.logger.info('{0} {1}.'.format(self.method, self.url))
if response.status_code == 200:
self.logger.info('request success: {0}\n{1}'.format(
response, response.content.strip()))
return response
else:
self.logger.error('request failed: {0} {1}'.format(
response, response.reason))
def persist_tweet(self, tweet):
try:
tweet_json = json.loads(tweet)
if (Tweet.is_retweet(tweet_json)):
Logger.warn('Ignorando RT...\n')
else:
self.collection.insert(tweet_json)
Logger.ok(tweet)
except BaseException as e:
Logger.error('Falha ao salvar tweet\nStacktrace: {}'.format(e))
class IndexDatabase:
def __init__(self):
db = MongoClient(host=os.getenv('MONGO_HOST', config.MONGO['host']),
port=int(
os.getenv('MONGO_PORT',
config.MONGO['port']))).inverted_index
self._constants = db.constants
self._index = db.index
self._userdata = db.userdata
self.__logger = Logger().get_logger(__name__)
def get(self, word, doc=None):
query = self._index.find_one({word: {'$exists': True}})
if query is not None:
return query[word] if doc is None else query[word][doc]
else:
self.__logger.error(f'No such word: {word}')
raise KeyError
def get_constant(self, name):
query = self._constants.find_one({name: {'$exists': True}})
if query is not None:
return query[name]
else:
self.__logger.error(f'No such constant: {name}')
raise KeyError
def get_words_list(self):
return [
list(word.keys())[0] for word in self._index.find({}, {'_id': 0})
]
def write_index(self, index, total_docs_count):
try:
self._constants.drop()
self._index.drop()
self._constants.insert_one({'total_docs_count': total_docs_count})
for word in index:
self._index.insert({word: index[word]}, check_keys=False)
except Exception:
self.__logger.exception('Error writing index to the database.')
def insert_api_keys(self, tokens):
for token in tokens:
self._userdata.update({'api_key': token},
{"$set": {
'api_key': token
}},
upsert=True)
def validate_api_key(self, key):
return self._userdata.find_one({'api_key': key}) is not None
class HTTPClient(object):
def __init__(self, url, method='GET', headers=None, cookie=None):
"""
:param method:
:param headers: Must be a dict. Such as headers={'Content_Type':'text/html'}
"""
self.logger = Logger(__name__).get_logger()
self.url = url
self.session = requests.session()
self.method = method.upper()
self.headers = headers
self.cookie = cookie
self._set_header()
self._set_cookie()
def _set_header(self):
"""设置header"""
if self.headers:
self.session.headers.update(self.headers)
self.logger.info('Set headers: {0}'.format(self.headers))
def _set_cookie(self):
"""设置cookie"""
if self.cookie:
self.session.cookies.update(self.cookie)
self.logger.info('Set cookies: {0}'.format(self.cookie))
def _check_method(self):
"""检查传入的method是否可用。"""
if self.method not in METHODS:
self.logger.exception(UnSupportMethod(u'不支持的method:{0}，请检查传入参数！'.format(self.method)))
else:
return True
def send(self, params=None, data=None, **kwargs):
"""send request to url.If response 200,return response, else return None."""
if self._check_method():
response = self.session.request(method=self.method, url=self.url, params=params, data=data, **kwargs)
self.logger.info('{0} {1}.'.format(self.method, self.url))
if response.status_code == 200:
self.logger.info('request success: {0}\n{1}'.format(response, response.content.strip()))
return response
else:
self.logger.error('request failed: {0} {1}'.format(response, response.reason))
def on_error(self, status_code):
Logger.error(status_code)
class TestCaseGenerator(object):
def __init__(self, project):
self.logger = Logger(__name__).return_logger()
self.project_name = project
self.config_file = 'config_%s.ini' % self.project_name
self.interface_file = '%s.xml' % self.project_name
self.test_file = 'test_%s.py' % self.project_name
self.interface_reader = XMLReader(self.interface_file)
self.tags = self.interface_reader.get_tags()
def generate(self):
with open(self.test_file, 'wb') as test_file:
test_file.write(self.import_string)
for tag in self.tags:
# interface = self.interface_reader.get_url(tag)
class_string = self.get_class(tag)
test_file.write(class_string)
test_file.write('\n\n\n')
@property
def import_string(self):
import_string = """# -*- coding: utf-8 -*-\nimport unittest\nimport json\n"""
rest_flag = webservice_flag = socket_flag = 0
for tag in self.tags:
interface_type = self.interface_reader.get_type(tag).lower()
if interface_type in ['rest', 'restful', 'http']:
rest_flag = 1
elif interface_type == 'webservice':
webservice_flag = 1
elif interface_type in ['tcp', 'socket']:
socket_flag = 1
else:
self.logger.error(u'没有找到合适的接口类型')
if rest_flag:
import_string += 'import requests\n'
if webservice_flag:
import_string += 'import suds\n'
if socket_flag:
import_string += 'import socket\n'
import_string += 'from src.utils.logger import Logger\n\n'
return import_string
def get_class(self, tag):
data_file = self.interface_reader.get_file(tag)
sheet = self.interface_reader.get_sheet(tag)
excel_reader = ExcelReader(data_file, sheet=sheet)
interface_type = self.interface_reader.get_type(tag).lower()
class_string = 'class Test%s(unittest.TestCase):\n\n' % tag
# todo: rest webservice socket
if interface_type in ['rest', 'restful', 'http']:
setup_string = self.get_rest_setup(tag)
cases_string = ''
teardown_string = self.get_rest_teardown(tag)
for num, case in enumerate(excel_reader.data):
case_string = self.get_rest_test(tag, num, case)
cases_string += case_string
class_string += setup_string + teardown_string + cases_string
return class_string
def get_rest_setup(self, tag):
setup_string = """ def setUp(self):\n pass\n\n"""
return setup_string
def get_rest_teardown(self, tag):
teardown_string = """ def tearDown(self):\n pass\n\n"""
return teardown_string
def get_rest_test(self, tag, num, case):
method = self.interface_reader.get_method(tag)
interface_url = self.interface_reader.get_url(tag)
test_string = """ def test_%s_%d(self):\n pass\n\n""" % (tag,
num)
return test_string
def get_setup(self, interface):
pass
def get_test(self, interface):
pass
def get_teardown(self, interface):
pass
