class InvertedIndex: def __init__(self): self._db = IndexDatabase() self._tokenizer = Tokenizer() self.__logger = Logger().get_logger(__name__) def calculate_relevance(self, request): scores = defaultdict(float) total_docs_count = self._db.get_constant('total_docs_count') for target_word in request: word_meta = self._db.get(target_word) target_docs = word_meta.keys() for doc in target_docs: idf = math.log(total_docs_count / len(target_docs)) scores[doc] += word_meta[doc] * abs(idf) / len(request) return scores def search(self, request, result_size): parsed_request = self._tokenizer.tokenize(request) if not parsed_request: self.__logger.error('Empty search request.') return None target_request = [ word for word in parsed_request if word in self._db.get_words_list() ] relevance = self.calculate_relevance(target_request) return sorted(relevance.items(), key=lambda item: item[1], reverse=True)[:result_size]
class HTTPClient(object): def __init__(self, url, method='GET', headers=None, cookie=None): """ :param method: :param headers: Must be a dict. Such as headers={'Content_Type':'text/html'} """ self.logger = Logger(__name__).get_logger() self.url = url self.session = requests.session() self.method = method.upper() self.headers = headers self.cookie = cookie self._set_header() self._set_cookie() def _set_header(self): """设置header""" if self.headers: self.session.headers.update(self.headers) self.logger.info('Set headers: {0}'.format(self.headers)) def _set_cookie(self): """设置cookie""" if self.cookie: self.session.cookies.update(self.cookie) self.logger.info('Set cookies: {0}'.format(self.cookie)) def _check_method(self): """检查传入的method是否可用。""" if self.method not in METHODS: self.logger.exception( UnSupportMethod(u'不支持的method:{0},请检查传入参数!'.format( self.method))) else: return True def send(self, params=None, data=None, **kwargs): """send request to url.If response 200,return response, else return None.""" if self._check_method(): response = self.session.request(method=self.method, url=self.url, params=params, data=data, **kwargs) self.logger.info('{0} {1}.'.format(self.method, self.url)) if response.status_code == 200: self.logger.info('request success: {0}\n{1}'.format( response, response.content.strip())) return response else: self.logger.error('request failed: {0} {1}'.format( response, response.reason))
def persist_tweet(self, tweet): try: tweet_json = json.loads(tweet) if (Tweet.is_retweet(tweet_json)): Logger.warn('Ignorando RT...\n') else: self.collection.insert(tweet_json) Logger.ok(tweet) except BaseException as e: Logger.error('Falha ao salvar tweet\nStacktrace: {}'.format(e))
class IndexDatabase: def __init__(self): db = MongoClient(host=os.getenv('MONGO_HOST', config.MONGO['host']), port=int( os.getenv('MONGO_PORT', config.MONGO['port']))).inverted_index self._constants = db.constants self._index = db.index self._userdata = db.userdata self.__logger = Logger().get_logger(__name__) def get(self, word, doc=None): query = self._index.find_one({word: {'$exists': True}}) if query is not None: return query[word] if doc is None else query[word][doc] else: self.__logger.error(f'No such word: {word}') raise KeyError def get_constant(self, name): query = self._constants.find_one({name: {'$exists': True}}) if query is not None: return query[name] else: self.__logger.error(f'No such constant: {name}') raise KeyError def get_words_list(self): return [ list(word.keys())[0] for word in self._index.find({}, {'_id': 0}) ] def write_index(self, index, total_docs_count): try: self._constants.drop() self._index.drop() self._constants.insert_one({'total_docs_count': total_docs_count}) for word in index: self._index.insert({word: index[word]}, check_keys=False) except Exception: self.__logger.exception('Error writing index to the database.') def insert_api_keys(self, tokens): for token in tokens: self._userdata.update({'api_key': token}, {"$set": { 'api_key': token }}, upsert=True) def validate_api_key(self, key): return self._userdata.find_one({'api_key': key}) is not None
class HTTPClient(object): def __init__(self, url, method='GET', headers=None, cookie=None): """ :param method: :param headers: Must be a dict. Such as headers={'Content_Type':'text/html'} """ self.logger = Logger(__name__).get_logger() self.url = url self.session = requests.session() self.method = method.upper() self.headers = headers self.cookie = cookie self._set_header() self._set_cookie() def _set_header(self): """设置header""" if self.headers: self.session.headers.update(self.headers) self.logger.info('Set headers: {0}'.format(self.headers)) def _set_cookie(self): """设置cookie""" if self.cookie: self.session.cookies.update(self.cookie) self.logger.info('Set cookies: {0}'.format(self.cookie)) def _check_method(self): """检查传入的method是否可用。""" if self.method not in METHODS: self.logger.exception(UnSupportMethod(u'不支持的method:{0},请检查传入参数!'.format(self.method))) else: return True def send(self, params=None, data=None, **kwargs): """send request to url.If response 200,return response, else return None.""" if self._check_method(): response = self.session.request(method=self.method, url=self.url, params=params, data=data, **kwargs) self.logger.info('{0} {1}.'.format(self.method, self.url)) if response.status_code == 200: self.logger.info('request success: {0}\n{1}'.format(response, response.content.strip())) return response else: self.logger.error('request failed: {0} {1}'.format(response, response.reason))
def on_error(self, status_code): Logger.error(status_code)
class TestCaseGenerator(object): def __init__(self, project): self.logger = Logger(__name__).return_logger() self.project_name = project self.config_file = 'config_%s.ini' % self.project_name self.interface_file = '%s.xml' % self.project_name self.test_file = 'test_%s.py' % self.project_name self.interface_reader = XMLReader(self.interface_file) self.tags = self.interface_reader.get_tags() def generate(self): with open(self.test_file, 'wb') as test_file: test_file.write(self.import_string) for tag in self.tags: # interface = self.interface_reader.get_url(tag) class_string = self.get_class(tag) test_file.write(class_string) test_file.write('\n\n\n') @property def import_string(self): import_string = """# -*- coding: utf-8 -*-\nimport unittest\nimport json\n""" rest_flag = webservice_flag = socket_flag = 0 for tag in self.tags: interface_type = self.interface_reader.get_type(tag).lower() if interface_type in ['rest', 'restful', 'http']: rest_flag = 1 elif interface_type == 'webservice': webservice_flag = 1 elif interface_type in ['tcp', 'socket']: socket_flag = 1 else: self.logger.error(u'没有找到合适的接口类型') if rest_flag: import_string += 'import requests\n' if webservice_flag: import_string += 'import suds\n' if socket_flag: import_string += 'import socket\n' import_string += 'from src.utils.logger import Logger\n\n' return import_string def get_class(self, tag): data_file = self.interface_reader.get_file(tag) sheet = self.interface_reader.get_sheet(tag) excel_reader = ExcelReader(data_file, sheet=sheet) interface_type = self.interface_reader.get_type(tag).lower() class_string = 'class Test%s(unittest.TestCase):\n\n' % tag # todo: rest webservice socket if interface_type in ['rest', 'restful', 'http']: setup_string = self.get_rest_setup(tag) cases_string = '' teardown_string = self.get_rest_teardown(tag) for num, case in enumerate(excel_reader.data): case_string = self.get_rest_test(tag, num, case) cases_string += case_string class_string += setup_string + teardown_string + cases_string return class_string def get_rest_setup(self, tag): setup_string = """ def setUp(self):\n pass\n\n""" return setup_string def get_rest_teardown(self, tag): teardown_string = """ def tearDown(self):\n pass\n\n""" return teardown_string def get_rest_test(self, tag, num, case): method = self.interface_reader.get_method(tag) interface_url = self.interface_reader.get_url(tag) test_string = """ def test_%s_%d(self):\n pass\n\n""" % (tag, num) return test_string def get_setup(self, interface): pass def get_test(self, interface): pass def get_teardown(self, interface): pass