class Spider(): def __init__(self): self.storage = SpiderStorage() self.string_utils = StringUtils() def crawl(self, param): param = self.__extend_param(param) url = self.build_url(param) key = self.build_key(param) self.storage.set(key, url) def is_crawled(self, param): param = self.__extend_param(param) key = self.build_key(param) return self.storage.contains(key) def get_crawled(self, param): param = self.__extend_param(param) key = self.build_key(param) return self.storage.get(key) def __extend_param(self, param): output = {} if 'stock_symbol' in param: output['stock_symbol'] = param['stock_symbol'] if 'market_type' in param: market_type = param['market_type'] output['market_type'] = market_type output['market_mode'] = self.__extend_market_mode(market_type) if 'market_category' in param: output['market_category'] = param['market_category'] return output def __extend_market_mode(self, market_type): mode_map = { 'stock_exchange_market' : '2', 'otc_market' : '4', } return mode_map[market_type] def build_url(self, param): raise NotImplementedError def build_key(self, param): raise NotImplementedError
class Spider(): def __init__(self): self.storage = SpiderStorage() self.string_utils = StringUtils() def crawl(self, param): param = self.__extend_param(param) url = self.build_url(param) key = self.build_key(param) self.storage.set(key, url) def is_crawled(self, param): param = self.__extend_param(param) key = self.build_key(param) return self.storage.contains(key) def get_crawled(self, param): param = self.__extend_param(param) key = self.build_key(param) return self.storage.get(key) def __extend_param(self, param): output = {} if 'stock_symbol' in param: output['stock_symbol'] = param['stock_symbol'] if 'market_type' in param: market_type = param['market_type'] output['market_type'] = market_type output['market_mode'] = self.__extend_market_mode(market_type) if 'market_category' in param: output['market_category'] = param['market_category'] return output def __extend_market_mode(self, market_type): mode_map = { 'stock_exchange_market': '2', 'otc_market': '4', } return mode_map[market_type] def build_url(self, param): raise NotImplementedError def build_key(self, param): raise NotImplementedError
def __init__(self): self.storage = SpiderStorage() self.string_utils = StringUtils()