def __init__(self, urls):
self.urls = urls
self.headers = config.DOWNLOADER_HEADERS
self.timeout = config.DOWNLOADER_TIMEOUT
self.size = config.DOWNLOADER_SIZE
self.proxy_count = config.DOWNLOADER_PROXY_COUNT
self.dbredis = dbredis.DBRedis()
self.logger = logger.Logger()
def __init__(self):
self.dbredis = dbredis.DBRedis()
self.logger = logger.Logger()
self.url = config.URL
with open(config.DATA_FILE, 'r') as datafile:
self.datas = [
data.replace('\n', '') for data in datafile.readlines()
]
def __init__(self):
self.timestamp = time.strftime("%Y%m%d_%H%M%S",
time.localtime(time.time()))
self.dbredis = dbredis.DBRedis()
self.dbpostgresql = dbpostgresql.DBPostgresql(timestamp=self.timestamp)
self.logger = logger.Logger()
# -*- coding: utf-8 -*-
import os
import sys
ROOT = os.getcwd()
sys.path.append(ROOT)
from util import dbredis
if __name__ == "__main__":
dbredis = dbredis.DBRedis()
dbredis.clear_start()
dbredis.set_start('start key', 'start value')
print('start len: ', dbredis.len_start())
print('start hash: ', dbredis.get_start())
dbredis.clear_index()
dbredis.set_index('index key', 'index value')
print('index len: ', dbredis.len_index())
print('index hash: ', dbredis.get_index())
print('proxy hash: ', dbredis.get_proxy(count=10))