def logpath(self, fpath):
"""尝试建立日志文件所在目录及日志文件"""
if not os.path.isabs(fpath):
fpath = os.path.join(current_file_directory(), fpath)
dirname = os.path.dirname(fpath)
if not os.path.isdir(dirname):
os.makedirs(dirname)
if not os.path.isfile(fpath):
with open(fpath, "w") as fp:
fp.close()
def scan(name):
"""
扫描当前目录下的模块
"""
sys.path.append(name)
result = {}
pwd = current_file_directory()
# 模块路径
module_path = os.path.join(pwd, name)
if not os.path.isdir(module_path):
os.makedirs(module_path)
result["mpath"] = module_path
# 数据存储数据
datadir = os.path.join(pwd, "data", name)
if not os.path.isdir(datadir):
os.makedirs(datadir)
#datadir = os.path.join(r"F:\SHLP\Source\data", name)
result["dpath"] = datadir
# 页面存储路径
pattern = r"(item_\d+)\.py$"
recompile = re.compile(pattern)
list_file = os.listdir(module_path)
ufile = filter(recompile.match, list_file)
models = [x[:-3] for x in ufile]
models.sort()
model_objs = map(lazy_import, models)
result["models"] = model_objs
# URL导入路径
urlname = "loadurl"
urlmodel = os.path.join(module_path, "%s.py" % urlname)
if os.path.isfile(urlmodel):
result["urlmodel"] = lazy_import(urlname)
return result
from common_func import current_file_directory
from proxy.common import proxy_ip
# 抓取代理线程数
workA = 1
# 爬取网址线程数
workB = 100
# 启用本地导入模式
loadLocal = True
# 代理，当不用代理时会自动限速 5s
usingProxy = True
iptable = "iptotal"
passwd = "123456"
orderid = '963599851676265'
mainpage = "http://imgs.soufun.com/"
pwd = current_file_directory()
datadir = os.path.join(pwd, "data")
# 线程终止
isAlive = True
if not os.path.isdir(datadir):
os.makedirs(datadir)
# URL日志
def myLog():
l = Log(logname="urlServer")
l.streamHandler()
l.fileHandler(logLevel=40, fpath=os.path.join(pwd, "Log/urlServer.log"))
return l.logger