def start_logger(spider_name):
# TODO: FIX read for files like spidername.log.1
filename = datetime.now().strftime("%Y-%m-%d." + spider_name + ".log")
logfile_ = logfile.LogFile(filename, GLOBAL_PATH + '/logs')
logger = ScrapyFileLogObserver(logfile_, logging.DEBUG)
tlog.addObserver(logger.emit)
def __init__(self, *args, **kwargs):
super(TaobaoSpider, self).__init__(*args, **kwargs)
self.login_data['logname'] ='your account'
self.login_data['originalLogpasswd'] = 'your password'
self.login_data['logpasswd'] = md5(self.login_data['originalLogpasswd']).hexdigest()
self.cookie_handle = cookielib.CookieJar()
self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookie_handle))
urllib2.install_opener(self.opener)
#匹配跳转的页面是tmall还是taobao
self.r_route_page_mark = re.compile(r"<h1 id=\"mallLogo\" >")
ScrapyFileLogObserver(open("spider.log", 'w'), level=log.INFO).start()
ScrapyFileLogObserver(open("spider_error.log", 'w'), level=log.ERROR).start()
def __init__(self, date=None, coursecode=None):
if date is None or coursecode is None:
self.historical = False
# start_url = "http://racing.hkjc.com/racing/Info/meeting/RaceCard/english/Local/"
# raise ValueError("Invalid spider parameters")
else:
self.racedate = date
self.racecode = coursecode
self.historical = True
logfile = open('testlog.log', 'w')
log_observer = ScrapyFileLogObserver(logfile, level=logging.DEBUG)
log_observer.start()
def start_logger(debug):
"""
Logger will log for file if debug set to True else will print to cmdline.
The logfiles will rotate after exceeding since of 1M and 100 count.
"""
if debug:
tlog.startLogging(sys.stdout)
else:
filename = datetime.now().strftime("%Y-%m-%d.scrapy.log")
logfile_ = logfile.LogFile(filename, 'logs/', maxRotatedFiles=100)
logger = ScrapyFileLogObserver(logfile_, logging.INFO)
tlog.addObserver(logger.emit)
def __init__(self, *args, **kwargs):
ScrapyFileLogObserver(open("spider.log", 'w'),
level=logging.INFO).start()
ScrapyFileLogObserver(open("spider_error.log", 'w'),
level=logging.ERROR).start()
def __init__(self, *args, **kwargs):
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
ScrapyFileLogObserver(open("spider.log", 'w'),
level=logging.INFO).start()
ScrapyFileLogObserver(open("spider_error.log", 'w'),
level=logging.ERROR).start()
import sys, os, datetime, errno
import logging
from scrapy.log import ScrapyFileLogObserver
try:
import sys, os, datetime, errno
today = datetime.datetime.utcnow().strftime("%Y%m%d")
logdir = None
# ACCUM is the root directory
try:
accum = os.environ["ACCUM"]
except:
accum = "/lfs1/users/wat"
logdir = os.path.join(accum, "log/escort/%s/www.eros.com/" % today)
# ensure log directory exists
try:
os.makedirs(logdir)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
logfile = open(os.path.join(logdir, "scrapy.log"), 'a')
log_observer = ScrapyFileLogObserver(logfile, level=logging.DEBUG)
log_observer.start()
except Exception as e:
print >> sys.stderr, "Failed to create log dir %r [%r]" % (logdir, e)
import sys
print >> sys.stderr, "SETTINGS: log file %r" % logfile
def __init__(self, name=None, **kwargs):
ScrapyFileLogObserver(open("spider.log", 'w'), level=log.INFO).start()
ScrapyFileLogObserver(open("spider_error.log", 'w'),
level=log.ERROR).start()
super(PostloopSpider, self).__init__(name, **kwargs)
import logging from scrapy.log import ScrapyFileLogObserver from scrapy.conf import settings logfile = open(settings['DF_LOG_FILE'], 'w') log_observer = ScrapyFileLogObserver(logfile, level=logging.WARNING) log_observer.start()
