def log(self, message, level=log.DEBUG):
if self.conf['RUN_TYPE'] == 'TASK' and self.conf['DO_ACTION']:
if self.conf['LOG_ENABLED'] and level >= Log.numeric_level(
self.conf['LOG_LEVEL']):
l = Log()
l.message = message
l.ref_object = self.ref_object.__class__.__name__ + " (" + str(
self.ref_object.id) + ")"
l.level = int(level)
l.spider_name = self.name
l.scraper = self.scraper
l.save()
#Delete old logs
if Log.objects.count() > self.conf['LOG_LIMIT']:
items = Log.objects.all()[self.conf['LOG_LIMIT']:]
for item in items:
item.delete()
super(DjangoBaseSpider, self).log(message, level)
def log(self, message, level=log.DEBUG):
if self.conf['RUN_TYPE'] == 'TASK' and self.conf['DO_ACTION']:
if self.conf['LOG_ENABLED'] and level >= Log.numeric_level(self.conf['LOG_LEVEL']):
l = Log()
l.message = message
l.ref_object = self.ref_object.__class__.__name__ + " (" + str(self.ref_object.id) + ")"
l.level = int(level)
l.spider_name = self.name
if hasattr(self, 'scraper_runtime'):
l.scraper_runtime = self.scraper_runtime
if hasattr(self, 'scraper'):
l.scraper = self.scraper
l.save()
#Delete old logs
if Log.objects.count() > self.conf['LOG_LIMIT']:
items = Log.objects.all()[self.conf['LOG_LIMIT']:]
for item in items:
item.delete()
super(DjangoBaseSpider, self).log(message, level)
def log(self, message, level=log.DEBUG):
if self.conf["RUN_TYPE"] == "TASK" and self.conf["DO_ACTION"]:
if self.conf["LOG_ENABLED"] and level >= Log.numeric_level(self.conf["LOG_LEVEL"]):
l = Log()
l.message = message
l.ref_object = self.ref_object.__class__.__name__ + " (" + str(self.ref_object.id) + ")"
l.level = int(level)
l.spider_name = self.name
l.scraper = self.scraper
l.save()
# Delete old logs
if Log.objects.count() > self.conf["LOG_LIMIT"]:
items = Log.objects.all()[self.conf["LOG_LIMIT"] :]
for item in items:
item.delete()
super(DjangoBaseSpider, self).log(message, level)
def log(self, message, level=log.DEBUG):
if self.conf['RUN_TYPE'] == 'TASK' and self.conf['DO_ACTION']:
if self.conf['LOG_ENABLED'] and level >= Log.numeric_level(self.conf['LOG_LEVEL']):
l = Log()
l.message = message
l.ref_object = self.ref_object.__class__.__name__ + " (" + str(self.ref_object.pk) + ")"
l.type = 'None'
l.level = int(level)
l.spider_name = self.name
l.scraper = self.scraper
# Look for corresponding log markers
lms = LogMarker.objects.filter(
Q(ref_object=l.ref_object) | Q(ref_object=''),
Q(spider_name=l.spider_name) | Q(spider_name=''),
Q(scraper=l.scraper) | Q(scraper__isnull=True),
)
for lm in lms:
if lm.message_contains in l.message:
if lm.custom_type:
l.type = lm.custom_type
else:
l.type = lm.get_mark_with_type_display()
l.save()
#Delete old logs
if Log.objects.count() > self.conf['LOG_LIMIT']:
items = Log.objects.all()[self.conf['LOG_LIMIT']:]
for item in items:
item.delete()
super(DjangoBaseSpider, self).log(message, level)
def log(self, message, level=log.DEBUG):
if self.conf['RUN_TYPE'] == 'TASK' and self.conf['DO_ACTION']:
if self.conf['LOG_ENABLED'] and level >= Log.numeric_level(self.conf['LOG_LEVEL']):
l = Log()
l.message = message
l.ref_object = self.ref_object.__class__.__name__ + " (" + str(self.ref_object.pk) + ")"
l.type = 'None'
l.level = int(level)
l.spider_name = self.name
l.scraper = self.scraper
# Look for corresponding log markers
lms = LogMarker.objects.filter(
Q(ref_object=l.ref_object) | Q(ref_object=''),
Q(spider_name=l.spider_name) | Q(spider_name=''),
Q(scraper=l.scraper) | Q(scraper__isnull=True),
)
for lm in lms:
if lm.message_contains in l.message:
if lm.custom_type:
l.type = lm.custom_type
else:
l.type = lm.get_mark_with_type_display()
l.save()
#Delete old logs
if Log.objects.count() > self.conf['LOG_LIMIT']:
items = Log.objects.all()[self.conf['LOG_LIMIT']:]
for item in items:
item.delete()
super(DjangoBaseSpider, self).log(message, level)
