def build_module(project, env=None):
'''Build project script as module'''
from pyspider.libs import base_handler
assert 'name' in project, 'need name of project'
assert 'script' in project, 'need script of project'
if env is None:
env = {}
# fix for old non-package version scripts
pyspider_path = os.path.join(os.path.dirname(__file__), "..")
if pyspider_path not in sys.path:
sys.path.insert(1, pyspider_path)
env = dict(env)
env.update({
'debug': project.get('status', 'DEBUG') == 'DEBUG',
})
loader = ProjectLoader(project)
module = loader.load_module(project['name'])
# logger inject
module.log_buffer = []
module.logging = module.logger = logging.Logger(project['name'])
if env.get('enable_stdout_capture', True):
handler = SaveLogHandler(module.log_buffer)
handler.setFormatter(LogFormatter(color=False))
else:
handler = logging.StreamHandler()
handler.setFormatter(LogFormatter(color=True))
module.logger.addHandler(handler)
if '__handler_cls__' not in module.__dict__:
BaseHandler = module.__dict__.get('BaseHandler',
base_handler.BaseHandler)
for each in list(six.itervalues(module.__dict__)):
if inspect.isclass(each) and each is not BaseHandler \
and issubclass(each, BaseHandler):
module.__dict__['__handler_cls__'] = each
_class = module.__dict__.get('__handler_cls__')
assert _class is not None, "need BaseHandler in project module"
instance = _class()
instance.__env__ = env
instance.project_name = project['name']
instance.project = project
return {
'loader': loader,
'module': module,
'class': _class,
'instance': instance,
'exception': None,
'exception_log': '',
'info': project,
'load_time': time.time(),
}
def logstr(self):
result = []
formater = LogFormatter(color=False)
for record in self.logs:
if isinstance(record, six.string_types):
result.append(pretty_unicode(record))
else:
if record.exc_info:
a, b, tb = record.exc_info
tb = hide_me(tb, globals())
record.exc_info = a, b, tb
result.append(pretty_unicode(formater.format(record)))
result.append(u'\n')
return u''.join(result)
def logstr(self):
result = []
formater = LogFormatter(color=False)
for record in self.logs:
if isinstance(record, basestring):
result.append(pretty_unicode(record))
else:
if record.exc_info:
a, b, tb = record.exc_info
tb = hide_me(tb, globals())
record.exc_info = a, b, tb
result.append(pretty_unicode(formater.format(record)))
result.append(u'\n')
return u''.join(result)
def logstr(self):
"""handler the log records to formatted string"""
result = []
formater = LogFormatter(color=False)
for record in self.logs:
if isinstance(record, six.string_types):
result.append(pretty_unicode(record))
else:
if record.exc_info:
a, b, tb = record.exc_info
tb = hide_me(tb, globals())
record.exc_info = a, b, tb
result.append(pretty_unicode(formater.format(record)))
result.append(u'\n')
return u''.join(result)
def load_module(self, fullname):
if self.mod is None:
mod = self.mod = imp.new_module(self.name)
else:
mod = self.mod
mod.__file__ = '<%s>' % self.name
mod.__loader__ = self
mod.__project__ = self.project
mod.__package__ = ''
# logger inject
log_buffer = []
mod.logging = mod.logger = logging.Logger(self.name)
handler = SaveLogHandler(log_buffer)
handler.setFormatter(LogFormatter(color=False))
mod.logger.addHandler(handler)
mod.log_buffer = log_buffer
code = self.get_code(fullname)
six.exec_(code, mod.__dict__)
linecache.clearcache()
if '__handler_cls__' not in mod.__dict__:
BaseHandler = mod.__dict__.get('BaseHandler', base_handler.BaseHandler)
for each in list(six.itervalues(mod.__dict__)):
if inspect.isclass(each) and each is not BaseHandler \
and issubclass(each, BaseHandler):
mod.__dict__['__handler_cls__'] = each
return mod
def logstr(self):
result = []
formater = LogFormatter(color=False)
for record in self.logs:
if isinstance(record, basestring):
result.append(record)
continue
else:
if record.exc_info:
a, b, tb = record.exc_info
tb = hide_me(tb, globals())
record.exc_info = a, b, tb
result.append(formater.format(record))
result.append('\n')
ret = ''.join(result)
if isinstance(ret, unicode):
return ret
else:
try:
return ret.decode('utf8')
except UnicodeDecodeError as e:
return repr(ret)
def logstr(self):
result = []
formater = LogFormatter(color=False)
for record in self.logs:
if isinstance(record, basestring):
result.append(record)
continue
else:
if record.exc_info:
a, b, tb = record.exc_info
tb = hide_me(tb, globals())
record.exc_info = a, b, tb
result.append(formater.format(record))
result.append('\n')
ret = ''.join(result)
if isinstance(ret, unicode):
return ret
else:
try:
return ret.decode('utf8')
except UnicodeDecodeError as e:
return repr(ret)