def checkRelationshipSchema(cls, baseModule):
"""
Walk all relationship schema definitions and confirm that they
have reciprical peers on the remote class.
"""
for relname, rel in cls._relations:
try:
remoteClass = importClass(rel.remoteClass, None)
except AttributeError as e:
logging.critical("RemoteClass '%s' from '%s.%s' not found",
rel.remoteClass, cls.__name__, relname)
continue
try:
rschema = lookupSchema(remoteClass, rel.remoteName)
except ZenSchemaError as e:
logging.critical("Inverse def '%s' for '%s.%s' not found on '%s'",
rel.remoteName, cls.__name__, relname,
rel.remoteClass)
continue
except Exception as e:
logging.critical("RemoteClass '%s' for '%s.%s' problem.",
rel.remoteName, cls.__name__, relname)
logging.critical(e)
continue
try:
localClass = importClass(rschema.remoteClass, None)
except AttributeError as e:
logging.critical(e)
if not issubclass(cls, localClass):
logging.critical(
"Inverse def '%s' from '%s.%s' wrong "
"remoteClass: '%s'", rel.remoteName, cls.__name__, relname,
rschema.remoteClass)
if rschema.remoteName != relname:
logging.critical(
"Inverse def '%s' from '%s.%s' wrong "
"remoteName: '%s'", rel.remoteName, cls.__name__, relname,
rschema.remoteName)
if rel.remoteType != rschema.__class__:
logging.critical("'%s.%s' inverse '%s' type %s != %s",
cls.__name__, relname, rel.remoteName,
rschema.__class__.__name__,
rel.remoteType.__name__)
def _setupCrawlers(self):
for crawlerPath, config in self.config['crawlers'].items():
try:
# Try relative import from crawlers sub-module.
# If that fails, import relative to root
try:
clazz = utils.importClass(crawlerPath, baseModule='crawlers.')
except ModuleNotFoundError:
clazz = utils.importClass(crawlerPath)
# Sanity check
if (not isinstance(clazz, type)
or not issubclass(clazz, RemoteCalendarCrawler)):
raise ValueError('Crawler must be a subclass of RemoteCalendarCrawler')
crawlerConfig = self.config['crawler'].copy()
crawlerConfig.update(config)
self.crawlers.append(clazz(crawlerConfig, self.requester))
except Exception as e:
logger.error(f'Unable to load crawler {crawlerPath}:')
logger.exception(e)
def checkRelationshipSchema(cls, baseModule):
"""
Walk all relationship schema definitions and confirm that they
have reciprical peers on the remote class.
"""
for relname, rel in cls._relations:
try:
remoteClass = importClass(rel.remoteClass, None)
except AttributeError, e:
logging.critical("RemoteClass '%s' from '%s.%s' not found",
rel.remoteClass, cls.__name__, relname)
continue
try:
rschema = lookupSchema(remoteClass, rel.remoteName)
except ZenSchemaError, e:
logging.critical("Inverse def '%s' for '%s.%s' not found on '%s'",
rel.remoteName, cls.__name__, relname,rel.remoteClass)
continue
def checkRelationshipSchema(cls, baseModule):
"""
Walk all relationship schema definitions and confirm that they
have reciprical peers on the remote class.
"""
for relname, rel in cls._relations:
try:
remoteClass = importClass(rel.remoteClass, None)
except AttributeError, e:
logging.critical("RemoteClass '%s' from '%s.%s' not found",
rel.remoteClass, cls.__name__, relname)
continue
try:
rschema = lookupSchema(remoteClass, rel.remoteName)
except ZenSchemaError, e:
logging.critical("Inverse def '%s' for '%s.%s' not found on '%s'",
rel.remoteName, cls.__name__, relname,
rel.remoteClass)
continue
rel.remoteClass, cls.__name__, relname)
continue
try:
rschema = lookupSchema(remoteClass, rel.remoteName)
except ZenSchemaError, e:
logging.critical("Inverse def '%s' for '%s.%s' not found on '%s'",
rel.remoteName, cls.__name__, relname,
rel.remoteClass)
continue
except Exception, e:
logging.critical("RemoteClass '%s' for '%s.%s' problem.",
rel.remoteName, cls.__name__, relname)
logging.critical(e)
continue
try:
localClass = importClass(rschema.remoteClass, None)
except AttributeError, e:
logging.critical(e)
if not issubclass(cls, localClass):
logging.critical(
"Inverse def '%s' from '%s.%s' wrong "
"remoteClass: '%s'", rel.remoteName, cls.__name__, relname,
rschema.remoteClass)
if rschema.remoteName != relname:
logging.critical(
"Inverse def '%s' from '%s.%s' wrong "
"remoteName: '%s'", rel.remoteName, cls.__name__, relname,
rschema.remoteName)
if rel.remoteType != rschema.__class__:
logging.critical("'%s.%s' inverse '%s' type %s != %s",
cls.__name__, relname, rel.remoteName,
logging.critical("RemoteClass '%s' from '%s.%s' not found",
rel.remoteClass, cls.__name__, relname)
continue
try:
rschema = lookupSchema(remoteClass, rel.remoteName)
except ZenSchemaError, e:
logging.critical("Inverse def '%s' for '%s.%s' not found on '%s'",
rel.remoteName, cls.__name__, relname,rel.remoteClass)
continue
except Exception, e:
logging.critical("RemoteClass '%s' for '%s.%s' problem.",
rel.remoteName, cls.__name__, relname)
logging.critical(e)
continue
try:
localClass = importClass(rschema.remoteClass, None)
except AttributeError, e:
logging.critical(e)
if not issubclass(cls, localClass):
logging.critical("Inverse def '%s' from '%s.%s' wrong "
"remoteClass: '%s'", rel.remoteName, cls.__name__,
relname,rschema.remoteClass)
if rschema.remoteName != relname:
logging.critical("Inverse def '%s' from '%s.%s' wrong "
"remoteName: '%s'", rel.remoteName, cls.__name__, relname,
rschema.remoteName)
if rel.remoteType != rschema.__class__:
logging.critical("'%s.%s' inverse '%s' type %s != %s",
cls.__name__, relname, rel.remoteName,
rschema.__class__.__name__, rel.remoteType.__name__)
import config
import utils
import sys
import os
sys.path.append(os.path.split(os.path.realpath(__file__))[0])
# Load logging module
if hasattr(config, 'LOGGING'):
logging = __import__(config.LOGGING)
else:
import filelogging as logging
# Load WeiboIterator
if hasattr(config, 'WEIBO_PROVIDER'):
WeiboProvider = utils.importClass(config.WEIBO_PROVIDER)
else:
from weibo import DefaultWeiboProvider as WeiboProvider
if __name__ == '__main__':
logger = logging.createLogger(config)
logger.info('Spider start')