class SpiderRule(base):
__tablename__ = "SCP_SPIDER_RULE"
spider_id = createColumn("spider_id", True)
seqnum = createColumn("seqnum", True)
link_extractor_type = createColumn("link_extractor_type")
url = createColumn("url")
method_name = createColumn("method_name") # call back method name
class SpiderHistoryUrl(base):
__tablename__ = "SCP_SPIDER_HISURL"
spider_id = createColumn("spider_id", True)
url = createColumn("url", True)
url_visit_status = createColumn("url_visit_status")
created_datetime = createColumn("created_datetime")
update_datetime = createColumn("update_datetime")
class ItemNameMap(base):
__tablename__ = "SCP_ITEM_NAME_MAP"
item_name = createColumn("item_name", True)
field_name = createColumn("field_name",
foreignKeyStr="SCP_ITEM_FIELD_TBL.field_name")
spider_id = createColumn("spider_id",
foreignKeyStr="SCP_ITEM_FIELD_TBL.spider_id")
class ItemFieldTbl(base):
__tablename__ = "SCP_ITEM_FIELD_TBL"
spider_id = createColumn("spider_id",
True,
foreignKeyStr="SCP_DOMAIN_TBL.spider_id")
field_name = createColumn("field_name", True)
# item_id = createColumn("item_id",True)
item_name = createColumn("item_name")
is_primary_key = createColumn("is_primary_key")
def createClass():
# 动态 ITEM 表
# createdClasses['nyw58ItemOrign'] = type('nyw58ItemOrign', (object,), {'createdVars': '123'})
# createdClasses['nyws'] = type('nyws', (object,), {'createdVars': '234'})
spiderTblList = getTableList(SpiderTbl)
for spiderTbl in spiderTblList:
# 映射类名称
classname = spiderTbl.spider_id + 'ItemOrigin'
fieldList = getItemTblList(spiderTbl.spider_id)
# 类字段信息
fieldDict = {}
# 表名称
fieldDict['__tablename__'] = 'scp_crawl_{}'.format(
spiderTbl.spider_id).upper()
for _field in fieldList:
fieldDict[_field.field_name] = createColumn(
_field.field_name, stringToBoolen(_field.is_primary_key))
createdClasses[classname] = type(classname, (base, ), fieldDict)
class SpiderRef(base):
__tablename__ = "SCP_SPIDER_REF"
spider_id = createColumn("spider_id", True)
seqnum = createColumn("seqnum", True)
ref_type = createColumn("ref_type")
ref_value = createColumn("ref_value")
class SpiderTbl(base):
__tablename__ = "SCP_SPIDER_TBL"
spider_id = createColumn("spider_id", True)
class_name = createColumn("class_name")
descr = createColumn("descr")
class DomainTbl(base):
__tablename__ = "SCP_DOMAIN_TBL"
domain_id = createColumn("domain_id", True)
descr = createColumn("descr")