class YaocaiItem(scrapy.Item):
type = scrapy.Field()
name = scrapy.Field()
effect = scrapy.Field()
images = scrapy.Field()
alias = scrapy.Field()
english = scrapy.Field()
product = scrapy.Field()
origin = scrapy.Field()
price = scrapy.Field()
specification = scrapy.Field()
variety = scrapy.Field()
encyclopedias = scrapy.Field()
identification = scrapy.Field()
outline = scrapy.Field()
discourse = scrapy.Field()
characters = scrapy.Field()
class SeletestItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
# define the fields for your item here like:
# name = scrapy.Field()
title = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(),
)
amount = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(), )
rate = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(), )
period = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(), )
start = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(), )
end = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(), )
invest_records = scrapy.Field(
output_processor=Join()
)
pay_type = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(), )
loaner_info = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(), )
loan_using = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(), )
loan_info = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(), )
progress = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(), )
code = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(), )
web_name = scrapy.Field(output_processor=Join())
url = scrapy.Field(output_processor=Join())
web_code = scrapy.Field(output_processor=Join())
item_code = scrapy.Field(output_processor=Join())
a = scrapy.Field()
b = scrapy.Field()
class SantiItem(scrapy.Item):
# define the fields for your item here like:
title = scrapy.Field()
content = scrapy.Field()
class TestchenItem(scrapy.Item):
pNo = scrapy.Field()
pName = scrapy.Field()
entName = scrapy.Field()
pAddr = scrapy.Field()
pApprovalName = scrapy.Field()
pApproveOrg = scrapy.Field()
pSupervision = scrapy.Field()
pubTime = scrapy.Field()
pBudget = scrapy.Field()
linkman = scrapy.Field()
tel = scrapy.Field()
mobile = scrapy.Field()
email = scrapy.Field()
fax = scrapy.Field()
bidTime = scrapy.Field()
bidAddr = scrapy.Field()
agentName = scrapy.Field()
agentAddr = scrapy.Field()
agentLinkman = scrapy.Field()
agentTel = scrapy.Field()
agentMobile = scrapy.Field()
agentEmail = scrapy.Field()
agentFax = scrapy.Field()
prov = scrapy.Field()
city = scrapy.Field()
district = scrapy.Field()
spider = scrapy.Field()
source = scrapy.Field()
sourceUrl = scrapy.Field()
getfileStartTime = scrapy.Field()
getfileEndTime = scrapy.Field()
getfileTimeDesc = scrapy.Field()
text = scrapy.Field()
files = scrapy.Field()
class ChemistItem(scrapy.Item):
# define the fields for your item here like:
name = scrapy.Field()
site_chemist_url = scrapy.Field()
address = scrapy.Field()
address2 = scrapy.Field()
city = scrapy.Field()
state = scrapy.Field()
country = scrapy.Field()
zip = scrapy.Field()
phone = scrapy.Field()
email = scrapy.Field()
cell = scrapy.Field()
hours_of_operation = scrapy.Field()
mode_of_payment = scrapy.Field()
keywords = scrapy.Field()
latitude = scrapy.Field()
longitude = scrapy.Field()
jd_verified = scrapy.Field()
class Hello05Item(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title = scrapy.Field()
link = scrapy.Field()
price = scrapy.Field()
class MemeImage(scrapy.Item):
image_urls = scrapy.Field()
images = scrapy.Field()
class CourseItem(scrapy.Item):
# define the fields for your item here like:
title = scrapy.Field()
start_date = scrapy.Field()
end_date = scrapy.Field()
url = scrapy.Field()
class CommentItem(scrapy.Item):
url = scrapy.Field()
thread = scrapy.Field()
op = scrapy.Field()
thread_date = scrapy.Field()
textpost = scrapy.Field()
comments = scrapy.Field()
vote_points = scrapy.Field()
upvoted = scrapy.Field()
comment = scrapy.Field()
user = scrapy.Field()
time = scrapy.Field()
class JianshuArticleItem(scrapy.Item):
url = scrapy.Field(output_processor=TakeFirst())
content = scrapy.Field()
class BooksItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
body = scrapy.Field()
class JobBoleArticleItem(scrapy.Item):
title = scrapy.Field()
create_date = scrapy.Field(input_processor=MapCompose(date_convert))
url = scrapy.Field()
url_object_id = scrapy.Field()
front_image_url = scrapy.Field(output_processor=MapCompose(return_value))
front_image_path = scrapy.Field()
praise_nums = scrapy.Field(input_processor=MapCompose(get_nums))
comment_nums = scrapy.Field(input_processor=MapCompose(get_nums))
fav_nums = scrapy.Field(input_processor=MapCompose(get_nums))
tags = scrapy.Field(input_processor=MapCompose(remove_comment_tags),
output_processor=Join(","))
content = scrapy.Field()
def get_insert_sql(self):
insert_sql = """
insert into jobbole_article(title, url, url_object_id, front_image_url, front_image_path,
comment_nums, fav_nums, praise_nums, tags, content, create_date)
values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
params = (self["title"], self["url"], self["url_object_id"],
self["front_image_url"], self["front_image_path"],
self["comment_nums"], self["fav_nums"], self["praise_nums"],
self["tags"], self["content"], self["create_date"])
return insert_sql, params
class LagouJob(scrapy.Item):
title = scrapy.Field()
url = scrapy.Field()
url_object_id = scrapy.Field()
salary = scrapy.Field()
job_city = scrapy.Field()
work_years = scrapy.Field()
degree_need = scrapy.Field()
job_type = scrapy.Field()
publish_time = scrapy.Field()
job_advantage = scrapy.Field()
job_desc = scrapy.Field()
job_addr = scrapy.Field()
company_name = scrapy.Field()
company_url = scrapy.Field()
tags = scrapy.Field()
crawl_time = scrapy.Field()
class StoreItem(scrapy.Item):
name = scrapy.Field()
mobile = scrapy.Field()
address = scrapy.Field()
class Demo94Item(scrapy.Item):
model = scrapy.Field()
price = scrapy.Field()
dist = scrapy.Field()
city = scrapy.Field()
date = scrapy.Field()
class SaichengItem(scrapy.Item):
'''存储赛程信息'''
lunci = scrapy.Field()
bs_num_id = scrapy.Field()
bs_time = scrapy.Field()
host_team = scrapy.Field()
h_team_id = scrapy.Field()
res_score = scrapy.Field()
guest_team = scrapy.Field()
g_team_id = scrapy.Field()
all_rang = scrapy.Field()
half_rang = scrapy.Field()
sizes_balls_a = scrapy.Field()
sizes_balls_h = scrapy.Field()
half_score = scrapy.Field()
def get_insert_data(self):
'''数据库执行sql语句'''
insert_sql = 'INSERT INTO schedule_data values (null,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
data = ( self['lunci'],self['bs_time'],self['host_team'],self['res_score'],self['guest_team'],
self['all_rang'],self['half_rang'],self['sizes_balls_a'],self['sizes_balls_h'],self['half_score'])
return insert_sql,data
class AscItem(scrapy.Item):
"""
Basic object for link extraction.
Initialized in parse_obj, yielded and saved to the JSON result file.
"""
url = scrapy.Field()
version = scrapy.Field()
artifact_id = scrapy.Field()
group_id = scrapy.Field()
body = scrapy.Field()
sig_hash = scrapy.Field()
sig_key_id = scrapy.Field()
sig_version = scrapy.Field()
sig_type = scrapy.Field()
sig_pub_alg = scrapy.Field()
sig_created = scrapy.Field()
sig_expires = scrapy.Field()
class Player_DataItem(scrapy.Item):
'''存储球员信息'''
play_id = scrapy.Field()
c_number = scrapy.Field()
c_name = scrapy.Field()
fan_name = scrapy.Field()
birthday = scrapy.Field()
stature = scrapy.Field()
weight = scrapy.Field()
weizhi = scrapy.Field()
nation = scrapy.Field()
selfvalue = scrapy.Field()
fin_date = scrapy.Field()
youshi = scrapy.Field()
style = scrapy.Field()
ruodian = scrapy.Field()
nowteam = scrapy.Field()
def get_insert_data(self):
insert_sql = '''INSERT INTO player_data(c_number,c_name,fan_name,birthday,stature,weight,weizhi,nation,selfvalue,fin_date,youshi,style,ruodian,nowteam)values ("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")'''
data = (self['c_number'],self['c_name'],self['fan_name'],self['birthday'],self['stature'],self['weight'],self['weizhi'],
self['nation'],self['selfvalue'],self['fin_date'],self['youshi'],self['style'],self['ruodian'],self['nowteam'])
return insert_sql,data
class MemebotItem(scrapy.Item):
image_urls = scrapy.Field()
images = scrapy.Field()
class RedditItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title = scrapy.Field()
img_link = scrapy.Field()
img = scrapy.Field()
class printerItem(scrapy.Item):
Brand = scrapy.Field()
Name = scrapy.Field()
Price = scrapy.Field()
PrinterUrl = scrapy.Field()
class BankofmarionvaItem(scrapy.Item):
title = scrapy.Field()
description = scrapy.Field()
date = scrapy.Field()
class ScrapingboxesItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
company = scrapy.Field()
price_ex_BTW = scrapy.Field()
minimum_purchase = scrapy.Field()
in_stock = scrapy.Field()
description = scrapy.Field()
url = scrapy.Field()
product_type = scrapy.Field()
inner_dim1 = scrapy.Field()
inner_dim2 = scrapy.Field()
inner_dim3 = scrapy.Field()
height_sorter = scrapy.Field()
diameter = scrapy.Field()
inner_variable_dimension_MIN = scrapy.Field()
inner_variable_dimension_MAX = scrapy.Field()
outer_variable_dimension_MIN = scrapy.Field()
outer_variable_dimension_MAX = scrapy.Field()
variable_height = scrapy.Field()
outer_dim1 = scrapy.Field()
outer_dim2 = scrapy.Field()
outer_dim3 = scrapy.Field()
standard_size = scrapy.Field()
color = scrapy.Field()
wall_thickness = scrapy.Field()
bottles = scrapy.Field()
price_table = scrapy.Field()
all_tags = scrapy.Field()
extra_dim = scrapy.Field()
extra_outer_dim = scrapy.Field()
indices_dict = scrapy.Field()
test_field = scrapy.Field()
price_incl_BTW = scrapy.Field()
lowest_price = scrapy.Field()
image_urls = scrapy.Field()
images = scrapy.Field()
class ComercioNew(scrapy.Item):
Date = scrapy.Field(input_processor=MapCompose(deleteBlanks))
Title = scrapy.Field(input_processor=MapCompose(deleteBlanks))
Views = scrapy.Field()
Indignado = scrapy.Field()
Triste = scrapy.Field()
Indiferente = scrapy.Field()
Sorprendido = scrapy.Field()
Contento = scrapy.Field()
Editor = scrapy.Field(input_processor=MapCompose(deleteBlanks))
Category = scrapy.Field(input_processor=MapCompose(deleteBlanks))
Tag = scrapy.Field(input_processor=MapCompose(deleteBlanks))
Hour = scrapy.Field()
Total_Reactions = scrapy.Field()
Views_Reacts_Ratio = scrapy.Field()
class ProxyItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
#爬虫十分简单，要ip+端口，所以一个字段就够用了
addr = scrapy.Field()
class Member(scrapy.Item):
member = scrapy.Field()
memberId = scrapy.Field()
district = scrapy.Field()
href = scrapy.Field()
vote = scrapy.Field()
class JdspiderItem(scrapy.Item):
#商品名称
p_Name = scrapy.Field()
#店铺名称
shop_name = scrapy.Field()
#商品ID
ProductID = scrapy.Field()
#正价
price = scrapy.Field()
#折扣价
PreferentialPrice = scrapy.Field()
# 评论总数
CommentCount = scrapy.Field()
# 好评度
GoodRateShow = scrapy.Field()
# 好评
GoodCount = scrapy.Field()
# 中评
GeneralCount = scrapy.Field()
# 差评
PoorCount = scrapy.Field()
#评论关键字
keyword=scrapy.Field()
#类别
type=scrapy.Field()
##品牌
brand=scrapy.Field()
X_type=scrapy.Field()
F_type=scrapy.Field()
Y_type=scrapy.Field()
class MemberVotes(scrapy.Item):
member = scrapy.Field()
memberId = scrapy.Field()
district = scrapy.Field()
href = scrapy.Field()
chamber = scrapy.Field()
rcs = scrapy.Field()
bill = scrapy.Field()
billTitle = scrapy.Field()
motion = scrapy.Field()
date = scrapy.Field()
vote = scrapy.Field()
aye = scrapy.Field()
nay = scrapy.Field()
nv = scrapy.Field()
excabs = scrapy.Field()
excvote = scrapy.Field()
totalvote = scrapy.Field()
result = scrapy.Field()
session = scrapy.Field()
class BestbuylaptopsItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title = scrapy.Field()
model = scrapy.Field()
price = scrapy.Field()
class StockSpider_items(scrapy.Item):
##基本資料
DATA_TYPE = scrapy.Field() #資料類型－財務報告
SUB_DATA_TYPE = scrapy.Field() #子資料類別－報告類型
CO_ID = scrapy.Field() #股號
CO_FULL_NAME = scrapy.Field() #公司全名
SYear = scrapy.Field() #年度
SSeason = scrapy.Field() #季度
##A
A1 = scrapy.Field() #A1:現金及約當現金
A2 = scrapy.Field() #A2:透過損益按公允價值衡量之金融資產－流動
A3 = scrapy.Field() #A3:透過其他綜合損益按公允價值衡量之金融資產－流動
A4 = scrapy.Field() #A4:按攤銷後成本衡量之金融資產－流動
A5 = scrapy.Field() #A5:避險之金融資產－流動
A5_5 = scrapy.Field() #A5_5:合約資產 - 流動
A6 = scrapy.Field() #A6:非流動負債合計
A7 = scrapy.Field() #A7:普通股股本
##B
B1 = scrapy.Field() #B1:營業收入合計
B2 = scrapy.Field() #B2:營業利益（損失）
B3 = scrapy.Field() #B3:營業外收入及支出合計
B4 = scrapy.Field() #B4:稀釋每股盈餘合計
pass
