def parse(self, response):
# datelist=response.css('div.datelist')
# for date in datelist:
# date=date.css.('div.dateheader').extract()[0]
# m=re.compile(r'(\d+)').findall(date)
zq = response.xpath('//ul[contains(@class,"zuqiu")]')
for z in zq:
title = z.xpath('.//li[@class="t4"]/a/text()').extract()
#lines=z.xpath('.//li[@class="t5"]/a/@href').extract()
#lines=z.xpath('.//li[@class="t5"]/a/@href').re(r'(ttzb\d+)')
lines = z.xpath('.//li[@class="t5"]/a/@href')
day = lines.re(r'(\d{8})')
line = lines.re(r'(ttzb\d+)')
if title and line and day:
time = datetime.strptime(
day[0] + ' ' +
z.xpath('.//li[@class="t1"]/text()').extract()[0],
'%Y%m%d %H:%M')
matches = Match.objects(
begin_time__gt=time - timedelta(minutes=5),
begin_time__lt=time + timedelta(minutes=5))
for match in matches:
if match.home_name in title[0] or match.away_name in title[
0]:
print title[0].encode('utf-8')
if not line[0] in match.channels:
match.channels.append(line[0])
match.update(channels=match.channels)
requests.get(
'http://localhost:8989/add_channel?channel_name=' +
line[0])
def start_requests():
matches = Match.objects(begin_time__lt=datetime.now(),
begin_time__gt=datetime.now() - timedelta(hours=3),
stream=1,
ttzb__gt=0)
for match in matches:
stream = get_stream(match.ttzb)
match.update(m3u8=stream)
def parse_fenxi(self, response):
match_id = int(
re.compile(r'https://live.leisu.com/shujufenxi-(\d+)').match(
response.url).group(1))
home = response.css('.vs-l')[0]
home_team_name = home.xpath('.//div/h1/text()').extract()[0]
home_head = 'http:' + home.xpath('.//i/@style').re(
r'background-image: url\((.*)\)')[0]
away = response.css('.vs-r')[0]
away_team_name = home.xpath('.//div/h1/text()').extract()[0]
away_head = 'http:' + away.xpath('.//i/@style').re(
r'background-image: url\((.*)\)')[0]
begin_time = response.xpath('//div[@class="page-info"]/p/span/text()')
match = Match(match_id=match_id,
home_name=home_team_name,
home_head=home_head,
away_name=away_team_name,
away_head=away_head,
begin_time=begin_time.extract()[0] + ' ' +
begin_time.extract()[1] + ':00')
match.save()
def find_ls_match(self, match):
# print match.away_name+'+++'+match.home_name
matches = Match.objects(
begin_time__gt=match.begin_time - timedelta(minutes=5),
begin_time__lt=match.begin_time + timedelta(minutes=5))
if len(matches) <= 0: return
for m_ls in matches:
if match.home_name in m_ls.home_name + m_ls.away_name or match.away_name in m_ls.away_name + m_ls.home_name or m_ls.away_name in match.home_name + match.away_name or m_ls.home_name in match.away_name + match.home_name:
print 'matched'
print match.away_name.encode(
'utf-8') + ' vs ' + match.home_name.encode('utf-8')
m_ls.update(wcj_id=str(match.match_id), stream=1, upsert=True)
return m_ls
def parse_fenxi(self, response):
league_id = response.meta.get('league_id')
begin_time = response.meta.get('begin_time')
league_name = response.meta.get('league_name')
league_img = response.meta.get('league_img')
yapei = response.meta.get('yapei')
shy = response.meta.get('shy')
half = response.meta.get('half')
corner = response.meta.get('corner')
home_score = response.meta.get('home_score')
away_score = response.meta.get('away_score')
[home_half,
away_half] = [x if self.is_empty(x) else 0 for x in half.split('-')]
[home_corner, away_corner
] = [x if self.is_empty(x) else 0 for x in corner.split('-')]
match_id = int(
re.compile(r'https://live.leisu.com/shujufenxi-(\d+)').match(
response.url).group(1))
home = response.xpath('//div[@class="team-home"]')[0]
home_id = home.xpath('.//@data-id').extract()[0]
home_team_name = home.xpath(
'.//div[@class="name"]/span/text()').extract()[0]
home_head = 'http:{}'.format(
home.xpath('.//div[@class="team-icon"]/@style').re(
r"background-image: url\('(.*)'\)")[0])
away = response.xpath('//div[@class="team-away"]')[0]
away_id = away.xpath('.//@data-id').extract()[0]
away_team_name = away.xpath(
'.//div[@class="name"]/span/text()').extract()[0]
away_head = 'http:{}'.format(
away.xpath('.//div[@class="team-icon"]/@style').re(
r'background-image: url\(\'(.*)\'\)')[0])
match = Match(match_id=match_id,
home_name=home_team_name,
home_head=home_head,
away_name=away_team_name,
away_head=away_head,
begin_time=datetime.fromtimestamp(int(begin_time)))
match.league_id = league_id
match.league_name = league_name
match.home_id = home_id
match.away_id = away_id
match.league_img = league_img
match.home_half = int(home_half)
match.away_half = int(away_half)
match.home_corner = int(home_corner)
match.away_corner = int(away_corner)
match.yapei = yapei
match.shy = shy
match.home_score = home_score
match.away_score = away_score
match_found = Match.objects(match_id=match_id)
match.home_red_card = response.meta.get('home_red_card')
match.home_yellow_card = response.meta.get('home_yellow_card')
match.away_red_card = response.meta.get('away_red_card')
match.away_yellow_card = response.meta.get('away_yellow_card')
match.status = 1
if match_found:
match_found.update_one(home_score=home_score,
away_score=away_score,
home_half=home_half,
away_half=away_half,
home_corner=home_corner,
away_corner=away_corner,
upsert=True)
else:
match.save()
yield scrapy.Request(self.team_detail_url.format(home_id),
callback=self.parse_team)
yield scrapy.Request(self.team_detail_url.format(away_id),
callback=self.parse_team)
def parse(self, response):
res = self.session.get(
'http://api.leisu.com/app/live/live?app=0&lang=0&platform=2&ver=2.6.1'
)
j = json.loads(res.text)
mathes = j['matches']
events = j['events']
for match in mathes:
m = Match()
m.match_id = match[0]
m.league_id = str(match[1])
m.league_name = events[str(m.league_id)][0]
m.begin_time = datetime.fromtimestamp(match[3])
home = match[5]
m.home_id = str(home[0])
m.home_name = home[1]
m.home_score = home[2]
away = match[6]
m.away_id = str(away[0])
m.away_name = away[1]
m.away_score = away[2]
m.home_head, m.away_head = self.headers(m.match_id)
m.stream = match[10]
m.ttzb = 0
m.m_from = 'leisu'
m.status = 1
match_found = Match.objects(match_id=m.match_id)
if match_found:
m.stream = 1 if match_found[0].channels else m.stream
match_found.update_one(home_score=m.home_score,
away_score=m.away_score,
stream=m.stream,
begin_time=m.begin_time,
upsert=True)
else:
m.save()
def parsem(self, response):
trs = response.css('tr.against')
for tr in trs:
league_name = tr.xpath(
'.//td[@class="matcha"]/a/text()').extract()[0]
#if league_name != 'NBA' and league_name not in leagues:
# continue
teams = tr.xpath('.//td[@class="teama"]/a/strong/text()').extract()
if len(teams) <= 0: continue
print teams[0].encode('utf-8') + 'vs' + teams[1].encode('utf-8')
live_link = tr.xpath('.//td[@class="live_link"]/a/@href').extract()
match_id = tr.xpath('.//td[@class="live_link"]/@id').extract()[0]
begin_time = tr.xpath('.//td[@class="tixing"]/@t').extract()[0]
if not begin_time: continue
# print match_id
match = Match(match_id=match_id)
match.home_name = teams[0]
match.away_name = teams[1]
match.league_name = league_name
match.begin_time = datetime.strptime(begin_time, '%Y-%m-%d %H:%M')
match.m_from = 'wuchajian'
match.channels = []
match_found = Match.objects(wcj_id=match_id)
match.stream = 1
match.status = 1
match.wcj_id = match_id
# print match.away_name + ' vs ' +match.home_name
if match_found:
self.handle_channel(live_link, match_found[0])
self.find_ls_match(match_found[0])
else:
#if match.league_name == 'NBA':
# match.home_head = head[match.home_name.strip()]
# match.away_head = head[match.away_name.strip()]
# match.save()
# self.handle_channel(live_link, match)
#elif match.league_name in leagues:
m = self.find_ls_match(match)
if m: self.handle_channel(live_link, m)
def parse(self,response):
#print response.body
#res=self.session.get('http://api.leisu.com/lanqiu/app/live?app=0&lang=0&platform=2&ver=2.5.3')
#print res.text
j=json.loads(response.body)
mathes=j[u'matches']
events=j[u'events']
teams=j[u'teams']
for match in mathes:
m = Match()
if match[2] !=1:continue
m.match_id=match[0]
m.league_id=str(match[2])
m.league_name=events[str(m.league_id)][0]
m.begin_time=datetime.fromtimestamp(match[4])
home=match[7]
m.home_id=str(home[0])
home_team=teams[str(home[0])]
m.home_name=home_team[0]
m.home_head=self.leisu_img_fmt.format(home_team[1])
m.home_score=sum(home[2:6])
away=match[6]
away_team=teams[str(away[0])]
m.away_id=str(away[0])
m.away_name=away_team[0]
m.away_score=sum(away[2:6])
m.away_head=self.leisu_img_fmt.format(away_team[1])
m.stream=1
m.ttzb=0
m.m_from='leisu'
m.status=1 if match[3]!=10 else 2
match_found = Match.objects(match_id=m.match_id)
print m.home_name.encode('utf-8')+'vs'+m.away_name.encode('utf-8')
if match_found:
match_found.update_one(home_score=m.home_score, away_score=m.away_score,stream=m.stream,begin_time=m.begin_time,
upsert=True)
else:
m.save()