def post_process(self, output):
# Check item - OddsTrifectaItem
items = [o for o in output if isinstance(o, OddsTrifectaItem)]
if len(items) <= 1:
raise ContractFail("len(OddsTrifectaItem)")
for item in items:
odds_url_re = re.match(
r"^https://www\.oddspark\.com/keiba/Odds\.do\?.*betType=8.*$",
item["odds_url"][0])
if not odds_url_re:
raise ContractFail("odds_url")
horse_number_re = re.match(r"^\d+ → \d+ → \d+$",
item["horse_number"][0])
if not horse_number_re:
raise ContractFail("horse_number")
odds_re = re.match(r"^\d+\.\d+$", item["odds"][0])
if not odds_re:
raise ContractFail("odds")
# Check request
requests = [o for o in output if isinstance(o, Request)]
if len(requests) != 0:
raise ContractFail("len(Request)")
def post_process(self, output):
if output:
output=output[0]
if not isinstance(output, str):
raise ContractFail("Output is not a valid String. Output is {}".format(output))
if len(re.findall(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', output)) < 1:
raise ContractFail("Output is not a valid link. Output is {}".format(output))
def post_process(self, output):
# Check items
items = list(filter(lambda o: isinstance(o, Item), output))
if len(items) == 0:
raise ContractFail("items is empty")
horse_items = list(filter(lambda o: isinstance(o, HorseItem), output))
if len(horse_items) != 1:
raise ContractFail("len(horse_items) is not 1")
if len(items) != len(horse_items):
raise ContractFail("Unknown item")
def post_process(self, output):
# Check items
items = list(filter(lambda o: isinstance(o, Item), output))
if len(items) == 0:
raise ContractFail("items is empty")
odds_trio_items = list(
filter(lambda o: isinstance(o, OddsTrioItem), output))
if len(odds_trio_items) == 0:
raise ContractFail("odds_trio_items is empty")
if len(items) != len(odds_trio_items):
raise ContractFail("Unknown item")
def post_process(self, output):
# Check requests
requests = list(filter(lambda o: isinstance(o, Request), output))
if len(requests) == 0:
raise ContractFail("requests is empty")
race_denmas = list(
filter(
lambda r: r.url.startswith(
"https://keiba.yahoo.co.jp/race/denma/"), requests))
if len(race_denmas) == 0:
raise ContractFail("race_denmas is empty")
if len(requests) != len(race_denmas):
raise ContractFail("Unknown request")
def post_process(self, output):
for x in output:
if is_item(x):
missing = [arg for arg in self.args if arg not in ItemAdapter(x) or ItemAdapter(x)[arg]==None]
if missing:
missing_str = ", ".join(missing)
raise ContractFail("Missing or None fields: %s. Item is %s" % (missing_str, x))
def post_process(self, output):
for x in output:
if isinstance(x, (BaseItem, dict)):
missing = [arg for arg in self.args if arg not in x]
if missing:
raise ContractFail(
"Missing fields: %s" % ", ".join(missing))
def post_process(self, output):
contract_file_path = '{}{}'.format(self.CONTRACTS_SPEC_DIR, self.args[0])
expected_output = json.load(open(contract_file_path, 'r'))
actual_output = list(map(lambda item: self.skip_dynamic_keys(dict(item)), output))
if expected_output != actual_output:
raise ContractFail(
"Output doesn't match. Actual: %s. Expected: %s" % (actual_output, expected_output))
def post_process(self, output):
# Check item
items = [o for o in output if isinstance(o, CalendarItem)]
if len(items) != 1:
raise ContractFail("CalendarItems is not 1")
item = items[0]
if len(item["calendar_url"]) != 1:
raise ContractFail("len(item.calendar_url) is not 1")
if not item["calendar_url"][0].startswith(
"https://www.oddspark.com/keiba/KaisaiCalendar.do"):
raise ContractFail("item.calendar_url is invalid")
if len(item["race_list_urls"]) <= 1:
raise ContractFail("len(item.race_list_urls) <= 1")
for race_list_url in item["race_list_urls"]:
if not race_list_url.startswith("/keiba/OneDayRaceList.do?"):
raise ContractFail("race_list_url is invalid")
# Check requests
requests = [o for o in output if isinstance(o, Request)]
if len(requests) <= 1:
raise ContractFail("len(requests) <= 1")
for request in requests:
if not request.url.startswith(
"https://www.oddspark.com/keiba/OneDayRaceList.do?"):
raise ContractFail("request is invalid")
def post_process(self, output):
# Check item
items = [o for o in output if isinstance(o, TrainerItem)]
if len(items) != 1:
raise ContractFail("len(TrainerItem)")
item = items[0]
trainer_url_re = re.match(
r"https://www\.oddspark\.com/keiba/TrainerDetail\.do\?trainerNb=\d+$",
item["trainer_url"][0])
if not trainer_url_re:
raise ContractFail("trainer_url")
if not item["trainer_name"][0]:
raise ContractFail("trainer_name")
birthday_re = re.match(r"^\d{4}年\d{1,2}月\d{1,2}日$",
item["birthday"][0])
if not birthday_re:
raise ContractFail("birthday")
gender_re = re.match(r"^男|女$", item["gender"][0])
if not gender_re:
raise ContractFail("gender")
if not item["belong_to"][0]:
raise ContractFail("belong_to")
# Check request
requests = [o for o in output if isinstance(o, Request)]
if len(requests) != 0:
raise ContractFail("requests is not empty")
def post_process(self, output):
for x in output:
if is_item(x):
missing = [
arg for arg in self.args if arg not in ItemAdapter(x)
]
if missing:
missing_fields = ", ".join(missing)
raise ContractFail(f"Missing fields: {missing_fields}")
def post_process(self, output):
# Check items
items = list(filter(lambda o: isinstance(o, Item), output))
if len(items) == 0:
raise ContractFail("items is empty")
race_payoff_items = list(
filter(lambda o: isinstance(o, RacePayoffItem), output))
if len(race_payoff_items) == 0:
raise ContractFail("race_payoff_items is empty")
race_result_items = list(
filter(lambda o: isinstance(o, RaceResultItem), output))
if len(race_result_items) == 0:
raise ContractFail("race_result_items is empty")
if len(items) != (len(race_payoff_items) + len(race_result_items)):
raise ContractFail("Unknown item")
def post_process(self, output):
# Check requests
requests = list(filter(lambda o: isinstance(o, Request), output))
if len(requests) == 0:
raise ContractFail("requests is empty")
odds_requests = list(
filter(
lambda r: re.match(
r"^https://keiba\.yahoo\.co\.jp/odds/st/\d+/\?umaBan=\d+$",
r.url), requests))
if len(odds_requests) == 0:
raise ContractFail("odds_requests is empty")
if len(requests) != len(odds_requests):
raise ContractFail("Unknown request")
# Check items
items = list(filter(lambda o: isinstance(o, Item), output))
if len(items) == 0:
raise ContractFail("items is empty")
odds_trifecta_items = list(
filter(lambda o: isinstance(o, OddsTrifectaItem), output))
if len(odds_trifecta_items) == 0:
raise ContractFail("odds_trifecta_items is empty")
if len(items) != len(odds_trifecta_items):
raise ContractFail("Unknown item")
def post_process(self, output):
occurrences = 0
for x in output:
if self.obj_type_verifier(x):
occurrences += 1
assertion = (self.min_bound <= occurrences <= self.max_bound)
if not assertion:
if self.min_bound == self.max_bound:
expected = self.min_bound
else:
expected = f'{self.min_bound}..{self.max_bound}'
raise ContractFail(
f"Returned {occurrences} {self.obj_name}, expected {expected}")
def post_process(self, output):
occurrences = 0
for x in output:
if isinstance(x, self.obj_type):
occurrences += 1
assertion = (self.min_bound <= occurrences <= self.max_bound)
if not assertion:
if self.min_bound == self.max_bound:
expected = self.min_bound
else:
expected = '%s..%s' % (self.min_bound, self.max_bound)
raise ContractFail("Returned %s %s, expected %s" %
(occurrences, self.obj_name, expected))
def post_process(self, output):
occurrences = 0
for x in output:
if self.checker.check_match(x):
occurrences += 1
assertion = (self.min_bound <= occurrences <= self.max_bound)
if not assertion:
if self.min_bound == self.max_bound:
expected = self.min_bound
else:
expected = '%s..%s' % (self.min_bound, self.max_bound)
raise ContractFail("Returned {} {} requests, expected {}".format(
occurrences, self.result_name, expected))
def post_process(self, output):
# Check item - OddsWinPlaceItem
items = [o for o in output if isinstance(o, OddsWinPlaceItem)]
if len(items) == 0:
raise ContractFail("len(OddsWinPlaceItem)")
for item in items:
odds_url_re = re.match(
r"^https://www\.oddspark\.com/keiba/Odds\.do\?.*betType=1.*$",
item["odds_url"][0])
if not odds_url_re:
raise ContractFail("odds_url")
horse_number_re = re.match(r"^\d+$", item["horse_number"][0])
if not horse_number_re:
raise ContractFail("horse_number")
horse_url_re = re.match(r"/keiba/HorseDetail\.do\?lineageNb=\d+$",
item["horse_url"][0])
if not horse_url_re:
raise ContractFail("horse_url")
odds_win_re = re.match(r"^\d+\.\d+$", item["odds_win"][0])
if not odds_win_re:
raise ContractFail("odds_win")
odds_place_re = re.match(r"^\d+\.\d+ - \d+\.\d+$",
item["odds_place"][0])
if not odds_place_re:
raise ContractFail("odds_place")
# Check request
requests = [o for o in output if isinstance(o, Request)]
if len(requests) != 0:
raise ContractFail("len(requests)")
def post_process(self, output):
if 'item' not in output[0]['feed'][0]:
raise ContractFail('Não nenhum item na saída')
def post_process(self, output):
# Check requests
requests = list(filter(lambda o: isinstance(o, Request), output))
if len(requests) == 0:
raise ContractFail("requests is empty")
odds_exacta_requests = list(
filter(
lambda r: r.url.startswith("https://keiba.yahoo.co.jp/odds/ut/"
), requests))
if len(odds_exacta_requests) != 1:
raise ContractFail("len(odds_exacta_requests) is not 1")
odds_quinella_requests = list(
filter(
lambda r: r.url.startswith("https://keiba.yahoo.co.jp/odds/ur/"
), requests))
if len(odds_quinella_requests) != 1:
raise ContractFail("len(odds_quinella_requests) is not 1")
odds_quinella_place_requests = list(
filter(
lambda r: r.url.startswith(
"https://keiba.yahoo.co.jp/odds/wide/"), requests))
if len(odds_quinella_place_requests) != 1:
raise ContractFail("len(odds_quinella_place_requests) is not 1")
odds_trifecta_requests = list(
filter(
lambda r: r.url.startswith("https://keiba.yahoo.co.jp/odds/st/"
), requests))
if len(odds_trifecta_requests) != 1:
raise ContractFail("len(odds_trifecta_requests) is not 1")
odds_trio_requests = list(
filter(
lambda r: r.url.startswith("https://keiba.yahoo.co.jp/odds/sf/"
), requests))
if len(odds_trio_requests) != 1:
raise ContractFail("len(odds_trio_requests) is not 1")
if len(requests) != (
len(odds_exacta_requests) + len(odds_quinella_requests) +
len(odds_quinella_place_requests) +
len(odds_trifecta_requests) + len(odds_trio_requests)):
raise ContractFail("Unknown request")
# Check items
items = list(filter(lambda o: isinstance(o, Item), output))
if len(items) == 0:
raise ContractFail("items is empty")
odds_win_place_items = list(
filter(lambda o: isinstance(o, OddsWinPlaceItem), output))
if len(odds_win_place_items) == 0:
raise ContractFail("odds_win_place_items is empty")
odds_bracket_quinella_items = list(
filter(lambda o: isinstance(o, OddsBracketQuinellaItem), output))
if len(odds_bracket_quinella_items) == 0:
raise ContractFail("odds_bracket_quinella_items is empty")
if len(items) != (len(odds_win_place_items) +
len(odds_bracket_quinella_items)):
raise ContractFail("Unknown item")
def post_process(self, output):
for x in output:
if isinstance(x, BaseItem):
for arg in self.args:
if not arg in x:
raise ContractFail("'%s' field is missing" % arg)
def post_process(self, output):
# Check requests
requests = list(filter(lambda o: isinstance(o, Request), output))
if len(requests) == 0:
raise ContractFail("requests is empty")
horse_requests = list(
filter(
lambda r: r.url.startswith(
"https://keiba.yahoo.co.jp/directory/horse/"), requests))
if len(horse_requests) == 0:
raise ContractFail("horse_requests is empty")
trainer_requests = list(
filter(
lambda r: r.url.startswith(
"https://keiba.yahoo.co.jp/directory/trainer/"), requests))
if len(trainer_requests) == 0:
raise ContractFail("trainer_requests is empty")
jockey_requests = list(
filter(
lambda r: r.url.startswith(
"https://keiba.yahoo.co.jp/directory/jocky/"), requests))
if len(jockey_requests) == 0:
raise ContractFail("jockey_requests is empty")
odds_requests = list(
filter(
lambda r: r.url.startswith(
"https://keiba.yahoo.co.jp/odds/tfw/"), requests))
if len(odds_requests) != 1:
raise ContractFail("len(odds_requests) is not 1")
race_result_requests = list(
filter(
lambda r: r.url.startswith(
"https://keiba.yahoo.co.jp/race/result/"), requests))
if len(race_result_requests) != 1:
raise ContractFail("len(race_result_requests) is not 1")
if len(requests) != (len(horse_requests) + len(trainer_requests) +
len(jockey_requests) + len(odds_requests) +
len(race_result_requests)):
raise ContractFail("Unknown request")
# Check items
items = list(filter(lambda o: isinstance(o, Item), output))
if len(items) == 0:
raise ContractFail("items is empty")
race_info_items = list(
filter(lambda o: isinstance(o, RaceInfoItem), output))
if len(race_info_items) != 1:
raise ContractFail("len(race_info_items) is not 1")
race_denma_items = list(
filter(lambda o: isinstance(o, RaceDenmaItem), output))
if len(race_denma_items) == 0:
raise ContractFail("race_denma_items is empty")
if len(items) != (len(race_info_items) + len(race_denma_items)):
raise ContractFail("Unknown item")
def post_process(self, output):
if not isinstance(output, list):
raise ContractFail("Output is not a valid list. Output is {}".format(output))
for out in output:
if not isinstance(out, Selector):
raise ContractFail("Output is not a valid SelectorList. Output is {}".format(output))
def post_process(self, output):
# Check item - RaceInfoItem
items = [o for o in output if isinstance(o, RaceInfoItem)]
if len(items) != 1:
raise ContractFail("len(RaceInfoItem) != 1")
item = items[0]
if not item["race_denma_url"][0].startswith(
"https://www.oddspark.com/keiba/RaceList.do?"):
raise ContractFail("race_denma_url is invalid")
race_round_re = re.match(r"^R\d+$", item["race_round"][0])
if not race_round_re:
raise ContractFail("race_round is invalid")
if not item["race_name"][0]:
raise ContractFail("race_name is empty")
start_date_re = re.match(r"^\d{4}年\d{1,2}月\d{1,2}日\(\w\)$",
item["start_date"][0])
if not start_date_re:
raise ContractFail("start_date is invalid")
if not item["place_name"][0]:
raise ContractFail("place_name is invalid")
course_type_length = re.match(r"^\w\d+m$",
item["course_type_length"][0])
if not course_type_length:
raise ContractFail("course_type_length is invalid")
start_time_re = re.match(r"^発走時間 \d{1,2}:\d{1,2}$",
item["start_time"][0])
if not start_time_re:
raise ContractFail("start_time is invalid")
if not item["weather_url"][0].startswith("/local/images/ico-tenki-"):
raise ContractFail("weather_url is invalid")
if "course_condition" in item:
if not item["course_condition"][0].startswith(
"/local/images/ico-baba-"):
raise ContractFail("course_condition is invalid")
if "moisture" in item:
moisture_re = re.match(r"^[\d\.]+\%$", item["moisture"][0])
if not moisture_re:
raise ContractFail("moisture is invalid")
prize_money_re = re.match(
r"^.*賞金.*1着.*[\d,]+円.*2着.*[\d,]+円.*3着.*[\d,]+円.*4着.*[\d,]+円.*5着.*[\d,]+円.*$",
item["prize_money"][0])
if not prize_money_re:
raise ContractFail("prize_money is invalid")
# Check item - RaceDenmaItem
items = [o for o in output if isinstance(o, RaceDenmaItem)]
if len(items) == 0:
raise ContractFail("len(RaceDenmaItem) == 0")
for item in items:
if not item["race_denma_url"][0].startswith(
"https://www.oddspark.com/keiba/RaceList.do?"):
raise ContractFail("race_denma_url is invalid")
if not item["bracket_number"][0]:
raise ContractFail("bracket_number is empty")
if not item["horse_number"][0]:
raise ContractFail("horse_number is empty")
horse_url_re = re.match(r"^/keiba/HorseDetail\.do\?lineageNb=\d+$",
item["horse_url"][0])
if not horse_url_re:
raise ContractFail("horse_url is invalid")
jockey_url_re = re.match(r"^/keiba/JockeyDetail\.do\?jkyNb=\d+$",
item["jockey_url"][0])
if not jockey_url_re:
raise ContractFail("jockey_url is invalid")
jockey_weight_re = re.match(r"^.?\d+(\.\d+)?$",
item["jockey_weight"][0])
if not jockey_weight_re:
raise ContractFail("jockey_weight is invalid")
trainer_url_re = re.match(
r"^/keiba/TrainerDetail\.do\?trainerNb=\d+$",
item["trainer_url"][0])
if not trainer_url_re:
raise ContractFail("trainer_url is invalid")
odds_win_favorite_re = re.match(r"^[\d\.]+ +\(\d+人気\)$",
item["odds_win_favorite"][0])
if not odds_win_favorite_re:
raise ContractFail("odds_win_favorite is invalid")
horse_weight_re = re.match(r"^\d+$", item["horse_weight"][0])
if not horse_weight_re:
raise ContractFail("horse_weight is invalid")
horse_weight_diff_re = re.match(r"^[\+\-±]\d+$",
item["horse_weight_diff"][0])
if not horse_weight_diff_re:
raise ContractFail("horse_weight_diff is invalid")
# Check requests
requests = [o for o in output if isinstance(o, Request)]
count = sum(
r.url.startswith("https://www.oddspark.com/keiba/Odds.do?")
for r in requests)
if count <= 1:
raise ContractFail("Odds page not found")
count = sum(
r.url.startswith("https://www.oddspark.com/keiba/RaceResult.do?")
for r in requests)
if count != 1:
raise ContractFail("Race result page not found")
count = sum(
r.url.startswith(
"https://www.oddspark.com/keiba/HorseDetail.do?lineageNb=")
for r in requests)
if count <= 1:
raise ContractFail("Horse page not found")
count = sum(
r.url.startswith(
"https://www.oddspark.com/keiba/JockeyDetail.do?jkyNb=")
for r in requests)
if count <= 1:
raise ContractFail("Jockey page not found")
count = sum(
r.url.startswith(
"https://www.oddspark.com/keiba/TrainerDetail.do?trainerNb=")
for r in requests)
if count <= 1:
raise ContractFail("Trainer page not found")
def pre_process(self, response):
for header in self.args:
if header not in response.headers:
raise ContractFail('X-CustomHeader not present')
mail_body = {}
for x in response:
for arg in self.args:
if not arg in x:
missing_field.append(arg)
missing_field_list = set(missing_field)
if len(missing_field_list) > 0:
"""
Generate email body and send report on specified email address
and raise an exeption `ContractFail`.
"""
try:
mail_body['args'] = "', '".join(missing_field_list)
self.send_email("""Missing Item Field: ['%(args)s']""" %
mail_body)
except NameError, Argument:
print Argument
finally:
raise ContractFail("'%s' field is missing" % mail_body['args'])
def send_email(self, mail_body):
mailer = MailSender(mailfrom="*****@*****.**",
smtphost="smtp.gmail.com",
smtpport=587,
smtpuser="******",
smtppass="******")
return mailer.send(to=["*****@*****.**"],
subject="StockSpider: Stock Spiders Contract Error",
body=mail_body)
def post_process(self, output):
item = output[0]
if 'title' not in item:
raise ContractFail('title is invalid.')
def post_process(self, output):
for x in output:
if isinstance(x, (BaseItem, dict)):
for arg in self.args:
if arg in x:
raise ContractFail("'%s' field is present" % arg)
def post_process(self, output):
# Check item - RaceResultItem
items = [o for o in output if isinstance(o, RaceResultItem)]
if len(items) == 0:
raise ContractFail("RaceResultItem is empty")
for item in items:
if not item["race_result_url"][0].startswith(
"https://www.oddspark.com/keiba/RaceResult.do?"):
raise ContractFail("race_result_url is invalid")
if not item["result"][0]:
raise ContractFail("result is invalid")
if not item["bracket_number"][0]:
raise ContractFail("bracket_number is invalid")
if not item["horse_number"][0]:
raise ContractFail("horse_number is invalid")
horse_url_re = re.match(r"^/keiba/HorseDetail\.do\?lineageNb=\d+$",
item["horse_url"][0])
if not horse_url_re:
raise ContractFail("horse_url is invalid")
arrival_time_re = re.match(r"^(\d+:)\d+\.\d+$",
item["arrival_time"][0])
if not arrival_time_re:
raise ContractFail("arrival_time is invalid")
if "arrival_margin" in item:
if not item["arrival_margin"][0]:
raise ContractFail("arrival_margin is invalid")
final_600_meters_time_re = re.match(
r"^[\d\.]+$", item["final_600_meters_time"][0])
if not final_600_meters_time_re:
raise ContractFail("final_600_meters_time is invalid")
corner_passing_order_re = re.match(r"^\d+\-\d+\-\d+\-\d+$",
item["corner_passing_order"][0])
if not corner_passing_order_re:
raise ContractFail("corner_passing_order is invalid")
# Check item - RaceCornerPassingOrderItem
items = [
o for o in output if isinstance(o, RaceCornerPassingOrderItem)
]
if len(items) != 4:
raise ContractFail("len(RaceCornerPassingOrderItem) != 4")
for item in items:
if not item["race_result_url"][0].startswith(
"https://www.oddspark.com/keiba/RaceResult.do?"):
raise ContractFail("race_result_url is invalid")
corner_number_re = re.match(r"^.コーナー$", item["corner_number"][0])
if not corner_number_re:
raise ContractFail("corner_number is invalid")
if not item["passing_order"][0]:
raise ContractFail("passing_order is invalid")
# Check item - RaceRefundItem
items = [o for o in output if isinstance(o, RaceRefundItem)]
if len(items) == 0:
raise ContractFail("RaceRefundItem is empty")
for item in items:
if not item["race_result_url"][0].startswith(
"https://www.oddspark.com/keiba/RaceResult.do?"):
raise ContractFail("race_result_url is invalid")
if not item["betting_type"][0]:
raise ContractFail("betting_type is invalid")
if not item["horse_number"][0]:
raise ContractFail("horse_number is invalid")
refund_money_re = re.match(r"^[\d\,]+円$", item["refund_money"][0])
if not refund_money_re:
raise ContractFail("refund_money is invalid")
favorite_re = re.match(r"^\d+番人気$", item["favorite"][0])
if not favorite_re:
raise ContractFail("favorite is invalid")
# Check requests
requests = [o for o in output if isinstance(o, Request)]
if len(requests) != 0:
raise ContractFail("requests is not empty")
def post_process(self, output):
# Check item
items = [o for o in output if isinstance(o, RaceInfoMiniItem)]
if len(items) != 11:
raise ContractFail("len(RaceInfoMiniItem) != 11")
for item in items:
if len(item["race_list_url"]) != 1:
raise ContractFail("len(race_list_url) != 1")
if not item["race_list_url"][0].startswith(
"https://www.oddspark.com/keiba/OneDayRaceList.do?"):
raise ContractFail("race_list_url[0] is invalid")
if len(item["race_name"]) != 1:
raise ContractFail("len(race_name) != 1")
if not item["race_name"][0]:
raise ContractFail("race_name[0] is empty")
if len(item["race_denma_url"]) != 1:
raise ContractFail("len(race_denma_url) != 1")
if not item["race_denma_url"][0].startswith("/keiba/RaceList.do?"):
raise ContractFail("race_denma_url[0] is invalid")
if len(item["course_length"]) != 1:
raise ContractFail("len(course_length) != 1")
if not item["course_length"][0]:
raise ContractFail("course_length[0] is empty")
if len(item["start_time"]) != 1:
raise ContractFail("len(start_time) != 1")
start_time_re = re.match(r"^\d+:\d+$", item["start_time"][0])
if not start_time_re:
raise ContractFail("start_time is invalid pattern")
# Check requests
requests = [o for o in output if isinstance(o, Request)]
if len(requests) != 11:
raise ContractFail("len(requests) != 11")
for request in requests:
if not request.url.startswith(
"https://www.oddspark.com/keiba/RaceList.do?"):
raise ContractFail("request is invalid")
def post_process(self, output):
# Check item
items = [o for o in output if isinstance(o, HorseItem)]
if len(items) != 1:
raise ContractFail("len(HorseItem)")
item = items[0]
horse_url_re = re.match(
r"^https://www\.oddspark\.com/keiba/HorseDetail\.do\?lineageNb=\d+$",
item["horse_url"][0])
if not horse_url_re:
raise ContractFail("horse_url")
if not item["horse_name"][0]:
raise ContractFail("horse_name")
gender_age_re = re.match(r"^\w+｜\d+ 歳$", item["gender_age"][0])
if not gender_age_re:
raise ContractFail("gender_age")
birthday_re = re.match(r"^\d{4}年\d{1,2}月\d{1,2}日$",
item["birthday"][0])
if not birthday_re:
raise ContractFail("birthday")
coat_color_re = re.match(r"^\w+毛$", item["coat_color"][0])
if not coat_color_re:
raise ContractFail("coat_color")
trainer_url_re = re.match(r"^/keiba/TrainerDetail\.do\?trainerNb=\d+$",
item["trainer_url"][0])
if not trainer_url_re:
raise ContractFail("trainer_url")
if not item["owner"][0]:
raise ContractFail("owner")
if not item["breeder"][0]:
raise ContractFail("breeder")
if not item["breeding_farm"][0]:
raise ContractFail("breeding_farm")
if not item["parent_horse_name_1"][0]:
raise ContractFail("parent_horse_name_1")
if not item["parent_horse_name_2"][0]:
raise ContractFail("parent_horse_name_2")
if not item["grand_parent_horse_name_1"][0]:
raise ContractFail("grand_parent_horse_name_1")
if not item["grand_parent_horse_name_2"][0]:
raise ContractFail("grand_parent_horse_name_2")
if not item["grand_parent_horse_name_3"][0]:
raise ContractFail("grand_parent_horse_name_3")
if not item["grand_parent_horse_name_4"][0]:
raise ContractFail("grand_parent_horse_name_4")
# Check request
requests = [o for o in output if isinstance(o, Request)]
if len(requests) != 0:
raise ContractFail("len(requests)")