def gather_games():
db = DBsgf()
num = 0
for f, info in downloaded_sgf_files():
try:
db.add(game_item(f, info))
os.remove(f)
num += 1
except Exception as err:
print("error while loading file: '%s':" % f)
print(err.message)
print("%d new files added" % num)
def gather_games():
db = DBsgf()
num = 0
for f, info in downloaded_sgf_files():
try:
db.add(game_item(f, info))
os.remove(f)
num += 1
except Exception as err:
print("error while loading file: '%s':" % f)
print(err.message)
print("%d new files added" % num)
def parse(self, response):
db = DBsgf()
this_year = self.get_year(response)
# all tournament links:
# response.xpath('//tbody//a/@href').extract()
#
for selection in response.xpath('//tbody//tr')[1:]:
item = IgokisenNewsItem(this_year).parse(selection)
if not db.exists(item):
db.add(item)
url = response.urljoin(item['link'])
yield scrapy.Request(url, callback=self.parseTournamentGames)
def parse(self, response):
db = DBsgf()
this_year = self.get_year(response)
# all tournament links:
# response.xpath('//tbody//a/@href').extract()
#
for selection in response.xpath('//tbody//tr')[1:]:
item = IgokisenNewsItem(this_year).parse(selection)
if not db.exists(item):
db.add(item)
url = response.urljoin(item['link'])
yield scrapy.Request(url, callback=self.parseTournamentGames)
