def worker(start, offset): cookie = 'shaishufang=Mjc5MTYwfGZmY2VmYzIyYmMxZjhlZThjNzgzYjFlOGIxOWUwODg2' for i in range(start, start + offset): ui = UserIsbns(str(i), cookie) isbns = ui.run() saveLstToFile('isbns.txt', isbns)
def spider_closed(self, spider): # logging.info("Spider's destructor") # logging.info(self.ISBNS) # saveMatrixToFile(self.UID + '.csv', self.Books) # pass logging.info('Actual Total Books: ' + str(self.TotalBooks)) logging.info('Isbns in ISBNS: ' + str(len(self.ISBNS))) saveLstToFile(self.UID + '.csv', self.ISBNS)
def main(): """ for team in teamIds: for season in seasons: outputFile = DIR + team + "." + season + ".player.csv.processed" res = noneWithAVG(team, season) if len(res) == 0: continue print "INFO: ", outputFile saveMatrixToFile(outputFile, res) for team in teamIds: for season in seasons: outputFile = DIR + team + "." + season + ".player.csv.processed.avg" res = avg(team, season) if len(res) == 0: continue print "INFO: ", outputFile saveLstToFile(outputFile, res) for team in teamIds: for season in seasons: outputFile = DIR + team + "." + season + ".player.csv.processed.total" res = total(team, season) if len(res) == 0: continue print "INFO: ", outputFile saveLstToFile(outputFile, res) """ for team in teamIds: for season in seasons: outputFile = DIR + team + "." + season + ".player.csv.processed.norm" res = normalize(team, season) if len(res) == 0: continue print 'INFO: ', outputFile saveLstToFile(outputFile, res)