def preload(self): preload_items = [] retry_count = 0 while True: if retry_count > 3 or self.should_exit: break try: items = self.client.get_tenders() self.stat_queries += 1 except Exception as e: retry_count += 1 logger.error("GET %s retry %d count %d error %s", self.client.prefix_path, retry_count, len(preload_items), restkit_error(e, self.client)) self.sleep(5 * retry_count) self.reset() continue if not items: break preload_items.extend(items) if len(preload_items) >= 100: logger.info("Preload %d lots, last %s", len(preload_items), items[-1]['dateModified']) if len(items) < 10: break if len(preload_items) >= self.config['lot_preload']: break return preload_items
def get(self, item): auction = {} retry_count = 0 while not self.should_exit: try: auction = self.client.get_tender(item['id']) assert auction['data']['id'] == item['id'], "auction.id" assert auction['data']['dateModified'] >= item[ 'dateModified'], "auction.dateModified" break except Exception as e: if retry_count > 3: raise e retry_count += 1 logger.error("GET %s/%s retry %d error %s", self.client.prefix_path, str(item['id']), retry_count, restkit_error(e, self.client)) self.sleep(5) if retry_count > 1: self.reset() if item['dateModified'] != auction['data']['dateModified']: logger.debug("Auction dateModified mismatch %s %s %s", item['id'], item['dateModified'], auction['data']['dateModified']) item['dateModified'] = auction['data']['dateModified'] item = self.patch_version(item) auction['meta'] = item return auction
def preload(self): preload_items = [] while True: try: items = self.client.get_tenders() except Exception as e: logger.error("AuctionSource.preload error %s", restkit_error(e, self.client)) self.reset() break if self.should_exit: return [] if not items: break preload_items.extend(items) if len(preload_items) >= 100: logger.info("Preload %d auctions, last %s", len(preload_items), items[-1]['dateModified']) if len(items) < 10: break if len(preload_items) >= self.config['auction_preload']: break return preload_items
def get(self, item): plan = {} retry_count = 0 if self.cache_path: plan = self.cache_get(item) while not plan: if self.should_exit: break try: plan = self.client.get_tender(item['id']) assert plan['data']['id'] == item['id'], "plan.id" assert plan['data']['dateModified'] >= item['dateModified'], "plan.dateModified" except Exception as e: if retry_count > 3: raise e retry_count += 1 logger.error("GET %s/%s retry %d error %s", self.client.prefix_path, str(item['id']), retry_count, restkit_error(e, self.client)) self.sleep(5 * retry_count) if retry_count > 1: self.reset() plan = {} # save to cache if plan and self.cache_path: self.cache_put(plan) if item['dateModified'] != plan['data']['dateModified']: logger.debug("Plan dateModified mismatch %s %s %s", item['id'], item['dateModified'], plan['data']['dateModified']) item['dateModified'] = plan['data']['dateModified'] item = self.patch_version(item) plan['meta'] = item self.stat_getitem += 1 return self.patch_plan(plan)
def preload(self): preload_items = [] # try prelaod last tenders first if self.fast_client: try: items = self.fast_client.get_tenders() self.stat_queries += 1 if not len(items): logger.debug("Preload fast 0 tenders") raise ValueError() preload_items.extend(items) logger.info("Preload fast %d tenders, last %s", len(preload_items), items[-1]['dateModified']) except: pass retry_count = 0 while True: if retry_count > 3 or self.should_exit: break try: items = self.client.get_tenders() self.stat_queries += 1 except Exception as e: retry_count += 1 logger.error("GET %s retry %d count %d error %s", self.client.prefix_path, retry_count, len(preload_items), restkit_error(e, self.client)) self.sleep(5 * retry_count) self.reset() continue if not items: break preload_items.extend(items) if len(preload_items) >= 100: logger.info("Preload %d tenders, last %s", len(preload_items), items[-1]['dateModified']) if len(items) < 10: self.fast_client = None break if len(preload_items) >= self.config['tender_preload']: break return preload_items
def preload(self): preload_items = [] # try prelaod last plans first if self.fast_client: try: items = self.fast_client.get_tenders() if not len(items): logger.debug("Preload fast 0 plans") raise ValueError() preload_items.extend(items) logger.info("Preload fast %d plans, last %s", len(preload_items), items[-1]['dateModified']) except: pass while True: try: items = self.client.get_tenders() except Exception as e: logger.error("PlanSource.preload error %s", restkit_error(e, self.client)) self.reset() break if self.should_exit: return [] if not items: break preload_items.extend(items) if len(preload_items) >= 100: logger.info("Preload %d plans, last %s", len(preload_items), items[-1]['dateModified']) if len(items) < 10: self.fast_client = None break if len(preload_items) >= self.config['plan_preload']: break return preload_items
def get(self, item): lot = {} retry_count = 0 if self.cache_path: lot = self.cache_get(item) while not lot: if self.should_exit: break try: lot = self.client.get_tender(item['id']) assert lot['data']['id'] == item['id'], "lot.id" assert lot['data']['dateModified'] >= item['dateModified'], "lot.dateModified" except Exception as e: if retry_count > 3: raise e retry_count += 1 logger.error("GET %s/%s retry %d error %s", self.client.prefix_path, str(item['id']), retry_count, restkit_error(e, self.client)) self.sleep(5 * retry_count) if retry_count > 1: self.reset() lot = {} # save to cache if lot and self.cache_path: self.cache_put(lot) if item['dateModified'] != lot['data']['dateModified']: logger.debug("Lot dateModified mismatch %s %s %s", item['id'], item['dateModified'], lot['data']['dateModified']) item['dateModified'] = lot['data']['dateModified'] item = self.patch_version(item) lot['meta'] = item self.stat_getitem += 1 return self.patch_lot(lot)