def preload(self):
preload_items = []
retry_count = 0
while True:
if retry_count > 3 or self.should_exit:
break
try:
items = self.client.get_tenders()
self.stat_queries += 1
except Exception as e:
retry_count += 1
logger.error("GET %s retry %d count %d error %s", self.client.prefix_path,
retry_count, len(preload_items), restkit_error(e, self.client))
self.sleep(5 * retry_count)
self.reset()
continue
if not items:
break
preload_items.extend(items)
if len(preload_items) >= 100:
logger.info("Preload %d lots, last %s",
len(preload_items), items[-1]['dateModified'])
if len(items) < 10:
break
if len(preload_items) >= self.config['lot_preload']:
break
return preload_items
def get(self, item):
auction = {}
retry_count = 0
while not self.should_exit:
try:
auction = self.client.get_tender(item['id'])
assert auction['data']['id'] == item['id'], "auction.id"
assert auction['data']['dateModified'] >= item[
'dateModified'], "auction.dateModified"
break
except Exception as e:
if retry_count > 3:
raise e
retry_count += 1
logger.error("GET %s/%s retry %d error %s",
self.client.prefix_path, str(item['id']),
retry_count, restkit_error(e, self.client))
self.sleep(5)
if retry_count > 1:
self.reset()
if item['dateModified'] != auction['data']['dateModified']:
logger.debug("Auction dateModified mismatch %s %s %s", item['id'],
item['dateModified'], auction['data']['dateModified'])
item['dateModified'] = auction['data']['dateModified']
item = self.patch_version(item)
auction['meta'] = item
return auction
def preload(self):
preload_items = []
while True:
try:
items = self.client.get_tenders()
except Exception as e:
logger.error("AuctionSource.preload error %s",
restkit_error(e, self.client))
self.reset()
break
if self.should_exit:
return []
if not items:
break
preload_items.extend(items)
if len(preload_items) >= 100:
logger.info("Preload %d auctions, last %s", len(preload_items),
items[-1]['dateModified'])
if len(items) < 10:
break
if len(preload_items) >= self.config['auction_preload']:
break
return preload_items
def get(self, item):
plan = {}
retry_count = 0
if self.cache_path:
plan = self.cache_get(item)
while not plan:
if self.should_exit:
break
try:
plan = self.client.get_tender(item['id'])
assert plan['data']['id'] == item['id'], "plan.id"
assert plan['data']['dateModified'] >= item['dateModified'], "plan.dateModified"
except Exception as e:
if retry_count > 3:
raise e
retry_count += 1
logger.error("GET %s/%s retry %d error %s", self.client.prefix_path,
str(item['id']), retry_count, restkit_error(e, self.client))
self.sleep(5 * retry_count)
if retry_count > 1:
self.reset()
plan = {}
# save to cache
if plan and self.cache_path:
self.cache_put(plan)
if item['dateModified'] != plan['data']['dateModified']:
logger.debug("Plan dateModified mismatch %s %s %s",
item['id'], item['dateModified'],
plan['data']['dateModified'])
item['dateModified'] = plan['data']['dateModified']
item = self.patch_version(item)
plan['meta'] = item
self.stat_getitem += 1
return self.patch_plan(plan)
def preload(self):
preload_items = []
# try prelaod last tenders first
if self.fast_client:
try:
items = self.fast_client.get_tenders()
self.stat_queries += 1
if not len(items):
logger.debug("Preload fast 0 tenders")
raise ValueError()
preload_items.extend(items)
logger.info("Preload fast %d tenders, last %s",
len(preload_items), items[-1]['dateModified'])
except:
pass
retry_count = 0
while True:
if retry_count > 3 or self.should_exit:
break
try:
items = self.client.get_tenders()
self.stat_queries += 1
except Exception as e:
retry_count += 1
logger.error("GET %s retry %d count %d error %s",
self.client.prefix_path, retry_count,
len(preload_items), restkit_error(e, self.client))
self.sleep(5 * retry_count)
self.reset()
continue
if not items:
break
preload_items.extend(items)
if len(preload_items) >= 100:
logger.info("Preload %d tenders, last %s", len(preload_items),
items[-1]['dateModified'])
if len(items) < 10:
self.fast_client = None
break
if len(preload_items) >= self.config['tender_preload']:
break
return preload_items
def preload(self):
preload_items = []
# try prelaod last plans first
if self.fast_client:
try:
items = self.fast_client.get_tenders()
if not len(items):
logger.debug("Preload fast 0 plans")
raise ValueError()
preload_items.extend(items)
logger.info("Preload fast %d plans, last %s",
len(preload_items), items[-1]['dateModified'])
except:
pass
while True:
try:
items = self.client.get_tenders()
except Exception as e:
logger.error("PlanSource.preload error %s",
restkit_error(e, self.client))
self.reset()
break
if self.should_exit:
return []
if not items:
break
preload_items.extend(items)
if len(preload_items) >= 100:
logger.info("Preload %d plans, last %s", len(preload_items),
items[-1]['dateModified'])
if len(items) < 10:
self.fast_client = None
break
if len(preload_items) >= self.config['plan_preload']:
break
return preload_items
def get(self, item):
lot = {}
retry_count = 0
if self.cache_path:
lot = self.cache_get(item)
while not lot:
if self.should_exit:
break
try:
lot = self.client.get_tender(item['id'])
assert lot['data']['id'] == item['id'], "lot.id"
assert lot['data']['dateModified'] >= item['dateModified'], "lot.dateModified"
except Exception as e:
if retry_count > 3:
raise e
retry_count += 1
logger.error("GET %s/%s retry %d error %s", self.client.prefix_path,
str(item['id']), retry_count, restkit_error(e, self.client))
self.sleep(5 * retry_count)
if retry_count > 1:
self.reset()
lot = {}
# save to cache
if lot and self.cache_path:
self.cache_put(lot)
if item['dateModified'] != lot['data']['dateModified']:
logger.debug("Lot dateModified mismatch %s %s %s",
item['id'], item['dateModified'],
lot['data']['dateModified'])
item['dateModified'] = lot['data']['dateModified']
item = self.patch_version(item)
lot['meta'] = item
self.stat_getitem += 1
return self.patch_lot(lot)