def get_generator_test_batched_for_id(id):
max_test = db.get_count_on_video(id) - options.max_length
for batch in chunks(db.get_datas_on_one(id), options.batch_size):
imgs, gts = zip(*batch)
yield model.preprocess(
np.asarray([db.load_imgs(img) for img in imgs]),
np.asarray([db.get_groundtruth(gt, 255.0) for gt in gts]))
def testChunks(self):
'''
chunks - split an iterable in chunks of given size
'''
TO_TEST = [('My beautiful tortoise', 5,
['My be', 'autif', 'ul to', 'rtois', 'e']),
(['a', 'ab', 'abc', 'abcd'], 2,
[['a', 'ab'], ['abc', 'abcd']])]
for iterable, size, expected in TO_TEST:
self.assertEqual(U.chunks(iterable, size), expected)
# Doesn't work with dicts
self.assertRaises(TypeError, U.chunks, dict(a=1, b=2), 2)
def run(self, index_file, method_id=None):
self.init(index_file)
method_id = method_id or self._settings['EXTRACTING_METHOD_ID']
count = 0
for block in chunks(self.PICTURES_BLOCK, index.items()):
# Save vectors to database
query = ''
for id, features in block:
query += self._dbc.make_sql(
sql.CREATE_FEATURE_VECTOR, {
'picture_id': id,
'method_id': method_id,
'features': features,
})
if query:
self._dbc.perform(query, retries=self.RETRIES_COUNT)
self._dbc.commit()
count += len(block)
logging.info("Block imported. %s vectors saved" % (count, ))
logging.info("Import is finished. Total %s vectors saved." % (count, ))
def run(self, filename, retailer_id, parser=None, mapper=None, mapper_params=None,
log_file=None, datafeed_id=None, readonly=False, searched_offers=()):
print('*** Datafeed loader runned in {} mode ***'.format('READONLY' if readonly else 'WRITE'))
self.readonly = readonly or bool(searched_offers)
assert all([retailer_id, datafeed_id or self.readonly, filename]), 'Not all required params is vaild'
self.init()
self.product_logger = loggers.Logger(log_file, retailer_id, datafeed_id, self.readonly)
self.init_vendor_stop_words(retailer_id)
self.products_originals = []
parser = PARSERS[parser](self.product_logger, filename, mapper, mapper_params)
parser.set_observed_offers(searched_offers)
with parser.decompressor() as f:
parser.init_parser(f)
# Load offers
self.updated = self.inserted = self.skipped = 0
for block in chunks(self.PRODUCT_BLOCK, parser.offers()):
if searched_offers:
if parser.is_all_searched():
break
else:
continue
# Update products in database
u, i, s = self.update_products(block, datafeed_id, retailer_id)
self.updated += u
self.inserted += i
self.skipped += s
print("Block imported. Updated {o.updated}, inserted {o.inserted}, skipped {o.skipped}".format(o=self))
if searched_offers:
return parser._observed
def get_generator_batched():
for batch in chunks(db.get_datas(), options.batch_size):
imgs, gts = zip(*batch)
yield model.preprocess(np.asarray([db.load_imgs(img) for img in imgs]),
np.asarray(gts))
def get_validation_generator_batched():
for batch in chunks(db.get_tests(), options.batch_size):
imgs, gts = zip(*batch)
yield model.preprocess(
np.asarray([db.load_imgs(img) for img in imgs]),
np.asarray([db.get_groundtruth(gt, 255.0) for gt in gts]))