def get_images(filenames): if not filenames: raise StopIteration dynamo = Config().get_dynamodb() table = Config().table('image') batch_list = BatchList(dynamo) batch_list.add_batch(table, filenames, attributes_to_get=['tags', HASH_KEY]) response = dynamo.batch_get_item(batch_list) items = response['Responses'][table.name]['Items'] for item in items: item['filename'] = item.pop(HASH_KEY) item['tags'] = json.loads(item['tags']) unprocessed_keys = [] if response['UnprocessedKeys'] \ and table.name in response['UnprocessedKeys']: for key in response['UnprocessedKeys'][table.name]['Keys']: unprocessed_keys.append(key['HashKeyElement']) for item in items: yield item if not unprocessed_keys: raise StopIteration for item in get_images(unprocessed_keys): yield item
def test_memoization(self, boto, _table_prefix): _table_prefix.return_value = 'foo' config = Config() mock_table = Mock() config._tables = {'foo-tags': mock_table} table = config.table('tags') eq_(table, mock_table) eq_(boto.connect_dynamodb.call_count, 0)
def add_image_to_tags(filename, tag_names): items = list(get_tag_items(tag_names)) for item in items: filenames = json.loads(item['filenames']) filenames.append(filename) item['filenames'] = json.dumps(filenames) table = Config().table('tag') for new_tag in set(tag_names) - set(x[HASH_KEY] for x in items): items.append(table.new_item(hash_key = new_tag, attrs={'filenames': json.dumps([filename])})) _submit_items(table, items)
def test_build_parser(self): (_, conf) = tempfile.mkstemp() with open(conf, 'w') as config_file: config_file.write("""[catsnap] bucket = boogles table_prefix = bugglez""") config = Config() with patch('catsnap.Config.CONFIG_FILE', conf) as _: parser = config._parser() eq_(parser.get('catsnap', 'bucket'), 'boogles') eq_(parser.get('catsnap', 'table_prefix'), 'bugglez') eq_(config.bucket_name(), 'boogles') eq_(config._table_prefix(), 'bugglez')
def get_tag_items(tag_names): if not tag_names: raise StopIteration dynamo = Config().get_dynamodb() table = Config().table('tag') batch_list = BatchList(dynamo) batch_list.add_batch(table, tag_names, attributes_to_get=['filenames', HASH_KEY]) response = dynamo.batch_get_item(batch_list) items = response['Responses'][table.name]['Items'] unprocessed_keys = [] if response['UnprocessedKeys'] \ and table.name in response['UnprocessedKeys']: for key in response['UnprocessedKeys'][table.name]['Keys']: unprocessed_keys.append(key['HashKeyElement']) for item in items: yield item for item in get_tag_items(unprocessed_keys): yield item