def test_no_retry(self, mock_retry):
mock_retry.side_effect = Retry()
responses.add(
responses.GET,
self.untappd_url,
body=self.json_data,
status=200,
)
self.assertFalse(UntappdMetadata.objects.exists())
look_up_beer(self.beer.id)
self.assertTrue(
UntappdMetadata.objects.filter(beer=self.beer).exists())
mock_retry.assert_not_called()
def test_retry_no_existing_data(self):
responses.add(
responses.GET,
self.untappd_url,
status=429,
headers=self.limit_headers,
)
self.assertFalse(UntappdMetadata.objects.exists())
with self.assertRaises(Retry):
look_up_beer(self.beer.id)
self.assertFalse(
UntappdMetadata.objects.filter(beer=self.beer).exists())
untappd_timestamp = APIRateLimitTimestamp.objects.get()
self.assertEqual(
untappd_timestamp.rate_limit_expires_at,
UTC.localize(datetime.datetime(2019, 11, 15, 3, 0, 0)),
)
def test_retry_lockout_expired(self):
APIRateLimitTimestamp.objects.create(
api_type="untappd",
rate_limit_expires_at=UTC.localize(
datetime.datetime(2019, 11, 15, 3, 0, 0)),
)
responses.add(
responses.GET,
self.untappd_url,
status=200,
body=self.json_data,
)
self.assertFalse(UntappdMetadata.objects.exists())
look_up_beer(self.beer.id)
self.assertTrue(
UntappdMetadata.objects.filter(beer=self.beer).exists())
self.assertFalse(APIRateLimitTimestamp.objects.exists())
def test_retry_lockout_active(self):
APIRateLimitTimestamp.objects.create(
api_type="untappd",
rate_limit_expires_at=UTC.localize(
datetime.datetime(2019, 11, 15, 3, 0, 0)),
)
responses.add(
responses.GET,
self.untappd_url,
status=200,
# will trigger an exception if this gets called
body="{}",
)
self.assertFalse(UntappdMetadata.objects.exists())
with self.assertRaises(Retry):
look_up_beer(self.beer.id)
self.assertFalse(
UntappdMetadata.objects.filter(beer=self.beer).exists())
untappd_timestamp = APIRateLimitTimestamp.objects.get()
self.assertEqual(
untappd_timestamp.rate_limit_expires_at,
UTC.localize(datetime.datetime(2019, 11, 15, 3, 0, 0)),
)
def get_beer(self, name, manufacturer, pricing=None, venue=None, **defaults):
if not self.styles:
self.fetch_styles()
name = name.strip()
LOG.debug(
"get_beer(): name %s, mfg %s, defaults %s",
name,
manufacturer,
defaults,
)
mfg_name = manufacturer.name.replace("™", "").replace("®", "")
for ending in COMMON_BREWERY_ENDINGS:
if mfg_name.endswith(ending):
mfg_name = mfg_name.replace(ending, "").strip()
name = self.reformat_beer_name(name, mfg_name)
unique_fields = (
"manufacturer_url",
"untappd_url",
"beer_advocate_url",
"taphunter_url",
"taplist_io_pk",
"beermenus_slug",
"rate_beer_url",
)
field_names = {i.name for i in Beer._meta.fields}
bogus_defaults = set(defaults).difference(field_names)
if bogus_defaults:
raise ValueError(f'Unknown field(s) {", ".join(sorted(bogus_defaults))}')
for key, val in list(defaults.items()):
if val and key.endswith("_url"):
unquoted = unquote(val)
if unquoted != val:
LOG.debug(
"Replacing unquoted value for %s (%s) with %s",
key,
val,
unquoted,
)
defaults[key] = unquoted
fix_urls(defaults)
unique_fields_present = {
field: value
for field, value in defaults.items()
if field in set(unique_fields) and value
}
serving_sizes = {i.volume_oz: i for i in ServingSize.objects.all()}
if "style" in defaults and not isinstance(defaults["style"], Style):
defaults["style"] = self.get_style(defaults["style"])
elif not defaults.get("style"):
defaults["style"] = self.guess_style(name)
beer = None
if unique_fields_present:
filter_expr = Q()
for field, value in unique_fields_present.items():
if value:
filter_expr |= Q(**{field: value})
# get all possible matches
# after moderation, this should only be one
queryset = Beer.objects.filter(filter_expr)
options = list(queryset)
if len(options) > 1:
# pick the one which has the preferred field set based on order
for field in unique_fields_present:
for option in options:
if getattr(option, field):
beer = option
break
elif options:
beer = options[0]
else:
LOG.debug("No match found based on URL")
try:
abv = defaults.pop("abv")
except KeyError:
pass
else:
if isinstance(abv, str):
if abv.endswith("%"):
abv = abv[:-1]
abv = Decimal(abv)
defaults["abv"] = abv
if not beer:
try:
beer = (
Beer.objects.filter(
Q(name=name) | Q(alternate_names__name=name),
manufacturer=manufacturer,
)
.distinct()
.get()
)
LOG.debug("looked up %s for %s", beer, name)
except Beer.DoesNotExist:
LOG.debug("beer %s not found", name)
beer = None
if (
defaults.get("style")
and defaults["style"].name.casefold() in name.casefold()
):
subbed_name = re.sub(
rf'\s{defaults["style"].name}$',
"",
name,
flags=re.IGNORECASE,
).strip()
try:
beer = (
Beer.objects.filter(
Q(name=subbed_name)
| Q(alternate_names__name=subbed_name),
manufacturer=manufacturer,
)
.distinct()
.get()
)
except Beer.DoesNotExist:
LOG.debug("Substituted name %s does not exist", subbed_name)
else:
LOG.debug("Successfully replaced %s with %s", name, beer)
if not beer:
beer = Beer.objects.create(
name=name,
manufacturer=manufacturer,
**defaults,
)
except Beer.MultipleObjectsReturned:
LOG.error(
"Found duplicate results for name %s from mfg %s!",
name,
manufacturer,
)
# just take the first one
beer = Beer.objects.filter(
Q(name=name) | Q(alternate_names__name=name),
manufacturer=manufacturer,
)[0]
needs_update = False
if beer.logo_url and beer.logo_url == manufacturer.logo_url:
beer.logo_url = None
needs_update = True
if not beer.automatic_updates_blocked:
for field, value in defaults.items():
# instead of using update_or_create(), only update fields *if*
# they're set in `defaults`
if not value or getattr(beer, field) == value:
# it's either unset or is already set to this value
continue
if field == "logo_url":
# these don't have to be unique
if beer.logo_url:
if venue and venue.tap_list_provider == "taphunter":
LOG.info(
"Not trusting beer logo for %s from TapHunter"
" because TH does not distinguish between "
"beer and brewery logos",
beer,
)
continue
found = False
for target, provider in PROVIDER_BREWERY_LOGO_STRINGS.items():
if target in value:
LOG.info(
"Not overwriting logo for beer %s (%s)"
" with brewery logo from %s",
beer,
beer.logo_url,
provider,
)
found = True
break
if found:
continue
elif field.endswith("_url"):
if (
Beer.objects.exclude(id=beer.id)
.filter(**{field: value})
.exists()
):
LOG.warning(
"skipping updating %s (%s) for %s (PK %s)"
" because it would conflict",
field,
value,
beer.name,
beer.id,
)
continue
saved_value = getattr(beer, field)
if value != saved_value:
# TODO mark as unmoderated
setattr(beer, field, value)
needs_update = True
if manufacturer.logo_url and not beer.logo_url:
beer.logo_url = manufacturer.logo_url
needs_update = True
if needs_update:
beer.save()
if pricing:
if not venue:
raise ValueError("You must specify a venue with a price")
beer.prices.filter(venue=venue).delete()
for price_info in pricing:
if price_info["price"] > 500:
LOG.warning(
"Skipping bogus price %s for %s oz of %s",
price_info["price"],
price_info["volume_oz"],
beer,
)
continue
LOG.debug("Rounding volume")
price_info["volume_oz"] = Decimal(round(price_info["volume_oz"], 1))
try:
serving_size = serving_sizes[price_info["volume_oz"]]
except KeyError:
serving_size = ServingSize.objects.get_or_create(
volume_oz=price_info["volume_oz"],
defaults={"name": price_info["name"]},
)[0]
serving_sizes[price_info["volume_oz"]] = serving_size
try:
BeerPrice.objects.create(
serving_size=serving_size,
beer=beer,
venue=venue,
price=price_info["price"],
)
except InvalidOperation:
LOG.error(
"Unable to handle price %s for beer %s capacity %s",
price_info["price"],
beer,
serving_size,
)
raise
if beer.untappd_url:
# queue up an async fetch
try:
# if it has an untappd URL, queue a lookup for the next in line
look_up_beer.delay(beer.id)
except OperationalError as exc:
if str(exc).casefold() == "max number of clients reached".casefold():
LOG.error("Reached redis limit!")
# fall back to doing it synchronously
look_up_beer(beer.id) # pylint: disable=no-value-for-parameter
else:
raise
return beer
def get_beer(self,
name,
manufacturer,
pricing=None,
venue=None,
**defaults):
name = name.strip()
LOG.debug(
'get_beer(): name %s, mfg %s, defaults %s',
name,
manufacturer,
defaults,
)
unique_fields = (
'manufacturer_url',
'untappd_url',
'beer_advocate_url',
'taphunter_url',
'taplist_io_pk',
)
field_names = {i.name for i in Beer._meta.fields}
bogus_defaults = set(defaults).difference(field_names)
if bogus_defaults:
raise ValueError(
f'Unknown field(s) {", ".join(sorted(bogus_defaults))}')
for key, val in list(defaults.items()):
if val and key.endswith('_url'):
unquoted = unquote(val)
if unquoted != val:
LOG.debug(
'Replacing unquoted value for %s (%s) with %s',
key,
val,
unquoted,
)
defaults[key] = unquoted
fix_urls(defaults)
unique_fields_present = {
field: value
for field, value in defaults.items()
if field in set(unique_fields) and value
}
serving_sizes = {i.volume_oz: i for i in ServingSize.objects.all()}
if 'style' in defaults and not isinstance(defaults['style'], Style):
defaults['style'] = self.get_style(defaults['style'])
# TODO: delete this
if isinstance(defaults.get('style'), Style):
defaults['style'] = defaults.pop('style')
beer = None
if unique_fields_present:
filter_expr = Q()
for field, value in unique_fields_present.items():
if value:
filter_expr |= Q(**{field: value})
# get all possible matches
# after moderation, this should only be one
queryset = Beer.objects.filter(filter_expr)
options = list(queryset)
if len(options) > 1:
# pick the one which has the preferred field set based on order
for field in unique_fields_present:
for option in options:
if getattr(option, field):
beer = option
break
elif options:
beer = options[0]
else:
LOG.debug('No match found based on URL')
try:
abv = defaults.pop('abv')
except KeyError:
pass
else:
if isinstance(abv, str):
if abv.endswith('%'):
abv = abv[:-1]
abv = Decimal(abv)
defaults['abv'] = abv
if not beer:
try:
beer = Beer.objects.filter(
Q(name=name) | Q(alternate_names__name=name),
manufacturer=manufacturer,
).distinct().get()
except Beer.DoesNotExist:
beer = Beer.objects.create(
name=name,
manufacturer=manufacturer,
**defaults,
)
except Beer.MultipleObjectsReturned:
LOG.error(
'Found duplicate results for name %s from mfg %s!',
name,
manufacturer,
)
# just take the first one
beer = Beer.objects.filter(
Q(name=name) | Q(alternate_names__name=name),
manufacturer=manufacturer,
)[0]
needs_update = False
if beer.logo_url and beer.logo_url == manufacturer.logo_url:
beer.logo_url = None
needs_update = True
if not beer.automatic_updates_blocked:
for field, value in defaults.items():
# instead of using update_or_create(), only update fields *if*
# they're set in `defaults`
if not value or getattr(beer, field) == value:
# it's either unset or is already set to this value
continue
if field == 'logo_url':
# these don't have to be unique
if beer.logo_url:
if venue and venue.tap_list_provider == 'taphunter':
LOG.info(
'Not trusting beer logo for %s from TapHunter'
' because TH does not distinguish between '
'beer and brewery logos', beer)
continue
found = False
for target, provider in PROVIDER_BREWERY_LOGO_STRINGS.items(
):
if target in value:
LOG.info(
'Not overwriting logo for beer %s (%s) with brewery logo'
' from %s',
beer,
beer.logo_url,
provider,
)
found = True
break
if found:
continue
elif field.endswith('_url'):
if Beer.objects.exclude(id=beer.id).filter(**{
field: value
}).exists():
LOG.warning(
'skipping updating %s (%s) for %s (PK %s)'
' because it would conflict',
field,
value,
beer.name,
beer.id,
)
continue
saved_value = getattr(beer, field)
if value != saved_value:
# TODO mark as unmoderated
setattr(beer, field, value)
needs_update = True
if manufacturer.logo_url and not beer.logo_url:
beer.logo_url = manufacturer.logo_url
needs_update = True
if needs_update:
beer.save()
if pricing:
if not venue:
raise ValueError('You must specify a venue with a price')
for price_info in pricing:
if price_info['price'] > 500:
LOG.warning(
'Skipping bogus price %s for %s oz of %s',
price_info['price'],
price_info['volume_oz'],
beer,
)
continue
try:
serving_size = serving_sizes[price_info['volume_oz']]
except KeyError:
serving_size = ServingSize.objects.get_or_create(
volume_oz=price_info['volume_oz'],
defaults={'name': price_info['name']},
)[0]
serving_sizes[price_info['volume_oz']] = serving_size
try:
BeerPrice.objects.update_or_create(
serving_size=serving_size,
beer=beer,
venue=venue,
defaults={'price': price_info['price']})
except InvalidOperation:
LOG.error(
'Unable to handle price %s for beer %s capacity %s',
price_info['price'], beer, serving_size)
raise
if beer.untappd_url:
# queue up an async fetch
try:
# if it has an untappd URL, queue a lookup for the next in line
look_up_beer.delay(beer.id)
except OperationalError as exc:
if str(exc).casefold(
) == 'max number of clients reached'.casefold():
LOG.error('Reached redis limit!')
# fall back to doing it synchronously
look_up_beer(beer.id)
else:
raise
return beer