def get_all(self):
all_emojis = google_spreadsheets.keys(include_empty_keys=True)
corjis = []
for this_emoji in all_emojis:
corgi_urls = ""
if settings.Config.REMOTE_CACHE_RETRIEVE:
try:
corgi_urls = s3.get_all(this_emoji)
except CorgiNotFoundException as e:
logger.warn("Corji not found for emoji %s", this_emoji)
if not corgi_urls:
corgi_urls = google_spreadsheets.get_all(this_emoji)
emoji_name = emoji.demojize(this_emoji).replace(":", "")
corjis.append({
"urls": corgi_urls,
"emoji": this_emoji,
"emoji_name": emoji_name
})
return {
"count": len(corjis),
"emojis": [corji["emoji"] for corji in corjis],
"results": corjis
}
def get(self, emoji=None):
"""Returns an emoji associated with the given emoji.
If no emoji is supplied, a random corji is returned."""
if not emoji:
emoji = random.choice(google_spreadsheets.keys())
corgi_urls = []
# Check for skin-toned emojis and fallback to the undecorated one.
if len(emoji) == 2 and emoji_contains_skin_tone(emoji):
emoji = emoji[0]
# First we'll try using S3.
if settings.Config.REMOTE_CACHE_RETRIEVE:
try:
corgi_urls = s3.get_all(emoji)
except CorgiNotFoundException as e:
logger.warn("Corji not found for emoji %s", emoji)
# If S3 is a no-go, fall back to Spreadsheets.
if not corgi_urls:
corgi_urls = google_spreadsheets.get_all(emoji)
# TODO: do this smarter somehow.
for url in corgi_urls:
try:
requests.get(url)
except Exception as e:
logger.warn("URL {} is invalid; not returning.".format(url))
corgi_urls.remove(url)
return {
"count": len(corgi_urls),
"emoji": emoji if text_contains_emoji(emoji) else "",
"results": corgi_urls
}
from tabulate import tabulate
from corji.data_sources import google_spreadsheets
from corji.logging import Logger
from corji.settings import Config
logger = Logger(Config.LOGGER_NAME,
Config.LOG_PATH,
Config.LOG_NAME)
SPREADSHEET_URL = Config.SPREADSHEET_URL
if __name__ == "__main__":
logger.debug("START: Spreadsheet URL defined: %s", SPREADSHEET_URL)
google_spreadsheets.load(SPREADSHEET_URL)
corgi_counter = Counter()
keys = google_spreadsheets.keys(include_empty_keys=True)
for emoji in keys:
corgis = google_spreadsheets.get_all(emoji)
for corgi in corgis:
try:
requests.get(corgi)
except:
print("FAILURE: {}".format(corgi))
corgi_counter[len(corgis)] += 1
print(tabulate(corgi_counter.items(),
headers=["# of URLs", "Count"],
tablefmt="psql"))
