def get_heuristics(card): h = {} if not card.text: # TODO: this may not be true, but it's definitely mostly true return try: for handler in _all_handlers: h.update(handler.get(card) or {}) except: log.exception('error getting heuristics for card: %r', card) raise return h
def insert_cards(names, redis_conn, post_insert_hook=None): """ Search gather for cards with the given name, insert that information into the database. This doesn't check for existing cards that have that name, and the act of inserting them :param names: the names of the cards to insert :param redis_conn: used to cache information about the cards :param post_insert_hook: this callable will be invoked with the card that was just inserted :return: if names were provided: { 'inserted': cards_to_insert, 'refetched': refetched_cards, 'mismatches': relevant_mismatches, 'invalid': invalid_names, } else, empty dict """ if not names: return {} invalid_names = [] refetched_names = [] inserted_cards = [] relevant_mismatches = {} content_map = {} unique_names = Counter(names) duplicate_inserts = [] inserted_names = Card.get_all_inserted_names() if post_insert_hook is None: post_insert_hook = lambda c: log.debug('inserted: {.name}', c) for name in unique_names: try: card_content = get_json_card_content(name) except CardFetchingError: invalid_names.append(name) log.debug('problem fetching card with name: %r', name) continue fetched_name = card_content['name'] if name != fetched_name: Card.add_new_mismatches({name: fetched_name}, redis_conn) relevant_mismatches[name] = fetched_name if fetched_name in inserted_names: refetched_names.append(fetched_name) continue else: # mitigation against isse #14 .e.g make a mismatch mapping # proactively to support searching with ascii names Card.add_new_mismatches({name: fetched_name}, redis_conn) if fetched_name not in content_map: content_map[fetched_name] = card_content card = Card(**card_content) try: card.save() except IntegrityError: # assume a different thread inserted it duplicate_inserts.append(card.name) card = None except: log.exception('unknown error inserting %r', card) card = None else: Card.mark_card_as_inserted(card, redis_conn) inserted_cards.append(card) finally: post_insert_hook(card) else: duplicate_inserts.append(fetched_name) refetched_cards = list(Card.objects.filter(name__in=refetched_names)) disposition = { 'inserted': inserted_cards, 'refetched': refetched_cards, 'mismatches': relevant_mismatches, 'invalid': invalid_names, } # this needs to actually account for mismatched names # if settings.DEBUG: # assertExpectatations(inserted_cards + refetched_cards + # invalid_names + duplicate_inserts, # unique_names) return disposition