def find_or_create(name): name = process_string(name) identifier = name.replace(' ', '_') try: card = Card.objects.get(identifier=identifier) except: card = Card.objects.create(identifier=identifier, name=name) return card
def find_or_create(name): name = process_string(name) identifier = name.replace(' ', '_') try: card = Card.objects.get(identifier=identifier) except: card = Card.objects.create( identifier=identifier, name=name ) return card
def find_or_create(set_number, card, card_set, rarity): set_number = process_string(set_number) rarity = Rarity.find_or_create(rarity) try: card_version = CardVersion.objects.get(set_number=set_number, rarity=rarity) except: card_version = CardVersion.objects.create(card=card, set_number=set_number, card_set=card_set, rarity=rarity) return card_version
def find_or_create(set_number, card, card_set, rarity): set_number = process_string(set_number) rarity = Rarity.find_or_create(rarity) try: card_version = CardVersion.objects.get(set_number=set_number, rarity=rarity) except: card_version = CardVersion.objects.create( card=card, set_number=set_number, card_set=card_set, rarity=rarity ) return card_version
def fetch_sets(output=output_print): step = Variable.objects.get(identifier='fetch-sets-step') # Fetch a list of sets and mark all sets for updating. if step.get() == 0: output(u' ### Fetching list of sets ### ') created = 0 response = unirest.get(API_SETS_LIST) if response.code == 200: for name in response.body: name = process_string(name) try: CardSet.objects.create(name=name) created += 1 except: pass CardSet.objects.all().update(requires_update=True) step.set(1) output(u'{:d} card sets created.'.format(created)) else: output(u'API call failed') # Fetch individual sets. elif step.get() == 1: output(u' --- Fetching individual sets --- ') limit = Variable.objects.get(identifier='fetch-sets-max').get() sets = CardSet.objects.filter(requires_update=True)[:limit] if len(sets): for card_set in sets: output(u'Fetching set {}...'.format(card_set.name)) response = unirest.get( API_SET.format(urllib.quote(card_set.name, ''))) if (response.code != 200 or response.body['status'] != 'success'): output(u'=!= Failed set {}.'.format(card_set.name)) card_set.with_language_code = True for card_source in response.body['data']['cards']: for card_version_source in card_source['numbers']: if not sn_has_language_code( card_version_source['print_tag']): card_set.with_language_code = False break if not card_set.with_language_code: break new_card_versions = {} for card_source in response.body['data']['cards']: card = Card.find_or_create(name=card_source['name']) for card_version_source in card_source['numbers']: set_number = sn_normalize( card_version_source['print_tag'], card_set.with_language_code) rarity = slugify(card_version_source['rarity']) if (set_number in new_card_versions and rarity in new_card_versions[set_number]): new_card_versions[set_number][rarity][ 'price_data'] = (combine_prices( new_card_versions[set_number][rarity] ['price_data'], card_version_source['price_data'])) else: if set_number not in new_card_versions: new_card_versions[set_number] = {} new_card_versions[set_number][rarity] = { 'card': card } new_card_versions[set_number][rarity][ 'price_data'] = ( card_version_source['price_data']) new_card_versions_pks = [] for set_number, rarities in new_card_versions.iteritems(): for rarity, data in rarities.iteritems(): card_version = CardVersion.find_or_create( set_number=set_number, card=data['card'], card_set=card_set, rarity=rarity) new_card_versions_pks.append(card_version.pk) data['card_version'] = card_version if (data['price_data'] and data['price_data']['status'] == 'success'): card_version.set_prices(data['price_data']) else: card_version.clear_prices() junk_card_versions = (CardVersion.objects.filter( card_set=card_set).exclude( pk__in=new_card_versions_pks).prefetch_related( 'user_card_versions', 'user_card_versions__user').select_related( 'rarity').distinct()) for card_version in junk_card_versions: set_number = sn_normalize(card_version.set_number, card_set.with_language_code) rarity = unicode(card_version.rarity.identifier) try: actual_card_version = new_card_versions[set_number][ rarity]['card_version'] except: try: actual_card_version = new_card_versions[ set_number].itervalues().next()['card_version'] except: card_version.dirty = True card_version.save() continue with transaction.atomic(): for item in card_version.user_card_versions.all(): try: user_card_version = ( UserCardVersion.objects.get( card_version=card_version, user=item.user)) user_card_version.have_count += item.have_count user_card_version.save() except: item.card_version = actual_card_version item.save() card_version.delete() card_set.requires_update = False card_set.save() output(u'Fetched.') else: step.set(0)
def get_wiki_data(identifier): Q = pq(url=(u'{}/{}'.format(WIKI_URL, identifier))) card = Q('.cardtable') statuses = Q('.cardtablestatuses') data = { 'image': (card.find('td.cardtable-cardimage').eq(0) .find('img').eq(0).attr('src')), 'type': ('trap' if card.find('img[alt="TRAP"]') else ('spell' if card.find('img[alt="SPELL"]') else ('monster' if card.find('th a[title="Type"]') else 'other'))), 'status_advanced': (statuses.find('th a[title="Advanced Format"]') .eq(0).parents('th').next().text()), 'status_traditional': ( statuses.find('th a[title="Traditional Format"]').eq(0) .parents('th').next().text()) } description_element = (card.find('td table table').eq(0) .find('tr').eq(2).find('td').eq(0)) description_element.html(re.sub(r'<br ?/?>', BREAK_TOKEN, description_element.html())) data['description'] = process_string(description_element.text()) data['description'] = data['description'].replace(BREAK_TOKEN, '<br />') try: data['number'] = process_string(card.find('th a[title="Card Number"]') .eq(0).parents('tr').eq(0).find('td a') .eq(0).text()) except: data['number'] = '' if (data['type'] == 'monster'): data['monster_attribute'] = (card.find('th a[title="Attribute"]') .eq(0).parents('tr').eq(0) .find('td a').eq(0).text()) try: data['monster_level'] = int(process_string( card.find('th a[title="Level"]').eq(0).parents('tr').eq(0) .find('td a').eq(0).text())) except: data['monster_level'] = int(process_string( card.find('th a[title="Rank"]').eq(0).parents('tr').eq(0) .find('td a').eq(0).text())) atk_def = (card.find('th a[title="ATK"]').eq(0) .parents('tr').eq(0).find('td').eq(0).text()).split('/') data['monster_attack'] = process_string(atk_def[0]) data['monster_defense'] = process_string(atk_def[1]) data['monster_types'] = (process_string( card.find('th a[title="Type"]').eq(0).parents('tr').eq(0) .find('td').eq(0).text())).split('/') elif (data['type'] == 'spell' or data['type'] == 'trap'): data['spell_trap_property'] = ( card.find('th a[title="Property"]').eq(0).parents('tr').eq(0) .find('td a').eq(0).text()) return data
def fetch_sets(output=output_print): step = Variable.objects.get(identifier='fetch-sets-step') # Fetch a list of sets and mark all sets for updating. if step.get() == 0: output(u' ### Fetching list of sets ### ') created = 0 response = unirest.get(API_SETS_LIST) if response.code == 200: for name in response.body: name = process_string(name) try: CardSet.objects.create(name=name) created += 1 except: pass CardSet.objects.all().update(requires_update=True) step.set(1) output(u'{:d} card sets created.'.format(created)) else: output(u'API call failed') # Fetch individual sets. elif step.get() == 1: output(u' --- Fetching individual sets --- ') limit = Variable.objects.get(identifier='fetch-sets-max').get() sets = CardSet.objects.filter(requires_update=True)[:limit] if len(sets): for card_set in sets: output(u'Fetching set {}...'.format(card_set.name)) response = unirest.get( API_SET.format(urllib.quote(card_set.name, ''))) if (response.code != 200 or response.body['status'] != 'success'): output(u'=!= Failed set {}.'.format(card_set.name)) card_set.with_language_code = True for card_source in response.body['data']['cards']: for card_version_source in card_source['numbers']: if not sn_has_language_code( card_version_source['print_tag']): card_set.with_language_code = False break if not card_set.with_language_code: break new_card_versions = {} for card_source in response.body['data']['cards']: card = Card.find_or_create( name=card_source['name'] ) for card_version_source in card_source['numbers']: set_number = sn_normalize( card_version_source['print_tag'], card_set.with_language_code ) rarity = slugify(card_version_source['rarity']) if (set_number in new_card_versions and rarity in new_card_versions[ set_number]): new_card_versions[set_number][rarity][ 'price_data'] = (combine_prices( new_card_versions[ set_number][rarity]['price_data'], card_version_source['price_data'])) else: if set_number not in new_card_versions: new_card_versions[set_number] = {} new_card_versions[set_number][rarity] = { 'card': card } new_card_versions[set_number][ rarity]['price_data'] = ( card_version_source['price_data']) new_card_versions_pks = [] for set_number, rarities in new_card_versions.iteritems(): for rarity, data in rarities.iteritems(): card_version = CardVersion.find_or_create( set_number=set_number, card=data['card'], card_set=card_set, rarity=rarity ) new_card_versions_pks.append(card_version.pk) data['card_version'] = card_version if (data['price_data'] and data['price_data']['status'] == 'success'): card_version.set_prices(data['price_data']) else: card_version.clear_prices() junk_card_versions = ( CardVersion.objects .filter(card_set=card_set) .exclude(pk__in=new_card_versions_pks) .prefetch_related('user_card_versions', 'user_card_versions__user') .select_related('rarity') .distinct()) for card_version in junk_card_versions: set_number = sn_normalize( card_version.set_number, card_set.with_language_code ) rarity = unicode(card_version.rarity.identifier) try: actual_card_version = new_card_versions[set_number][ rarity]['card_version'] except: try: actual_card_version = new_card_versions[ set_number].itervalues().next()['card_version'] except: card_version.dirty = True card_version.save() continue with transaction.atomic(): for item in card_version.user_card_versions.all(): try: user_card_version = ( UserCardVersion.objects .get(card_version=card_version, user=item.user)) user_card_version.have_count += item.have_count user_card_version.save() except: item.card_version = actual_card_version item.save() card_version.delete() card_set.requires_update = False card_set.save() output(u'Fetched.') else: step.set(0)