def fetch_sets(output=output_print): step = Variable.objects.get(identifier='fetch-sets-step') # Fetch a list of sets and mark all sets for updating. if step.get() == 0: output(u' ### Fetching list of sets ### ') created = 0 response = unirest.get(API_SETS_LIST) if response.code == 200: for name in response.body: name = process_string(name) try: CardSet.objects.create(name=name) created += 1 except: pass CardSet.objects.all().update(requires_update=True) step.set(1) output(u'{:d} card sets created.'.format(created)) else: output(u'API call failed') # Fetch individual sets. elif step.get() == 1: output(u' --- Fetching individual sets --- ') limit = Variable.objects.get(identifier='fetch-sets-max').get() sets = CardSet.objects.filter(requires_update=True)[:limit] if len(sets): for card_set in sets: output(u'Fetching set {}...'.format(card_set.name)) response = unirest.get( API_SET.format(urllib.quote(card_set.name, ''))) if (response.code != 200 or response.body['status'] != 'success'): output(u'=!= Failed set {}.'.format(card_set.name)) card_set.with_language_code = True for card_source in response.body['data']['cards']: for card_version_source in card_source['numbers']: if not sn_has_language_code( card_version_source['print_tag']): card_set.with_language_code = False break if not card_set.with_language_code: break new_card_versions = {} for card_source in response.body['data']['cards']: card = Card.find_or_create(name=card_source['name']) for card_version_source in card_source['numbers']: set_number = sn_normalize( card_version_source['print_tag'], card_set.with_language_code) rarity = slugify(card_version_source['rarity']) if (set_number in new_card_versions and rarity in new_card_versions[set_number]): new_card_versions[set_number][rarity][ 'price_data'] = (combine_prices( new_card_versions[set_number][rarity] ['price_data'], card_version_source['price_data'])) else: if set_number not in new_card_versions: new_card_versions[set_number] = {} new_card_versions[set_number][rarity] = { 'card': card } new_card_versions[set_number][rarity][ 'price_data'] = ( card_version_source['price_data']) new_card_versions_pks = [] for set_number, rarities in new_card_versions.iteritems(): for rarity, data in rarities.iteritems(): card_version = CardVersion.find_or_create( set_number=set_number, card=data['card'], card_set=card_set, rarity=rarity) new_card_versions_pks.append(card_version.pk) data['card_version'] = card_version if (data['price_data'] and data['price_data']['status'] == 'success'): card_version.set_prices(data['price_data']) else: card_version.clear_prices() junk_card_versions = (CardVersion.objects.filter( card_set=card_set).exclude( pk__in=new_card_versions_pks).prefetch_related( 'user_card_versions', 'user_card_versions__user').select_related( 'rarity').distinct()) for card_version in junk_card_versions: set_number = sn_normalize(card_version.set_number, card_set.with_language_code) rarity = unicode(card_version.rarity.identifier) try: actual_card_version = new_card_versions[set_number][ rarity]['card_version'] except: try: actual_card_version = new_card_versions[ set_number].itervalues().next()['card_version'] except: card_version.dirty = True card_version.save() continue with transaction.atomic(): for item in card_version.user_card_versions.all(): try: user_card_version = ( UserCardVersion.objects.get( card_version=card_version, user=item.user)) user_card_version.have_count += item.have_count user_card_version.save() except: item.card_version = actual_card_version item.save() card_version.delete() card_set.requires_update = False card_set.save() output(u'Fetched.') else: step.set(0)
def post_upload(self, request, *args, **kwargs): form = self.form_upload if form.is_valid(): mode = form.cleaned_data['mode'] invalid_set_numbers = set() set_numbers = set() set_numbers_data = {} for row in csv.reader(self.request.FILES['source']): if len(row) and row[0]: set_number = row[0].upper() try: count = abs(int(row[1])) except: count = 1 try: rarity = Rarity(identifier=slugify(row[2])).identifier except: rarity = None if mode == UploadForm.MODE_SUBSTRACT: count *= -1 # Check if the input is a valid set number. if re.match(r'\w+-[A-Z]*[0-9]+', set_number) is not None: # If it's a foreign set number, change it to EN. if (re.search(r'-[A-Z]{1,2}', set_number) is not None and set_number.find('-EN') == -1): set_number = re.sub( r'-[A-Z]{1,2}', '-EN', set_number) set_numbers.add(set_number) if (set_number in set_numbers_data): if (rarity in set_numbers_data[set_number]): set_numbers_data[set_number][rarity] += count else: set_numbers_data[set_number][rarity] = count else: set_numbers_data[set_number] = {rarity: count} else: invalid_set_numbers.add(set_number) card_versions = (CardVersion.objects .only('id', 'set_number', 'card__name', 'rarity__identifier') .select_related('card', 'rarity') .filter(set_number__in=set_numbers) .distinct() .order_by('set_number')) card_versions_by_rarities = {} card_versions_rarities = {} for card_version in card_versions: if (card_version.set_number not in card_versions_by_rarities): card_versions_by_rarities[card_version.set_number] = {} card_versions_rarities[card_version.set_number] = {} rarity = card_version.rarity card_versions_by_rarities[card_version.set_number][ card_version.rarity.identifier] = card_version card_versions_rarities[card_version.set_number][ rarity.identifier] = { 'card_version_pk': card_version.pk, 'name': rarity.name } not_found_set_numbers = {} not_found_set_numbers_total = 0 not_found_rarities = {} valid_set_numbers = {} valid_set_numbers_total = 0 for set_number, data in set_numbers_data.iteritems(): if set_number in card_versions_by_rarities: valid_set_numbers[set_number] = {} current_total = 0 if len(card_versions_by_rarities[set_number]) == 1: data = { (card_versions_by_rarities[set_number] .iterkeys().next()): ( sum([c for r, c in data.iteritems()])) } found = True for rarity, count in data.iteritems(): if rarity in card_versions_by_rarities[set_number]: card_version = ( card_versions_by_rarities[set_number][rarity]) valid_set_numbers[set_number][rarity] = { 'count': count, 'name': card_version.card.name, 'pk': card_version.pk, 'rarity_name': card_versions_rarities[ set_number][rarity]['name'] } current_total += count else: found = False break if not found: del valid_set_numbers[set_number] not_found_rarities[set_number] = { 'count': sum([c for r, c in data.iteritems()]), 'name': (card_versions_by_rarities[set_number] .itervalues().next().card.name), 'rarities': card_versions_rarities[ set_number] } else: valid_set_numbers_total += current_total else: not_found_set_numbers[set_number] = sum( [c for r, c in data.iteritems()]) not_found_set_numbers_total += ( not_found_set_numbers[set_number]) import_data = self.get_import_data() if len(not_found_rarities) > 0: import_data.step = ImportData.STEP_RARITIES else: import_data.step = ImportData.STEP_CONFIRM import_data.set_data({ 'mode': mode, 'invalid_set_numbers': list(invalid_set_numbers), 'invalid_set_numbers_count': len(invalid_set_numbers), 'valid_set_numbers': valid_set_numbers, 'valid_set_numbers_total': valid_set_numbers_total, 'all_valid_set_numbers': valid_set_numbers, 'all_valid_set_numbers_total': valid_set_numbers_total, 'not_found_set_numbers': not_found_set_numbers, 'not_found_set_numbers_total': not_found_set_numbers_total, 'not_found_rarities': not_found_rarities }) return redirect('import', step=ImportData.STEP_IDENTIFIERS[ import_data.step])
def fetch_sets(output=output_print): step = Variable.objects.get(identifier='fetch-sets-step') # Fetch a list of sets and mark all sets for updating. if step.get() == 0: output(u' ### Fetching list of sets ### ') created = 0 response = unirest.get(API_SETS_LIST) if response.code == 200: for name in response.body: name = process_string(name) try: CardSet.objects.create(name=name) created += 1 except: pass CardSet.objects.all().update(requires_update=True) step.set(1) output(u'{:d} card sets created.'.format(created)) else: output(u'API call failed') # Fetch individual sets. elif step.get() == 1: output(u' --- Fetching individual sets --- ') limit = Variable.objects.get(identifier='fetch-sets-max').get() sets = CardSet.objects.filter(requires_update=True)[:limit] if len(sets): for card_set in sets: output(u'Fetching set {}...'.format(card_set.name)) response = unirest.get( API_SET.format(urllib.quote(card_set.name, ''))) if (response.code != 200 or response.body['status'] != 'success'): output(u'=!= Failed set {}.'.format(card_set.name)) card_set.with_language_code = True for card_source in response.body['data']['cards']: for card_version_source in card_source['numbers']: if not sn_has_language_code( card_version_source['print_tag']): card_set.with_language_code = False break if not card_set.with_language_code: break new_card_versions = {} for card_source in response.body['data']['cards']: card = Card.find_or_create( name=card_source['name'] ) for card_version_source in card_source['numbers']: set_number = sn_normalize( card_version_source['print_tag'], card_set.with_language_code ) rarity = slugify(card_version_source['rarity']) if (set_number in new_card_versions and rarity in new_card_versions[ set_number]): new_card_versions[set_number][rarity][ 'price_data'] = (combine_prices( new_card_versions[ set_number][rarity]['price_data'], card_version_source['price_data'])) else: if set_number not in new_card_versions: new_card_versions[set_number] = {} new_card_versions[set_number][rarity] = { 'card': card } new_card_versions[set_number][ rarity]['price_data'] = ( card_version_source['price_data']) new_card_versions_pks = [] for set_number, rarities in new_card_versions.iteritems(): for rarity, data in rarities.iteritems(): card_version = CardVersion.find_or_create( set_number=set_number, card=data['card'], card_set=card_set, rarity=rarity ) new_card_versions_pks.append(card_version.pk) data['card_version'] = card_version if (data['price_data'] and data['price_data']['status'] == 'success'): card_version.set_prices(data['price_data']) else: card_version.clear_prices() junk_card_versions = ( CardVersion.objects .filter(card_set=card_set) .exclude(pk__in=new_card_versions_pks) .prefetch_related('user_card_versions', 'user_card_versions__user') .select_related('rarity') .distinct()) for card_version in junk_card_versions: set_number = sn_normalize( card_version.set_number, card_set.with_language_code ) rarity = unicode(card_version.rarity.identifier) try: actual_card_version = new_card_versions[set_number][ rarity]['card_version'] except: try: actual_card_version = new_card_versions[ set_number].itervalues().next()['card_version'] except: card_version.dirty = True card_version.save() continue with transaction.atomic(): for item in card_version.user_card_versions.all(): try: user_card_version = ( UserCardVersion.objects .get(card_version=card_version, user=item.user)) user_card_version.have_count += item.have_count user_card_version.save() except: item.card_version = actual_card_version item.save() card_version.delete() card_set.requires_update = False card_set.save() output(u'Fetched.') else: step.set(0)