def fetch_sets(output=output_print):
step = Variable.objects.get(identifier='fetch-sets-step')
# Fetch a list of sets and mark all sets for updating.
if step.get() == 0:
output(u' ### Fetching list of sets ### ')
created = 0
response = unirest.get(API_SETS_LIST)
if response.code == 200:
for name in response.body:
name = process_string(name)
try:
CardSet.objects.create(name=name)
created += 1
except:
pass
CardSet.objects.all().update(requires_update=True)
step.set(1)
output(u'{:d} card sets created.'.format(created))
else:
output(u'API call failed')
# Fetch individual sets.
elif step.get() == 1:
output(u' --- Fetching individual sets --- ')
limit = Variable.objects.get(identifier='fetch-sets-max').get()
sets = CardSet.objects.filter(requires_update=True)[:limit]
if len(sets):
for card_set in sets:
output(u'Fetching set {}...'.format(card_set.name))
response = unirest.get(
API_SET.format(urllib.quote(card_set.name, '')))
if (response.code != 200
or response.body['status'] != 'success'):
output(u'=!= Failed set {}.'.format(card_set.name))
card_set.with_language_code = True
for card_source in response.body['data']['cards']:
for card_version_source in card_source['numbers']:
if not sn_has_language_code(
card_version_source['print_tag']):
card_set.with_language_code = False
break
if not card_set.with_language_code:
break
new_card_versions = {}
for card_source in response.body['data']['cards']:
card = Card.find_or_create(name=card_source['name'])
for card_version_source in card_source['numbers']:
set_number = sn_normalize(
card_version_source['print_tag'],
card_set.with_language_code)
rarity = slugify(card_version_source['rarity'])
if (set_number in new_card_versions
and rarity in new_card_versions[set_number]):
new_card_versions[set_number][rarity][
'price_data'] = (combine_prices(
new_card_versions[set_number][rarity]
['price_data'],
card_version_source['price_data']))
else:
if set_number not in new_card_versions:
new_card_versions[set_number] = {}
new_card_versions[set_number][rarity] = {
'card': card
}
new_card_versions[set_number][rarity][
'price_data'] = (
card_version_source['price_data'])
new_card_versions_pks = []
for set_number, rarities in new_card_versions.iteritems():
for rarity, data in rarities.iteritems():
card_version = CardVersion.find_or_create(
set_number=set_number,
card=data['card'],
card_set=card_set,
rarity=rarity)
new_card_versions_pks.append(card_version.pk)
data['card_version'] = card_version
if (data['price_data']
and data['price_data']['status'] == 'success'):
card_version.set_prices(data['price_data'])
else:
card_version.clear_prices()
junk_card_versions = (CardVersion.objects.filter(
card_set=card_set).exclude(
pk__in=new_card_versions_pks).prefetch_related(
'user_card_versions',
'user_card_versions__user').select_related(
'rarity').distinct())
for card_version in junk_card_versions:
set_number = sn_normalize(card_version.set_number,
card_set.with_language_code)
rarity = unicode(card_version.rarity.identifier)
try:
actual_card_version = new_card_versions[set_number][
rarity]['card_version']
except:
try:
actual_card_version = new_card_versions[
set_number].itervalues().next()['card_version']
except:
card_version.dirty = True
card_version.save()
continue
with transaction.atomic():
for item in card_version.user_card_versions.all():
try:
user_card_version = (
UserCardVersion.objects.get(
card_version=card_version,
user=item.user))
user_card_version.have_count += item.have_count
user_card_version.save()
except:
item.card_version = actual_card_version
item.save()
card_version.delete()
card_set.requires_update = False
card_set.save()
output(u'Fetched.')
else:
step.set(0)
def post_upload(self, request, *args, **kwargs):
form = self.form_upload
if form.is_valid():
mode = form.cleaned_data['mode']
invalid_set_numbers = set()
set_numbers = set()
set_numbers_data = {}
for row in csv.reader(self.request.FILES['source']):
if len(row) and row[0]:
set_number = row[0].upper()
try:
count = abs(int(row[1]))
except:
count = 1
try:
rarity = Rarity(identifier=slugify(row[2])).identifier
except:
rarity = None
if mode == UploadForm.MODE_SUBSTRACT:
count *= -1
# Check if the input is a valid set number.
if re.match(r'\w+-[A-Z]*[0-9]+', set_number) is not None:
# If it's a foreign set number, change it to EN.
if (re.search(r'-[A-Z]{1,2}', set_number) is not None
and set_number.find('-EN') == -1):
set_number = re.sub(
r'-[A-Z]{1,2}', '-EN', set_number)
set_numbers.add(set_number)
if (set_number in set_numbers_data):
if (rarity in set_numbers_data[set_number]):
set_numbers_data[set_number][rarity] += count
else:
set_numbers_data[set_number][rarity] = count
else:
set_numbers_data[set_number] = {rarity: count}
else:
invalid_set_numbers.add(set_number)
card_versions = (CardVersion.objects
.only('id', 'set_number', 'card__name',
'rarity__identifier')
.select_related('card', 'rarity')
.filter(set_number__in=set_numbers)
.distinct()
.order_by('set_number'))
card_versions_by_rarities = {}
card_versions_rarities = {}
for card_version in card_versions:
if (card_version.set_number not in card_versions_by_rarities):
card_versions_by_rarities[card_version.set_number] = {}
card_versions_rarities[card_version.set_number] = {}
rarity = card_version.rarity
card_versions_by_rarities[card_version.set_number][
card_version.rarity.identifier] = card_version
card_versions_rarities[card_version.set_number][
rarity.identifier] = {
'card_version_pk': card_version.pk,
'name': rarity.name
}
not_found_set_numbers = {}
not_found_set_numbers_total = 0
not_found_rarities = {}
valid_set_numbers = {}
valid_set_numbers_total = 0
for set_number, data in set_numbers_data.iteritems():
if set_number in card_versions_by_rarities:
valid_set_numbers[set_number] = {}
current_total = 0
if len(card_versions_by_rarities[set_number]) == 1:
data = {
(card_versions_by_rarities[set_number]
.iterkeys().next()): (
sum([c for r, c in data.iteritems()]))
}
found = True
for rarity, count in data.iteritems():
if rarity in card_versions_by_rarities[set_number]:
card_version = (
card_versions_by_rarities[set_number][rarity])
valid_set_numbers[set_number][rarity] = {
'count': count,
'name': card_version.card.name,
'pk': card_version.pk,
'rarity_name': card_versions_rarities[
set_number][rarity]['name']
}
current_total += count
else:
found = False
break
if not found:
del valid_set_numbers[set_number]
not_found_rarities[set_number] = {
'count': sum([c for r, c in data.iteritems()]),
'name': (card_versions_by_rarities[set_number]
.itervalues().next().card.name),
'rarities': card_versions_rarities[
set_number]
}
else:
valid_set_numbers_total += current_total
else:
not_found_set_numbers[set_number] = sum(
[c for r, c in data.iteritems()])
not_found_set_numbers_total += (
not_found_set_numbers[set_number])
import_data = self.get_import_data()
if len(not_found_rarities) > 0:
import_data.step = ImportData.STEP_RARITIES
else:
import_data.step = ImportData.STEP_CONFIRM
import_data.set_data({
'mode': mode,
'invalid_set_numbers': list(invalid_set_numbers),
'invalid_set_numbers_count': len(invalid_set_numbers),
'valid_set_numbers': valid_set_numbers,
'valid_set_numbers_total': valid_set_numbers_total,
'all_valid_set_numbers': valid_set_numbers,
'all_valid_set_numbers_total': valid_set_numbers_total,
'not_found_set_numbers': not_found_set_numbers,
'not_found_set_numbers_total': not_found_set_numbers_total,
'not_found_rarities': not_found_rarities
})
return redirect('import', step=ImportData.STEP_IDENTIFIERS[
import_data.step])
def fetch_sets(output=output_print):
step = Variable.objects.get(identifier='fetch-sets-step')
# Fetch a list of sets and mark all sets for updating.
if step.get() == 0:
output(u' ### Fetching list of sets ### ')
created = 0
response = unirest.get(API_SETS_LIST)
if response.code == 200:
for name in response.body:
name = process_string(name)
try:
CardSet.objects.create(name=name)
created += 1
except:
pass
CardSet.objects.all().update(requires_update=True)
step.set(1)
output(u'{:d} card sets created.'.format(created))
else:
output(u'API call failed')
# Fetch individual sets.
elif step.get() == 1:
output(u' --- Fetching individual sets --- ')
limit = Variable.objects.get(identifier='fetch-sets-max').get()
sets = CardSet.objects.filter(requires_update=True)[:limit]
if len(sets):
for card_set in sets:
output(u'Fetching set {}...'.format(card_set.name))
response = unirest.get(
API_SET.format(urllib.quote(card_set.name, '')))
if (response.code != 200
or response.body['status'] != 'success'):
output(u'=!= Failed set {}.'.format(card_set.name))
card_set.with_language_code = True
for card_source in response.body['data']['cards']:
for card_version_source in card_source['numbers']:
if not sn_has_language_code(
card_version_source['print_tag']):
card_set.with_language_code = False
break
if not card_set.with_language_code:
break
new_card_versions = {}
for card_source in response.body['data']['cards']:
card = Card.find_or_create(
name=card_source['name']
)
for card_version_source in card_source['numbers']:
set_number = sn_normalize(
card_version_source['print_tag'],
card_set.with_language_code
)
rarity = slugify(card_version_source['rarity'])
if (set_number in new_card_versions and
rarity in new_card_versions[
set_number]):
new_card_versions[set_number][rarity][
'price_data'] = (combine_prices(
new_card_versions[
set_number][rarity]['price_data'],
card_version_source['price_data']))
else:
if set_number not in new_card_versions:
new_card_versions[set_number] = {}
new_card_versions[set_number][rarity] = {
'card': card
}
new_card_versions[set_number][
rarity]['price_data'] = (
card_version_source['price_data'])
new_card_versions_pks = []
for set_number, rarities in new_card_versions.iteritems():
for rarity, data in rarities.iteritems():
card_version = CardVersion.find_or_create(
set_number=set_number,
card=data['card'],
card_set=card_set,
rarity=rarity
)
new_card_versions_pks.append(card_version.pk)
data['card_version'] = card_version
if (data['price_data'] and
data['price_data']['status'] == 'success'):
card_version.set_prices(data['price_data'])
else:
card_version.clear_prices()
junk_card_versions = (
CardVersion.objects
.filter(card_set=card_set)
.exclude(pk__in=new_card_versions_pks)
.prefetch_related('user_card_versions',
'user_card_versions__user')
.select_related('rarity')
.distinct())
for card_version in junk_card_versions:
set_number = sn_normalize(
card_version.set_number,
card_set.with_language_code
)
rarity = unicode(card_version.rarity.identifier)
try:
actual_card_version = new_card_versions[set_number][
rarity]['card_version']
except:
try:
actual_card_version = new_card_versions[
set_number].itervalues().next()['card_version']
except:
card_version.dirty = True
card_version.save()
continue
with transaction.atomic():
for item in card_version.user_card_versions.all():
try:
user_card_version = (
UserCardVersion.objects
.get(card_version=card_version,
user=item.user))
user_card_version.have_count += item.have_count
user_card_version.save()
except:
item.card_version = actual_card_version
item.save()
card_version.delete()
card_set.requires_update = False
card_set.save()
output(u'Fetched.')
else:
step.set(0)
