Example #1
0
    def find_or_create(name):
        name = process_string(name)
        identifier = name.replace(' ', '_')

        try:
            card = Card.objects.get(identifier=identifier)
        except:
            card = Card.objects.create(identifier=identifier, name=name)

        return card
Example #2
0
    def find_or_create(name):
        name = process_string(name)
        identifier = name.replace(' ', '_')

        try:
            card = Card.objects.get(identifier=identifier)
        except:
            card = Card.objects.create(
                identifier=identifier,
                name=name
            )

        return card
Example #3
0
    def find_or_create(set_number, card, card_set, rarity):
        set_number = process_string(set_number)
        rarity = Rarity.find_or_create(rarity)

        try:
            card_version = CardVersion.objects.get(set_number=set_number,
                                                   rarity=rarity)
        except:
            card_version = CardVersion.objects.create(card=card,
                                                      set_number=set_number,
                                                      card_set=card_set,
                                                      rarity=rarity)

        return card_version
Example #4
0
    def find_or_create(set_number, card, card_set, rarity):
        set_number = process_string(set_number)
        rarity = Rarity.find_or_create(rarity)

        try:
            card_version = CardVersion.objects.get(set_number=set_number,
                                                   rarity=rarity)
        except:
            card_version = CardVersion.objects.create(
                card=card,
                set_number=set_number,
                card_set=card_set,
                rarity=rarity
            )

        return card_version
Example #5
0
def fetch_sets(output=output_print):
    step = Variable.objects.get(identifier='fetch-sets-step')

    # Fetch a list of sets and mark all sets for updating.
    if step.get() == 0:
        output(u' ### Fetching list of sets ### ')
        created = 0

        response = unirest.get(API_SETS_LIST)

        if response.code == 200:

            for name in response.body:
                name = process_string(name)

                try:
                    CardSet.objects.create(name=name)
                    created += 1
                except:
                    pass

            CardSet.objects.all().update(requires_update=True)

            step.set(1)

            output(u'{:d} card sets created.'.format(created))

        else:
            output(u'API call failed')

    # Fetch individual sets.
    elif step.get() == 1:
        output(u' --- Fetching individual sets --- ')
        limit = Variable.objects.get(identifier='fetch-sets-max').get()
        sets = CardSet.objects.filter(requires_update=True)[:limit]

        if len(sets):
            for card_set in sets:
                output(u'Fetching set {}...'.format(card_set.name))

                response = unirest.get(
                    API_SET.format(urllib.quote(card_set.name, '')))

                if (response.code != 200
                        or response.body['status'] != 'success'):
                    output(u'=!= Failed set {}.'.format(card_set.name))

                card_set.with_language_code = True

                for card_source in response.body['data']['cards']:
                    for card_version_source in card_source['numbers']:
                        if not sn_has_language_code(
                                card_version_source['print_tag']):
                            card_set.with_language_code = False
                            break

                    if not card_set.with_language_code:
                        break

                new_card_versions = {}

                for card_source in response.body['data']['cards']:
                    card = Card.find_or_create(name=card_source['name'])

                    for card_version_source in card_source['numbers']:
                        set_number = sn_normalize(
                            card_version_source['print_tag'],
                            card_set.with_language_code)
                        rarity = slugify(card_version_source['rarity'])

                        if (set_number in new_card_versions
                                and rarity in new_card_versions[set_number]):
                            new_card_versions[set_number][rarity][
                                'price_data'] = (combine_prices(
                                    new_card_versions[set_number][rarity]
                                    ['price_data'],
                                    card_version_source['price_data']))
                        else:
                            if set_number not in new_card_versions:
                                new_card_versions[set_number] = {}

                            new_card_versions[set_number][rarity] = {
                                'card': card
                            }

                            new_card_versions[set_number][rarity][
                                'price_data'] = (
                                    card_version_source['price_data'])

                new_card_versions_pks = []

                for set_number, rarities in new_card_versions.iteritems():
                    for rarity, data in rarities.iteritems():
                        card_version = CardVersion.find_or_create(
                            set_number=set_number,
                            card=data['card'],
                            card_set=card_set,
                            rarity=rarity)

                        new_card_versions_pks.append(card_version.pk)

                        data['card_version'] = card_version

                        if (data['price_data']
                                and data['price_data']['status'] == 'success'):
                            card_version.set_prices(data['price_data'])
                        else:
                            card_version.clear_prices()

                junk_card_versions = (CardVersion.objects.filter(
                    card_set=card_set).exclude(
                        pk__in=new_card_versions_pks).prefetch_related(
                            'user_card_versions',
                            'user_card_versions__user').select_related(
                                'rarity').distinct())

                for card_version in junk_card_versions:
                    set_number = sn_normalize(card_version.set_number,
                                              card_set.with_language_code)
                    rarity = unicode(card_version.rarity.identifier)

                    try:
                        actual_card_version = new_card_versions[set_number][
                            rarity]['card_version']
                    except:
                        try:
                            actual_card_version = new_card_versions[
                                set_number].itervalues().next()['card_version']
                        except:
                            card_version.dirty = True
                            card_version.save()
                            continue

                    with transaction.atomic():
                        for item in card_version.user_card_versions.all():
                            try:
                                user_card_version = (
                                    UserCardVersion.objects.get(
                                        card_version=card_version,
                                        user=item.user))
                                user_card_version.have_count += item.have_count
                                user_card_version.save()
                            except:
                                item.card_version = actual_card_version
                                item.save()
                    card_version.delete()

                card_set.requires_update = False
                card_set.save()

                output(u'Fetched.')
        else:
            step.set(0)
Example #6
0
def get_wiki_data(identifier):

    Q = pq(url=(u'{}/{}'.format(WIKI_URL, identifier)))

    card = Q('.cardtable')
    statuses = Q('.cardtablestatuses')

    data = {
        'image': (card.find('td.cardtable-cardimage').eq(0)
                  .find('img').eq(0).attr('src')),
        'type': ('trap' if card.find('img[alt="TRAP"]') else
                 ('spell' if card.find('img[alt="SPELL"]') else
                  ('monster' if card.find('th a[title="Type"]') else
                   'other'))),
        'status_advanced': (statuses.find('th a[title="Advanced Format"]')
                            .eq(0).parents('th').next().text()),
        'status_traditional': (
            statuses.find('th a[title="Traditional Format"]').eq(0)
            .parents('th').next().text())
    }

    description_element = (card.find('td table table').eq(0)
                           .find('tr').eq(2).find('td').eq(0))

    description_element.html(re.sub(r'<br ?/?>', BREAK_TOKEN,
                                    description_element.html()))

    data['description'] = process_string(description_element.text())

    data['description'] = data['description'].replace(BREAK_TOKEN, '<br />')

    try:
        data['number'] = process_string(card.find('th a[title="Card Number"]')
                                        .eq(0).parents('tr').eq(0).find('td a')
                                        .eq(0).text())
    except:
        data['number'] = ''

    if (data['type'] == 'monster'):
        data['monster_attribute'] = (card.find('th a[title="Attribute"]')
                                     .eq(0).parents('tr').eq(0)
                                     .find('td a').eq(0).text())

        try:
            data['monster_level'] = int(process_string(
                card.find('th a[title="Level"]').eq(0).parents('tr').eq(0)
                .find('td a').eq(0).text()))
        except:
            data['monster_level'] = int(process_string(
                card.find('th a[title="Rank"]').eq(0).parents('tr').eq(0)
                .find('td a').eq(0).text()))

        atk_def = (card.find('th a[title="ATK"]').eq(0)
                   .parents('tr').eq(0).find('td').eq(0).text()).split('/')

        data['monster_attack'] = process_string(atk_def[0])
        data['monster_defense'] = process_string(atk_def[1])

        data['monster_types'] = (process_string(
            card.find('th a[title="Type"]').eq(0).parents('tr').eq(0)
            .find('td').eq(0).text())).split('/')

    elif (data['type'] == 'spell' or data['type'] == 'trap'):
        data['spell_trap_property'] = (
            card.find('th a[title="Property"]').eq(0).parents('tr').eq(0)
            .find('td a').eq(0).text())

    return data
Example #7
0
def fetch_sets(output=output_print):
    step = Variable.objects.get(identifier='fetch-sets-step')

    # Fetch a list of sets and mark all sets for updating.
    if step.get() == 0:
        output(u' ### Fetching list of sets ### ')
        created = 0

        response = unirest.get(API_SETS_LIST)

        if response.code == 200:

            for name in response.body:
                name = process_string(name)

                try:
                    CardSet.objects.create(name=name)
                    created += 1
                except:
                    pass

            CardSet.objects.all().update(requires_update=True)

            step.set(1)

            output(u'{:d} card sets created.'.format(created))

        else:
            output(u'API call failed')

    # Fetch individual sets.
    elif step.get() == 1:
        output(u' --- Fetching individual sets --- ')
        limit = Variable.objects.get(identifier='fetch-sets-max').get()
        sets = CardSet.objects.filter(requires_update=True)[:limit]

        if len(sets):
            for card_set in sets:
                output(u'Fetching set {}...'.format(card_set.name))

                response = unirest.get(
                    API_SET.format(urllib.quote(card_set.name, '')))

                if (response.code != 200
                        or response.body['status'] != 'success'):
                    output(u'=!= Failed set {}.'.format(card_set.name))

                card_set.with_language_code = True

                for card_source in response.body['data']['cards']:
                    for card_version_source in card_source['numbers']:
                        if not sn_has_language_code(
                                card_version_source['print_tag']):
                            card_set.with_language_code = False
                            break

                    if not card_set.with_language_code:
                        break

                new_card_versions = {}

                for card_source in response.body['data']['cards']:
                    card = Card.find_or_create(
                        name=card_source['name']
                    )

                    for card_version_source in card_source['numbers']:
                        set_number = sn_normalize(
                            card_version_source['print_tag'],
                            card_set.with_language_code
                        )
                        rarity = slugify(card_version_source['rarity'])

                        if (set_number in new_card_versions and
                                rarity in new_card_versions[
                                    set_number]):
                            new_card_versions[set_number][rarity][
                                'price_data'] = (combine_prices(
                                    new_card_versions[
                                        set_number][rarity]['price_data'],
                                    card_version_source['price_data']))
                        else:
                            if set_number not in new_card_versions:
                                new_card_versions[set_number] = {}

                            new_card_versions[set_number][rarity] = {
                                'card': card
                            }

                            new_card_versions[set_number][
                                rarity]['price_data'] = (
                                    card_version_source['price_data'])

                new_card_versions_pks = []

                for set_number, rarities in new_card_versions.iteritems():
                    for rarity, data in rarities.iteritems():
                        card_version = CardVersion.find_or_create(
                            set_number=set_number,
                            card=data['card'],
                            card_set=card_set,
                            rarity=rarity
                        )

                        new_card_versions_pks.append(card_version.pk)

                        data['card_version'] = card_version

                        if (data['price_data'] and
                                data['price_data']['status'] == 'success'):
                            card_version.set_prices(data['price_data'])
                        else:
                            card_version.clear_prices()

                junk_card_versions = (
                    CardVersion.objects
                    .filter(card_set=card_set)
                    .exclude(pk__in=new_card_versions_pks)
                    .prefetch_related('user_card_versions',
                                      'user_card_versions__user')
                    .select_related('rarity')
                    .distinct())

                for card_version in junk_card_versions:
                    set_number = sn_normalize(
                        card_version.set_number,
                        card_set.with_language_code
                    )
                    rarity = unicode(card_version.rarity.identifier)

                    try:
                        actual_card_version = new_card_versions[set_number][
                            rarity]['card_version']
                    except:
                        try:
                            actual_card_version = new_card_versions[
                                set_number].itervalues().next()['card_version']
                        except:
                            card_version.dirty = True
                            card_version.save()
                            continue

                    with transaction.atomic():
                        for item in card_version.user_card_versions.all():
                            try:
                                user_card_version = (
                                    UserCardVersion.objects
                                    .get(card_version=card_version,
                                         user=item.user))
                                user_card_version.have_count += item.have_count
                                user_card_version.save()
                            except:
                                item.card_version = actual_card_version
                                item.save()
                    card_version.delete()

                card_set.requires_update = False
                card_set.save()

                output(u'Fetched.')
        else:
            step.set(0)