Esempio n. 1
0
def upload_card_set(request):
    if request.method == 'POST':
        form = UploadCardSetForm(request.POST, request.FILES)
        if form.is_valid():
            # Read cards file
            try:
                cards = Mnemosyne2Cards.read(request.FILES['file'])
            except Exception:
                # FIXME be more specific about what to catch
                cards = None

            #print >>sys.stderr, cards
            if cards:
                # Create new card set
                card_set = CardSet(name=form.cleaned_data['name'])
                card_set.save()

                # Create cards
                for question, answer in cards:
                    card = Card(card_set=card_set, question=question, answer=answer)
                    card.save()

        # TODO: provide feedback for invalid uploads
    else:
        form = UploadCardSetForm()
    return render_to_response('upload.html',
            {'form': form},
            context_instance=RequestContext(request))
Esempio n. 2
0
    def handle(self, *args, **options):
        self.stdout.write("start scraping from Bulbapedia...\n")
        p = re.compile("official (count|total) (?P<count>\d+)")

        CardSet.objects.all().delete()

        html = json.load(urlopen(
            BASE_URL.format(CARDSET_PAGE)))['parse']['text']['*']
        set_tbl = BeautifulSoup(html).find("table").find("table").find_all(
            "tr")[1:]

        for r in set_tbl:
            data = r.find_all("td")
            logo_temp = NamedTemporaryFile()

            try:
                eng_cs = CardSet(country="GB")
                eng_cs.set_no = int(r.find("th").text.strip())
                eng_cs.name = data[2].text.strip()
                try:
                    logo_url = data[1].a.img['src']
                    logo_temp.write(urlopen(logo_url).read())
                    logo_ext = urlparse(logo_url).path.split('.')[-1]
                    logo_filename = "{0}_{1}.{2}".format(
                        eng_cs.country, eng_cs.set_no, logo_ext)
                    logo_temp.flush()
                    eng_cs.logo.save(logo_filename, File(logo_temp))
                except (TypeError, AttributeError):
                    pass
                try:
                    eng_cs.partial_url = data[2].a['title']
                except TypeError:
                    pass
                eng_cs.release = datetime.strptime(data[6].text.strip(),
                                                   "%B %d, %Y")
                count = data[4].find("span", "explain")
                try:
                    eng_cs.official_count = int(
                        p.search(count['title']).group("count"))
                except (TypeError, AttributeError):
                    if data[4].text.strip() != "":
                        eng_cs.official_count = int(data[4].text.strip())

                self.stdout.write("scraped: {0}\n".format(eng_cs))
                eng_cs.save()
            except ValueError:
                # If there is no valid set_no
                pass

            try:
                jap_cs = CardSet(country="JP")
                jap_cs.set_no = int(data[0].text.strip())
                jap_cs.name = data[3].text.strip()
                try:
                    logo_url = data[1].a.img['src']
                    logo_temp.write(urlopen(logo_url).read())
                    logo_ext = urlparse(logo_url).path.split('.')[-1]
                    logo_filename = "{0}_{1}.{2}".format(
                        jap_cs.country, jap_cs.set_no, logo_ext)
                    logo_temp.flush()
                    jap_cs.logo.save(logo_filename, File(logo_temp))
                except (TypeError, AttributeError):
                    pass
                try:
                    jap_cs.partial_url = data[2].a['title']
                except TypeError:
                    pass
                jap_cs.release = datetime.strptime(data[7].text.strip(),
                                                   "%B %d, %Y")
                count = data[5].find("span", "explain")
                try:
                    jap_cs.official_count = int(
                        p.search(count['title']).group("count"))
                except (TypeError, AttributeError):
                    if data[5].text.strip() != "":
                        jap_cs.official_count = int(data[5].text.strip())

                self.stdout.write("scraped: {0}\n".format(jap_cs))
                jap_cs.save()
            except ValueError:
                # If there is no valid set_no
                pass

        self.stdout.write("total sets {0}\n".format(
            str(CardSet.objects.all().count())))
def doit():
    dbname = ':memory:'
    dbname = os.path.join(DATA_DIR, 'tmpdb.db')
    db = sqlite3.connect(dbname)
    c = db.cursor()

    # TODO read from spreadsheet into temp database "dbname"
    
    settings.DATABASES['default']['AUTOCOMMIT'] = False
    print dir(django.db.transaction)
    print dir(django.db.transaction.connections)
    django.db.transaction.commit_manually()
    #django.db.transaction.set_autocommit(False)
    
    
    """this is a terrible way to delete stuff....
    Game.objects.all().delete()
    BlackCard.objects.all().delete()
    WhiteCard.objects.all().delete()
    CardSet.objects.all().delete()
    # So instead use raw SQL
    """
    dmc = connection.cursor()  # Django Model Database Cursor
    for tablename in ['card_set_white_card', 'cards_game', 'cards_player', 'black_cards', 'card_set', 'card_set_black_card', 'white_cards']:
        dmc.execute('DELETE FROM %s' % tablename)
    dmc.close()
    
    cardset_dict = {}
    for card_ver in 'v1.0', 'v1.2', 'v1.3', 'v1.4':
        cardset = CardSet(name=card_ver, description=card_ver)
        cardset.save()
        cardset_dict[card_ver] = cardset
    
    #c.execute(""" select b."Text" as text, b."Special" as special, b."v1" as v10, b."v1.2" as v12, b."v1.3" as v13, b."v1.4" as v14 from "Main Deck Black" b order by text LIMIT 3""")
    c.execute(""" select b."Text" as text, b."Special" as special, b."v1" as v10, b."v1.2" as v12, b."v1.3" as v13, b."v1.4" as v14 from "Main Deck Black" b order by text """)
    print c.description
    for row_id, row in enumerate(c.fetchall(), 1):
        draw = 0

        print row_id, row
        card_text = row[0]
        special = row[1]
        v10 = row[2]
        v12 = row[3]
        v13 = row[4]
        v14 = row[5]
        if v10:
            # sync with other naming conventions
            v10 = 'v1.0'
        print (card_text, special, v10, v12, v13, v14)
        
        card_text = card_text.replace('______', DEFAULT_BLANK_MARKER)
        if '_' in card_text:
            raise NotImplementedError('found an underscore, this may not be a real problem')
        
        pick = card_text.count(DEFAULT_BLANK_MARKER)
        if pick < 1:
            pick = 1
        
        if special:
            print row
            if special == 'PICK 2':
                pick = 2
            elif special == 'DRAW 2, PICK 3':
                draw = 2
                pick = 3
            else:
                raise NotImplementedError('unrecognized special')
        
        watermark = v10 or v12 or v13 or v14  # pick the first version it showed up in (or we could leave blank)
        black_card = BlackCard(text=card_text, draw=draw, pick=pick, watermark=watermark)
        print black_card
        black_card.save()
        tmp_dict = {'v1.0':v10, 'v1.2':v12, 'v1.3':v13, 'v1.4':v14}
        for card_ver in tmp_dict:
            #print card_ver, tmp_dict[card_ver]
            if tmp_dict[card_ver]:
                cardset = cardset_dict[card_ver]
                cardset.black_card.add(black_card)

    #c.execute(""" select w."Text" as text, w."v1.0" as v10, w."v1.2" as v12, w."v1.3" as v13, w."v1.4" as v14 from "Main Deck White" w order by text LIMIT 5""")
    c.execute(""" select w."Text" as text, w."v1.0" as v10, w."v1.2" as v12, w."v1.3" as v13, w."v1.4" as v14 from "Main Deck White" w order by text""")
    print c.description
    for row_id, row in enumerate(c.fetchall(), 1):
        print row_id, row
        card_text = row[0]
        v10 = row[1]
        v12 = row[2]
        v13 = row[3]
        v14 = row[4]
        
        watermark = v10 or v12 or v13 or v14  # pick the first version it showed up in (or we could leave blank)
        white_card = WhiteCard(text=card_text, watermark=watermark)
        print white_card
        white_card.save()
        tmp_dict = {'v1.0':v10, 'v1.2':v12, 'v1.3':v13, 'v1.4':v14}
        for card_ver in tmp_dict:
            #print card_ver, tmp_dict[card_ver]
            if tmp_dict[card_ver]:
                cardset = cardset_dict[card_ver]
                cardset.white_card.add(white_card)
    
    #c.execute(""" select b."col2" as text, b.col3 as special, b.col4 as expansion_name from "Expansions Black" b  where text NOT NULL and text != '' and text != 'Text' and expansion_name NOT NULL and expansion_name != '' order by text LIMIT 15""")
    c.execute(""" select b."col2" as text, b.col3 as special, b.col4 as expansion_name from "Expansions Black" b  where text NOT NULL and text != '' and text != 'Text' and expansion_name NOT NULL and expansion_name != '' order by text""")
    print c.description
    for row_id, row in enumerate(c.fetchall(), 1):
        draw = 0
    
        print row_id, row
        card_text = row[0]
        special = row[1]
        expansion_name = row[2]

        card_text = replace_blank(card_text)
        
        pick = card_text.count(DEFAULT_BLANK_MARKER)
        if pick < 1:
            pick = 1
        
        if special:
            print row
            if special == 'PICK 2':
                pick = 2
            elif special == 'DRAW 2, PICK 3':
                draw = 2
                pick = 3
            else:
                raise NotImplementedError('unrecognized special')

        cardset = cardset_dict.get(expansion_name)
        # This shouldn't be needed, should have created earlier with black card expansion
        if cardset is None:
            cardset = CardSet(name=expansion_name, description=expansion_name, base_deck=False)
            cardset.save()
            cardset_dict[expansion_name] = cardset
        
        watermark = expansion_name  # could leave it blank
        black_card = BlackCard(text=card_text, draw=draw, pick=pick, watermark=watermark)
        print black_card
        black_card.save()
        cardset.black_card.add(black_card)
    
    # TODO handle "[italic]", etc.
    #c.execute(""" select w."col2" as text, w.col3 as expansion_name from "Expansions White" w where text NOT NULL and text != '' and expansion_name NOT NULL and expansion_name != '' order by expansion_name, text LIMIT 15""")
    c.execute(""" select w.col2 as text, w.col3 as expansion_name from "Expansions White" w where text NOT NULL and text != '' and text != 'Text' and expansion_name NOT NULL and expansion_name != '' order by expansion_name, text""")
    print c.description
    for row_id, row in enumerate(c.fetchall(), 1):
        print row_id, row
        card_text = row[0]
        expansion_name = row[1]
        
        cardset = cardset_dict.get(expansion_name)
        # This shouldn't be needed, should have created earlier with black card expansion
        if cardset is None:
            cardset = CardSet(name=expansion_name, description=expansion_name, base_deck=False)
            cardset.save()
            cardset_dict[expansion_name] = cardset
        
        watermark = expansion_name  # could leave it blank
        white_card = WhiteCard(text=card_text, watermark=watermark)
        print white_card
        white_card.save()
        cardset.white_card.add(white_card)
    
    
    # this is probably not needed due to manual commit later
    for tmp_cardset_name in cardset_dict:
        cardset = cardset_dict[tmp_cardset_name]
        cardset.save()

    django.db.transaction.commit()
    
    c.close()
    db.commit()
    db.close()
Esempio n. 4
0
    def handle(self, *args, **options):
        self.stdout.write("start scraping from Bulbapedia...\n")
        p = re.compile("official (count|total) (?P<count>\d+)")

        CardSet.objects.all().delete()

        html = json.load(urlopen(BASE_URL.format(CARDSET_PAGE))
            )['parse']['text']['*']
        set_tbl = BeautifulSoup(html).find("table").find(
            "table").find_all("tr")[1:]

        for r in set_tbl:
            data = r.find_all("td")
            logo_temp = NamedTemporaryFile()

            try:
                eng_cs = CardSet(country="GB")
                eng_cs.set_no = int(r.find("th").text.strip())
                eng_cs.name = data[2].text.strip()
                try:
                    logo_url = data[1].a.img['src']
                    logo_temp.write(urlopen(logo_url).read())
                    logo_ext = urlparse(logo_url).path.split('.')[-1]
                    logo_filename="{0}_{1}.{2}".format(eng_cs.country,
                        eng_cs.set_no, logo_ext)
                    logo_temp.flush()
                    eng_cs.logo.save(logo_filename, File(logo_temp))
                except (TypeError, AttributeError):
                    pass
                try:
                    eng_cs.partial_url = data[2].a['title']
                except TypeError:
                    pass
                eng_cs.release = datetime.strptime(data[6].text.strip(),
                    "%B %d, %Y")
                count = data[4].find("span", "explain")
                try:
                    eng_cs.official_count = int(p.search(count['title']
                                                         ).group("count"))
                except (TypeError, AttributeError):
                    if data[4].text.strip() != "":
                        eng_cs.official_count = int(data[4].text.strip())

                self.stdout.write("scraped: {0}\n".format(eng_cs))
                eng_cs.save()
            except ValueError:
                # If there is no valid set_no
                pass

            try:
                jap_cs = CardSet(country="JP")
                jap_cs.set_no = int(data[0].text.strip())
                jap_cs.name = data[3].text.strip()
                try:
                    logo_url = data[1].a.img['src']
                    logo_temp.write(urlopen(logo_url).read())
                    logo_ext = urlparse(logo_url).path.split('.')[-1]
                    logo_filename="{0}_{1}.{2}".format(jap_cs.country,
                        jap_cs.set_no, logo_ext)
                    logo_temp.flush()
                    jap_cs.logo.save(logo_filename, File(logo_temp))
                except (TypeError, AttributeError):
                    pass
                try:
                    jap_cs.partial_url = data[2].a['title']
                except TypeError:
                    pass
                jap_cs.release = datetime.strptime(data[7].text.strip(),
                    "%B %d, %Y")
                count = data[5].find("span", "explain")
                try:
                    jap_cs.official_count = int(p.search(count['title']
                                                         ).group("count"))
                except (TypeError, AttributeError):
                    if data[5].text.strip() != "":
                        jap_cs.official_count = int(data[5].text.strip())

                self.stdout.write("scraped: {0}\n".format(jap_cs))
                jap_cs.save()
            except ValueError:
                # If there is no valid set_no
                pass

        self.stdout.write("total sets {0}\n".format(
            str(CardSet.objects.all().count())))