예제 #1
0
def update_raw_db():
    print "#### Update raw information"
    for idol in raw_information.keys():
        card = models.Card.objects.filter(name=idol).order_by('id')[0]
        raw_information[idol]['main'] = True
        idol, created = models.Idol.objects.update_or_create(name=idol, defaults=raw_information[idol])

    generate_settings()
예제 #2
0
def update_raw_db():
    print "#### Update raw information"
    for idol in raw_information.keys():
        card = models.Card.objects.filter(name=idol).order_by('id')[0]
        raw_information[idol]['main'] = True
        idol, created = models.Idol.objects.update_or_create(
            name=idol, defaults=raw_information[idol])

    print "#### Update cardsinfo.json"
    j = json.dumps({
        'max_stats': {
            'Smile':
            models.Card.objects.order_by('-idolized_maximum_statistics_smile')
            [:1][0].idolized_maximum_statistics_smile,
            'Pure':
            models.Card.objects.order_by('-idolized_maximum_statistics_pure')
            [:1][0].idolized_maximum_statistics_pure,
            'Cool':
            models.Card.objects.order_by('-idolized_maximum_statistics_cool')
            [:1][0].idolized_maximum_statistics_cool,
        },
        'songs_max_stats':
        models.Song.objects.order_by('-expert_notes')[0].expert_notes,
        'idols':
        ValuesQuerySetToDict(
            models.Card.objects.values('name').annotate(
                total=Count('name')).order_by('-total', 'name')),
        'sub_units': [
            card['sub_unit'] for card in models.Idol.objects.filter(
                sub_unit__isnull=False).values('sub_unit').distinct()
        ],
        'years': [
            idol['year'] for idol in models.Idol.objects.filter(
                year__isnull=False).values('year').distinct()
        ],
        'collections':
        ValuesQuerySetToDict(
            models.Card.objects.filter(
                japanese_collection__isnull=False).exclude(
                    japanese_collection__exact='').values(
                        'japanese_collection').annotate(
                            total=Count('name')).order_by(
                                '-total', 'japanese_collection')),
        'skills':
        ValuesQuerySetToDict(
            models.Card.objects.filter(
                skill__isnull=False).values('skill').annotate(
                    total=Count('skill')).order_by('-total'))
    })
    f = open('cardsinfo.json', 'w')
    print >> f, j
    f.close()
예제 #3
0
def update_raw_db():
    print "#### Update raw information"
    for idol in raw_information.keys():
        card = models.Card.objects.filter(name=idol).order_by('id')[0]
        raw_information[idol]['main'] = True
        idol, created = models.Idol.objects.update_or_create(name=idol, defaults=raw_information[idol])

    print "#### Update cardsinfo.json"
    j = json.dumps({
        'max_stats': {
            'Smile': models.Card.objects.order_by('-idolized_maximum_statistics_smile')[:1][0].idolized_maximum_statistics_smile,
            'Pure': models.Card.objects.order_by('-idolized_maximum_statistics_pure')[:1][0].idolized_maximum_statistics_pure,
            'Cool': models.Card.objects.order_by('-idolized_maximum_statistics_cool')[:1][0].idolized_maximum_statistics_cool,
        },
        'songs_max_stats': models.Song.objects.order_by('-expert_notes')[0].expert_notes,
        'idols': ValuesQuerySetToDict(models.Card.objects.values('name').annotate(total=Count('name')).order_by('-total', 'name')),
        'sub_units': [card['sub_unit'] for card in models.Idol.objects.filter(sub_unit__isnull=False).values('sub_unit').distinct()],
        'years': [idol['year'] for idol in models.Idol.objects.filter(year__isnull=False).values('year').distinct()],
        'collections': ValuesQuerySetToDict(models.Card.objects.filter(japanese_collection__isnull=False).exclude(japanese_collection__exact='').values('japanese_collection').annotate(total=Count('name')).order_by('-total', 'japanese_collection')),
        'skills': ValuesQuerySetToDict(models.Card.objects.filter(skill__isnull=False).values('skill').annotate(total=Count('skill')).order_by('-total'))
    })
    f = open('cardsinfo.json', 'w')
    print >> f, j
    f.close()
예제 #4
0
    def handle(self, *args, **options):

        local = 'local' in args
        redownload = 'redownload' in args

        if 'delete' in args:
             models.Card.objects.all().delete()
             models.Event.objects.all().delete()
             models.Idol.objects.all().delete()
             return

        h = HTMLParser.HTMLParser()
        japantz = pytz.timezone('Asia/Tokyo')

        types = {'Normals': 'N', 'Rares': 'R', 'Super Rares': 'SR', 'Ultra Rares': 'UR'}
        specials = {'None': 0, 'Promo Cards': 1, 'Special Cards': 2}

        print '### Import card ids & stats from decaf wiki'
        if local:
            f = open('decaf.html', 'r')
        else:
            f = urllib2.urlopen('http://decaf.kouhi.me/lovelive/index.php?title=List_of_Cards&action=edit')

        currentType = types['Normals']
        special = specials['None']
        for line in f.readlines():
            line = h.unescape(line)
            data = str(line).split('||')
            if len(data) == 1:
                name = clean(data[0].translate(None, '='))
                if name in types.keys():
                    currentType = types[name]
                    special = specials['None']
                    note = ""
                    center = None
                if name in specials.keys():
                    special = specials[name]
                    note = ""
                    center = None
            elif len(data) > 2:
                id = int(clean(data[0]))
                print 'Importing card #', id, '...',; sys.stdout.flush()
                name = cleanwithquotes(data[1].split('|')[1].split('#')[0])
                type = clean(data[2])

                hp = 0
                minStats = (0, 0, 0)
                intStats = (0, 0, 0)
                maxStats = (0, 0, 0)
                nextData = 7
                skill = None
                promo = None
                release_date = None
                event_en = ''
                event_jp = ''
                event = None

                if len(data) > 3:
                    if special != 2:
                        hp = int(clean(data[3]))
                    else:
                        note = clean(data[3].split('|')[-1])
                if len(data) > 6:
                    minStats = (int(clean(data[4])), int(clean(data[5])), int(clean(data[6])))
                if currentType != 'N' and special == 0 and len(data) > 14: # intermediate stats
                    intStats = (int(clean(data[8])), int(clean(data[9])), int(clean(data[10])))
                    maxStats = (int(clean(data[12])), int(clean(data[13])), int(clean(data[14])))
                    nextData = 15
                elif len(data) > 10:
                    maxStats = (int(clean(data[8])), int(clean(data[9])), int(clean(data[10])))
                    nextData = 11
                if len(data) > nextData:
                    skill = clean(data[nextData].split('|')[-1])
                if len(data) > nextData + 1:
                    center = clean(data[nextData + 1].split('|')[-1])

                soup = BeautifulSoup(data[1])
                soupsmall = soup.small
                if soupsmall is not None:
                    soupspan = soupsmall.span
                    if soupspan is not None:
                        if special == 1:
                            promo = soupspan.text.split('c/w ')[-1].replace('[[', '').replace('|', ' (').replace(']]', ') ')
                        else:
                            if 'Added on' in soupspan.text:
                                release_date_str = soupspan.text.split('Added on ')[1]
                                release_date = datetime.datetime.fromtimestamp(time.mktime(time.strptime(release_date_str, '%B %d, %Y')))
                            elif 'event prize' in soupspan.text:
                                event_name = soupspan.text.split(' event prize')[0].replace(']]', '').replace('[[', '')
                                event_jp = event_name.split('|')[0]
                                event_en = event_name.split('|')[-1]
                                event, created = models.Event.objects.update_or_create(japanese_name=event_jp, defaults={
                                    'romaji_name': event_en,
                                })

                defaults = {
                    'name': name,
                    'rarity': currentType,
                    'attribute': type,
                    'is_promo': special == 1,
                    'promo_item': promo,
                    'is_special': special == 2,
                    'hp': hp,
                    'minimum_statistics_smile': minStats[0],
                    'minimum_statistics_pure': minStats[1],
                    'minimum_statistics_cool': minStats[2],
                    'non_idolized_maximum_statistics_smile': intStats[0],
                    'non_idolized_maximum_statistics_pure': intStats[1],
                    'non_idolized_maximum_statistics_cool': intStats[2],
                    'idolized_maximum_statistics_smile': maxStats[0],
                    'idolized_maximum_statistics_pure': maxStats[1],
                    'idolized_maximum_statistics_cool': maxStats[2],
                    'skill': skill,
                    'skill_details': (note if note else None),
                    'center_skill': center,
                }
                if promo:
                    defaults['release_date'] = None
                if release_date is not None:
                    defaults['release_date'] = release_date
                if event is not None:
                    defaults['event'] = event
                idol, created = models.Idol.objects.get_or_create(name=name)
                defaults['idol'] = idol
                card, created = models.Card.objects.update_or_create(id=id, defaults=defaults)
                print 'Done'
        f.close()

        print '### Import events from decaf wiki'
        if local:
            f = open('events.html', 'r')
        else:
            f = urllib2.urlopen('http://decaf.kouhi.me/lovelive/index.php?title=List_of_Events&action=edit')

        for line in f.readlines():
            line = h.unescape(line)
            data = str(line).split('||')
            if len(data) > 1:
                dates = data[0].replace('|', '').split(' - ')
                beginning = eventDateFromString(clean(dates[0]) + ' 4pm', timezone=japantz)
                end = eventDateFromString(str(beginning.year) + '/' + clean(dates[1]) + ' 3pm', timezone=japantz)
                name = clean(data[1].replace('[[', '').replace(']]', '').split('|')[-1]).replace('μs', 'μ\'s')
                t1_points = optInt(clean(data[3]))
                i = 4
                if 'rowspan' in data[i] or len(data) == 7 or len(data) == 8:
                    t1_new_rank = optInt(clean(data[i].split('|')[-1]))
                    if t1_new_rank: t1_rank = t1_new_rank
                    i = i + 1
                t2_points = optInt(data[i])
                i = i + 1
                if len(data) > i and ('rowspan' in data[i] or len(data) == 7 or len(data) == 8):
                    t2_new_rank = optInt(clean(data[i].split('|')[-1]))
                    if t2_new_rank: t2_rank = t2_new_rank
                    i = i + 1
                note = None
                if len(data) > i:
                    note = optString(clean(data[i].split('|')[-1]))
                print 'Import event ', name, '...',; sys.stdout.flush()
                defaults = {
                    'beginning': beginning,
                    'end': end,
                    'japanese_t1_points': t1_points,
                    'japanese_t1_rank': (None if not t1_points else t1_rank),
                    'japanese_t2_points': t2_points,
                    'japanese_t2_rank': t2_rank,
                    'note': note,
                }
                event, created = models.Event.objects.update_or_create(japanese_name=name, defaults=defaults)
                models.Card.objects.filter(event=event).update(release_date=beginning)
                print 'Done'
                if not local and (redownload or not event.image):
                    print "  Import event image...",
                    try:
                        f_event = urllib2.urlopen('http://decaf.kouhi.me/lovelive/index.php?title=' + urllib.quote(name))
                        event_soup = BeautifulSoup(f_event.read())
                        content = event_soup.find('div', { 'id': 'mw-content-text'})
                        if content is not None:
                            image = content.find('img')
                            if image is not None:
                                image = 'http://decaf.kouhi.me/' + image.get('src')
                                event.image.save(name + '.jpg', downloadShrunkedImage(image))
                                print 'Done'
                        f_event.close()
                    except:
                        print "No page found"
        f.close()

        print '### Import EN events T1/T2 cutoffs from decaf wiki'
        if local:
            f = open('eventsEN.html', 'r')
        else:
            f = urllib2.urlopen('http://decaf.kouhi.me/lovelive/index.php?title=English_Version_Info&action=edit')

        for line in f.readlines():
            line = h.unescape(line)
            data = str(line).split('||')
            if len(data) >= 5 and len(data) <= 8:
                dates = data[0].replace('|', '').split(' - ')
                beginning = eventDateFromString(dates[0])
                end = eventDateFromString(str(beginning.year) + '/' + dates[1])
                names = data[1].replace('[[', '').replace(']]', '').replace('μs', 'μ\'s').split('|')
                japanese_name = clean(names[-2])
                english_name = clean(names[-1])
                t1_points = optInt(clean(data[3]))
                i = 4
                if 'rowspan' in data[i] or len(data) == 7 or len(data) == 8:
                    t1_new_rank = optInt(clean(data[i].split('|')[-1]))
                    if t1_new_rank: t1_rank = t1_new_rank
                    i = i + 1
                t2_points = optInt(data[i])
                i = i + 1
                if len(data) > i and ('rowspan' in data[i] or len(data) == 7 or len(data) == 8):
                    t2_new_rank = optInt(clean(data[i].split('|')[-1]))
                    if t2_new_rank: t2_rank = t2_new_rank
                    i = i + 1
                note = None
                if len(data) > i:
                    note = optString(clean(data[i].split('|')[-1]))
                print 'Import event ', english_name, '...',; sys.stdout.flush()
                defaults = {
                    'english_name': english_name,
                    'english_beginning': beginning,
                    'english_end': end,
                    'english_t1_points': t1_points,
                    'english_t1_rank': (None if not t1_points else t1_rank),
                    'english_t2_points': t2_points,
                    'english_t2_rank': t2_rank,
                }
                event, created = models.Event.objects.update_or_create(japanese_name=japanese_name, defaults=defaults)

                print 'Done'
        f.close()

        print '### Import video stories'
        if local:
            f = open('videos.csv', 'r')
        else:
            f = urllib2.urlopen('https://docs.google.com/spreadsheets/d/1AlLTBEuxEBXSVcxE6PpyE8ZcjWfvAQQCsgnKQyFQlPY/export?gid=0&format=csv')
        reader = csv.reader(f)
        for line in reader:
            id = optInt(line[1])
            video = optString(clean(line[2]))
            japanese_video = optString(clean(line[3]))
            if id is not None and (video is not None or japanese_video is not None):
                print 'Add video story to #', id, '... ',
                card, created = models.Card.objects.update_or_create(id=id, defaults={
                    'video_story': video,
                    'japanese_video_story': japanese_video,
                })
                print 'Done'
        f.close()

        print '### Import card pictures and skills details from wikia'
        if local:
            f = open('wikia.html', 'r')
        else:
            f = urllib2.urlopen('http://love-live.wikia.com/wiki/Love_Live!_School_Idol_Festival_Card_List')
        soup = BeautifulSoup(f.read())

        for tr in soup.find_all('tr'):
            tds = tr.find_all('td')
            if len(tds) > 4:
                id = ""
                normal = ''
                idolized = ''
                skill = None
                id = tds[0].string
                if id is not None:
                    id = int(clean(str(id)))
                    print 'Import images & skill for #', id, '...',; sys.stdout.flush()
                normaltd = tds[1].a
                if normaltd is not None:
                    normal = wikiaImageURL(normaltd.get('href'))
                idolizedtd = tds[2].a
                if idolizedtd is not None:
                    idolized = wikiaImageURL(idolizedtd.get('href'))
                if tds[3].b is not None:
                    skill_title = clean(tds[3].b.extract())
                if tds[3].br is not None:
                    tds[3].br.extract()
                if tds[3].text is not None:
                    skill = clean(tds[3].text)
                if id is not None:
                    defaults = {
                        'card_url': normal,
                        'card_idolized_url': idolized,
                    }
                    if skill:
                        defaults['skill_details'] = skill
                    card, created = models.Card.objects.update_or_create(id=id, defaults=defaults)
                    if normal and (redownload or not card.card_image):
                        card.card_image.save(str(card.id) + '.jpg', downloadBestWikiaImage(normal))
                    if idolized and (redownload or not card.card_idolized_image):
                        card.card_idolized_image.save(str(card.id) + 'idolized.jpg', downloadBestWikiaImage(idolized))
                    print 'Done'
        f.close()
        
        print '### Import japanese information for R/SR/UR'
        if local:
            fs = [open('jpcards.html', 'r')]
        else:
            fs = [urllib2.urlopen('http://www59.atwiki.jp/lovelive-sif/pages/210.html'),
                 urllib2.urlopen('http://www59.atwiki.jp/lovelive-sif/pages/378.html'),
                 urllib2.urlopen('http://www59.atwiki.jp/lovelive-sif/pages/379.html'),
            ]
        for f in fs:
            soup = BeautifulSoup("".join(line.strip() for line in f.read().split("\n")))

            for tr in soup.find_all('tr'):
	            tds = tr.find_all('td')
	            if len(tds) > 4:
	                id = tds[0].string
	                try:
	                    id = int(id)
	                except (ValueError, TypeError):
	                    continue
	                if id != None:
	                    print 'Import for #', id, '...',; sys.stdout.flush()
	                    picture = tds[1].img
	                    if picture is not None:
	                        picture = wikiaImageURL(picture.get('src'))
	                    if tds[2].span is not None:
	                        tmp = tds[2].span.extract()
	                    if tds[2].br is not None:
	                        tmp = tds[2].br.extract()
	                    name = clean(tds[2].string)
	                    if name is not None and '(' in name:
	                        version = clean(name.split('(')[-1].split(')')[0])
	                        name = clean(name.split('(')[0])
	                    elif name is not None and '(' in name:
	                        version = clean(name.split('(')[-1].split(')')[0])
	                        name = clean(name.split('(')[0])
	                    else:
	                        version = ''
	                    if len(tds) == 5: # special card
	                        skill_name = None
	                        skill_details = clean(tds[-1].string)
	                        center_skill_name = None
	                        center_skill_details = None
	                    elif len(tds) == 18: # all info specified
	                        skill_name, skill_details = extract_skill(tds[-2])
	                        center_skill_name, center_skill_details = extract_skill(tds[-1])
	                    elif len(tds) == 16: # take center skill from previous line
	                        skill_name, skill_details = extract_skill(tds[-1])
	                    # elif len(tds) == 15: # take skill + center skill from previous line
	                    defaults = {}
	                    if picture is not None:
	                        defaults['round_card_url'] = picture
	                    if version is not None:
	                        defaults['japanese_collection'] = version
	                    if skill_name is not None:
	                        defaults['japanese_skill'] = skill_name
	                    if skill_details is not None:
	                        defaults['japanese_skill_details'] = skill_details
	                    if center_skill_name is not None:
	                        defaults['japanese_center_skill'] = center_skill_name
	                    if center_skill_details is not None:
	                        defaults['japanese_center_skill_details'] = center_skill_details
	                    card, created = models.Card.objects.update_or_create(id=id, defaults=defaults)
	                    card.idol.japanese_name = name
	                    card.save()
	                    if picture and (redownload or not card.round_card_image):
	                        print 'Download image...',; sys.stdout.flush()
	                        card.round_card_image.save(str(card.id) + 'round.jpg', downloadFile(picture))
	                    print 'Done'

            f.close()

        print '### Import japanese information for N (round image + name)'
        if local:
            f = open('jpcardsN.html', 'r')
        else:
            f = urllib2.urlopen('http://www59.atwiki.jp/lovelive-sif/pages/101.html')
        soup = BeautifulSoup("".join(line.strip() for line in f.read().split("\n")))

        for tr in soup.find_all('tr'):
            tds = tr.find_all('td')
            if len(tds) > 4:
                id = tds[0].string
                if id != None:
                    print 'Import for #', id, '...',; sys.stdout.flush()
                    picture = tds[1].img
                    if picture is not None:
                        picture = wikiaImageURL(picture.get('src'))
                    if tds[2].span is not None:
                        tmp = tds[2].span.extract()
                    if tds[2].br is not None:
                        tmp = tds[2].br.extract()
                    name = clean(tds[2].string)
                    defaults = {
                        'japanese_name': name,
                    }
                    if picture is not None:
                        defaults['round_card_url'] = picture
                    card, created = models.Card.objects.update_or_create(id=id, defaults=defaults)
                    if picture and (redownload or not card.round_card_image):
                        print 'Download image...',; sys.stdout.flush()
                        card.round_card_image.save(str(card.id) + 'round.jpg', downloadFile(picture))
                    print 'Done'

        f.close()

        idols = getGirls()
        if not local:
            print "### Import idols"
            for (idx, (idol, _)) in enumerate(idols):
                if not idol:
                    continue
                idol, created = models.Idol.objects.get_or_create(name=idol)
                if idol.attribute and not redownload:
                    continue
                print '  Import Idol', idol, '...',
                f = urllib2.urlopen('http://decaf.kouhi.me/lovelive/index.php?title=' + urllib.quote(idol.name))
                soup = BeautifulSoup(f.read())
                content = soup.find('div', { 'id': 'mw-content-text'})
                if content is not None:
                    content.find('div', { 'id', 'toc' }).extract()
                    defaults = {}
                    wikitable = None
                    if idx <= 9:
                        wikitable = content.find('table', { 'class': 'wikitable' })
                    ul_ = content.find('ul')
                    ul = ul_.find_all('li')
                    for li in ul:
                        if li.b is not None:
                            title = clean(clean(li.b.extract()).replace(':', ''))
                            content = clean(li.text)
                            if title is not None and content is not None and content != '?' and content != ' ?' and content != 'B? / W? / H?' and content != '' and content != '?cm':
                                if title == 'Age':
                                    defaults['age'] = content
                                elif title == 'Birthday':
                                    split = content.replace(')', '').split('(')
                                    birthday = dateutil.parser.parse(clean(split[0]))
                                    sign = clean(split[-1])
                                    defaults['birthday'] = birthday
                                    defaults['astrological_sign'] = sign
                                elif title == 'Japanese Name':
                                    defaults['japanese_name'] = content
                                elif title == 'Blood Type':
                                    defaults['blood'] = content
                                elif title == 'Height':
                                    defaults['height'] = content.replace('cm', '')
                                elif title == 'Three Sizes':
                                    defaults['measurements'] = content
                                elif title == 'Favorite Food' or title == 'Favorite Foods':
                                    defaults['favorite_food'] = content
                                elif title == 'Least Favorite Food' or title == 'Least Favorite Foods':
                                    defaults['least_favorite_food'] = content
                                elif title == 'Hobbies':
                                    defaults['hobbies'] = content
                                elif title == 'Main Attribute':
                                    defaults['attribute'] = content
                                elif title == 'Year':
                                    defaults['year'] = content
                                elif title == 'CV':
                                    defaults['cv'] = content
                                    if li.a:
                                        defaults['cv_url'] = li.a.get('href')
                                else:
                                    print '/!\\ Unknown content', title, content
                    if wikitable is not None:
                        ps = wikitable.find_all('p')
                        if len(ps) >= 2:
                            if ps[0].br is not None:
                                ps[0].br.extract()
                            defaults['summary'] = clean(ps[0].text)
                            if ps[1].a is not None:
                                url = ps[1].a.get('href')
                                defaults['official_url'] = url

                    idol, created = models.Idol.objects.update_or_create(name=idol, defaults=defaults)

                f.close()
                print 'Done'

        print "#### Update raw information"
        for idol in raw_information.keys():
            card = models.Card.objects.filter(name=idol).order_by('id')[0]
            raw_information[idol]['main'] = True
            idol, created = models.Idol.objects.update_or_create(name=idol, defaults=raw_information[idol])

        print "#### Update cardsinfo.json"
        j = json.dumps({
            'max_stats': {
                'Smile': models.Card.objects.order_by('-idolized_maximum_statistics_smile')[:1][0].idolized_maximum_statistics_smile,
                'Pure': models.Card.objects.order_by('-idolized_maximum_statistics_pure')[:1][0].idolized_maximum_statistics_pure,
                'Cool': models.Card.objects.order_by('-idolized_maximum_statistics_cool')[:1][0].idolized_maximum_statistics_cool,
            },
            'idols': ValuesQuerySetToDict(models.Card.objects.values('name').annotate(total=Count('name')).order_by('-total', 'name')),
            'collections': ValuesQuerySetToDict(models.Card.objects.filter(japanese_collection__isnull=False).exclude(japanese_collection__exact='').values('japanese_collection').annotate(total=Count('name')).order_by('-total', 'japanese_collection')),
            'skills': ValuesQuerySetToDict(models.Card.objects.filter(skill__isnull=False).values('skill').annotate(total=Count('skill')).order_by('-total'))
        })
        f = open('cardsinfo.json', 'w')
        print >> f, j
        f.close()
예제 #5
0
    def handle(self, *args, **options):

        local = 'local' in args
        redownload = 'redownload' in args

        if 'delete' in args:
            models.Card.objects.all().delete()
            models.Event.objects.all().delete()
            models.Idol.objects.all().delete()
            return

        h = HTMLParser.HTMLParser()
        japantz = pytz.timezone('Asia/Tokyo')

        types = {
            'Normals': 'N',
            'Rares': 'R',
            'Super Rares': 'SR',
            'Ultra Rares': 'UR'
        }
        specials = {'None': 0, 'Promo Cards': 1, 'Special Cards': 2}

        print '### Import card ids & stats from decaf wiki'
        if local:
            f = open('decaf.html', 'r')
        else:
            f = urllib2.urlopen(
                'http://decaf.kouhi.me/lovelive/index.php?title=List_of_Cards&action=edit'
            )

        currentType = types['Normals']
        special = specials['None']
        for line in f.readlines():
            line = h.unescape(line)
            data = str(line).split('||')
            if len(data) == 1:
                name = clean(data[0].translate(None, '='))
                if name in types.keys():
                    currentType = types[name]
                    special = specials['None']
                    note = ""
                    center = None
                if name in specials.keys():
                    special = specials[name]
                    note = ""
                    center = None
            elif len(data) > 2:
                id = int(clean(data[0]))
                print 'Importing card #', id, '...',
                sys.stdout.flush()
                name = cleanwithquotes(data[1].split('|')[1].split('#')[0])
                type = clean(data[2])

                hp = 0
                minStats = (0, 0, 0)
                intStats = (0, 0, 0)
                maxStats = (0, 0, 0)
                nextData = 7
                skill = None
                promo = None
                release_date = None
                event_en = ''
                event_jp = ''
                event = None

                if len(data) > 3:
                    if special != 2:
                        hp = int(clean(data[3]))
                    else:
                        note = clean(data[3].split('|')[-1])
                if len(data) > 6:
                    minStats = (int(clean(data[4])), int(clean(data[5])),
                                int(clean(data[6])))
                if currentType != 'N' and special == 0 and len(
                        data) > 14:  # intermediate stats
                    intStats = (int(clean(data[8])), int(clean(data[9])),
                                int(clean(data[10])))
                    maxStats = (int(clean(data[12])), int(clean(data[13])),
                                int(clean(data[14])))
                    nextData = 15
                elif len(data) > 10:
                    maxStats = (int(clean(data[8])), int(clean(data[9])),
                                int(clean(data[10])))
                    nextData = 11
                if len(data) > nextData:
                    skill = clean(data[nextData].split('|')[-1])
                if len(data) > nextData + 1:
                    center = clean(data[nextData + 1].split('|')[-1])

                soup = BeautifulSoup(data[1])
                soupsmall = soup.small
                if soupsmall is not None:
                    soupspan = soupsmall.span
                    if soupspan is not None:
                        if special == 1:
                            promo = soupspan.text.split('c/w ')[-1].replace(
                                '[[', '').replace('|',
                                                  ' (').replace(']]', ') ')
                        else:
                            if 'Added on' in soupspan.text:
                                release_date_str = soupspan.text.split(
                                    'Added on ')[1]
                                release_date = datetime.datetime.fromtimestamp(
                                    time.mktime(
                                        time.strptime(release_date_str,
                                                      '%B %d, %Y')))
                            elif 'event prize' in soupspan.text:
                                event_name = soupspan.text.split(
                                    ' event prize')[0].replace(']]',
                                                               '').replace(
                                                                   '[[', '')
                                event_jp = event_name.split('|')[0]
                                event_en = event_name.split('|')[-1]
                                event, created = models.Event.objects.update_or_create(
                                    japanese_name=event_jp,
                                    defaults={
                                        'romaji_name': event_en,
                                    })

                defaults = {
                    'name': name,
                    'rarity': currentType,
                    'attribute': type,
                    'is_promo': special == 1,
                    'promo_item': promo,
                    'is_special': special == 2,
                    'hp': hp,
                    'minimum_statistics_smile': minStats[0],
                    'minimum_statistics_pure': minStats[1],
                    'minimum_statistics_cool': minStats[2],
                    'non_idolized_maximum_statistics_smile': intStats[0],
                    'non_idolized_maximum_statistics_pure': intStats[1],
                    'non_idolized_maximum_statistics_cool': intStats[2],
                    'idolized_maximum_statistics_smile': maxStats[0],
                    'idolized_maximum_statistics_pure': maxStats[1],
                    'idolized_maximum_statistics_cool': maxStats[2],
                    'skill': skill,
                    'skill_details': (note if note else None),
                    'center_skill': center,
                }
                if promo:
                    defaults['release_date'] = None
                if release_date is not None:
                    defaults['release_date'] = release_date
                if event is not None:
                    defaults['event'] = event
                idol, created = models.Idol.objects.get_or_create(name=name)
                defaults['idol'] = idol
                card, created = models.Card.objects.update_or_create(
                    id=id, defaults=defaults)
                print 'Done'
        f.close()

        print '### Import events from decaf wiki'
        if local:
            f = open('events.html', 'r')
        else:
            f = urllib2.urlopen(
                'http://decaf.kouhi.me/lovelive/index.php?title=List_of_Events&action=edit'
            )

        for line in f.readlines():
            line = h.unescape(line)
            data = str(line).split('||')
            if len(data) > 1:
                dates = data[0].replace('|', '').split(' - ')
                beginning = eventDateFromString(clean(dates[0]) + ' 4pm',
                                                timezone=japantz)
                end = eventDateFromString(str(beginning.year) + '/' +
                                          clean(dates[1]) + ' 3pm',
                                          timezone=japantz)
                name = clean(data[1].replace('[[', '').replace(
                    ']]', '').split('|')[-1]).replace('μs', 'μ\'s')
                t1_points = optInt(clean(data[3]))
                i = 4
                if 'rowspan' in data[i] or len(data) == 7 or len(data) == 8:
                    t1_new_rank = optInt(clean(data[i].split('|')[-1]))
                    if t1_new_rank: t1_rank = t1_new_rank
                    i = i + 1
                t2_points = optInt(data[i])
                i = i + 1
                if len(data) > i and ('rowspan' in data[i] or len(data) == 7
                                      or len(data) == 8):
                    t2_new_rank = optInt(clean(data[i].split('|')[-1]))
                    if t2_new_rank: t2_rank = t2_new_rank
                    i = i + 1
                note = None
                if len(data) > i:
                    note = optString(clean(data[i].split('|')[-1]))
                print 'Import event ', name, '...',
                sys.stdout.flush()
                defaults = {
                    'beginning': beginning,
                    'end': end,
                    'japanese_t1_points': t1_points,
                    'japanese_t1_rank': (None if not t1_points else t1_rank),
                    'japanese_t2_points': t2_points,
                    'japanese_t2_rank': t2_rank,
                    'note': note,
                }
                event, created = models.Event.objects.update_or_create(
                    japanese_name=name, defaults=defaults)
                models.Card.objects.filter(event=event).update(
                    release_date=beginning)
                print 'Done'
                if not local and (redownload or not event.image):
                    print "  Import event image...",
                    try:
                        f_event = urllib2.urlopen(
                            'http://decaf.kouhi.me/lovelive/index.php?title=' +
                            urllib.quote(name))
                        event_soup = BeautifulSoup(f_event.read())
                        content = event_soup.find('div',
                                                  {'id': 'mw-content-text'})
                        if content is not None:
                            image = content.find('img')
                            if image is not None:
                                image = 'http://decaf.kouhi.me/' + image.get(
                                    'src')
                                event.image.save(name + '.jpg',
                                                 downloadFile(image))
                                print 'Done'
                        f_event.close()
                    except:
                        print "No page found"
        f.close()

        print '### Import EN events T1/T2 cutoffs from decaf wiki'
        if local:
            f = open('eventsEN.html', 'r')
        else:
            f = urllib2.urlopen(
                'http://decaf.kouhi.me/lovelive/index.php?title=English_Version_Info&action=edit'
            )

        for line in f.readlines():
            line = h.unescape(line)
            data = str(line).split('||')
            if len(data) >= 5 and len(data) <= 8:
                dates = data[0].replace('|', '').split(' - ')
                beginning = eventDateFromString(dates[0])
                end = eventDateFromString(str(beginning.year) + '/' + dates[1])
                names = data[1].replace('[[', '').replace(']]', '').replace(
                    'μs', 'μ\'s').split('|')
                japanese_name = clean(names[-2])
                english_name = clean(names[-1])
                t1_points = optInt(clean(data[3]))
                i = 4
                if 'rowspan' in data[i] or len(data) == 7 or len(data) == 8:
                    t1_new_rank = optInt(clean(data[i].split('|')[-1]))
                    if t1_new_rank: t1_rank = t1_new_rank
                    i = i + 1
                t2_points = optInt(data[i])
                i = i + 1
                if len(data) > i and ('rowspan' in data[i] or len(data) == 7
                                      or len(data) == 8):
                    t2_new_rank = optInt(clean(data[i].split('|')[-1]))
                    if t2_new_rank: t2_rank = t2_new_rank
                    i = i + 1
                note = None
                if len(data) > i:
                    note = optString(clean(data[i].split('|')[-1]))
                print 'Import event ', english_name, '...',
                sys.stdout.flush()
                defaults = {
                    'english_name': english_name,
                    'english_beginning': beginning,
                    'english_end': end,
                    'english_t1_points': t1_points,
                    'english_t1_rank': (None if not t1_points else t1_rank),
                    'english_t2_points': t2_points,
                    'english_t2_rank': t2_rank,
                }
                event, created = models.Event.objects.update_or_create(
                    japanese_name=japanese_name, defaults=defaults)

                print 'Done'
        f.close()

        print '### Import video stories'
        if local:
            f = open('videos.csv', 'r')
        else:
            f = urllib2.urlopen(
                'https://docs.google.com/spreadsheets/d/1AlLTBEuxEBXSVcxE6PpyE8ZcjWfvAQQCsgnKQyFQlPY/export?gid=0&format=csv'
            )
        reader = csv.reader(f)
        for line in reader:
            id = optInt(line[1])
            video = optString(clean(line[2]))
            japanese_video = optString(clean(line[3]))
            if id is not None and (video is not None
                                   or japanese_video is not None):
                print 'Add video story to #', id, '... ',
                card, created = models.Card.objects.update_or_create(
                    id=id,
                    defaults={
                        'video_story': video,
                        'japanese_video_story': japanese_video,
                    })
                print 'Done'
        f.close()

        print '### Import card pictures and skills details from wikia'
        if local:
            f = open('wikia.html', 'r')
        else:
            f = urllib2.urlopen(
                'http://love-live.wikia.com/wiki/Love_Live!_School_Idol_Festival_Card_List'
            )
        soup = BeautifulSoup(f.read())

        for tr in soup.find_all('tr'):
            tds = tr.find_all('td')
            if len(tds) > 4:
                id = ""
                normal = ''
                idolized = ''
                skill = None
                id = tds[0].string
                if id is not None:
                    id = int(clean(str(id)))
                    print 'Import images & skill for #', id, '...',
                    sys.stdout.flush()
                normaltd = tds[1].a
                if normaltd is not None:
                    normal = wikiaImageURL(normaltd.get('href'))
                idolizedtd = tds[2].a
                if idolizedtd is not None:
                    idolized = wikiaImageURL(idolizedtd.get('href'))
                if tds[3].b is not None:
                    skill_title = clean(tds[3].b.extract())
                if tds[3].br is not None:
                    tds[3].br.extract()
                if tds[3].text is not None:
                    skill = clean(tds[3].text)
                if id is not None:
                    defaults = {
                        'card_url': normal,
                        'card_idolized_url': idolized,
                    }
                    if skill:
                        defaults['skill_details'] = skill
                    card, created = models.Card.objects.update_or_create(
                        id=id, defaults=defaults)
                    if normal and (redownload or not card.card_image):
                        card.card_image.save(
                            str(card.id) + '.jpg',
                            downloadBestWikiaImage(normal))
                    if idolized and (redownload
                                     or not card.card_idolized_image):
                        card.card_idolized_image.save(
                            str(card.id) + 'idolized.jpg',
                            downloadBestWikiaImage(idolized))
                    print 'Done'
        f.close()

        print '### Import japanese information for R/SR/UR'
        if local:
            f = open('jpcards.html', 'r')
        else:
            f = urllib2.urlopen(
                'http://www59.atwiki.jp/lovelive-sif/pages/34.html')
        soup = BeautifulSoup("".join(line.strip()
                                     for line in f.read().split("\n")))

        for tr in soup.find_all('tr'):
            tds = tr.find_all('td')
            if len(tds) > 4:
                id = tds[0].string
                try:
                    id = int(id)
                except (ValueError, TypeError):
                    continue
                if id != None:
                    print 'Import for #', id, '...',
                    sys.stdout.flush()
                    picture = tds[1].img
                    if picture is not None:
                        picture = wikiaImageURL(picture.get('src'))
                    if tds[2].span is not None:
                        tmp = tds[2].span.extract()
                    if tds[2].br is not None:
                        tmp = tds[2].br.extract()
                    name = clean(tds[2].string)
                    if name is not None and '(' in name:
                        version = clean(name.split('(')[-1].split(')')[0])
                        name = clean(name.split('(')[0])
                    elif name is not None and '(' in name:
                        version = clean(name.split('(')[-1].split(')')[0])
                        name = clean(name.split('(')[0])
                    else:
                        version = ''
                    if len(tds) == 5:  # special card
                        skill_name = None
                        skill_details = clean(tds[-1].string)
                        center_skill_name = None
                        center_skill_details = None
                    elif len(tds) == 18:  # all info specified
                        skill_name, skill_details = extract_skill(tds[-2])
                        center_skill_name, center_skill_details = extract_skill(
                            tds[-1])
                    elif len(
                            tds) == 16:  # take center skill from previous line
                        skill_name, skill_details = extract_skill(tds[-1])
                    # elif len(tds) == 15: # take skill + center skill from previous line
                    defaults = {}
                    if picture is not None:
                        defaults['round_card_url'] = picture
                    if version is not None:
                        defaults['japanese_collection'] = version
                    if skill_name is not None:
                        defaults['japanese_skill'] = skill_name
                    if skill_details is not None:
                        defaults['japanese_skill_details'] = skill_details
                    if center_skill_name is not None:
                        defaults['japanese_center_skill'] = center_skill_name
                    if center_skill_details is not None:
                        defaults[
                            'japanese_center_skill_details'] = center_skill_details
                    card, created = models.Card.objects.update_or_create(
                        id=id, defaults=defaults)
                    card.idol.japanese_name = name
                    card.save()
                    if picture and (redownload or not card.round_card_image):
                        print 'Download image...',
                        sys.stdout.flush()
                        card.round_card_image.save(
                            str(card.id) + 'round.jpg', downloadFile(picture))
                    print 'Done'

        f.close()

        print '### Import japanese information for N (round image + name)'
        if local:
            f = open('jpcardsN.html', 'r')
        else:
            f = urllib2.urlopen(
                'http://www59.atwiki.jp/lovelive-sif/pages/101.html')
        soup = BeautifulSoup("".join(line.strip()
                                     for line in f.read().split("\n")))

        for tr in soup.find_all('tr'):
            tds = tr.find_all('td')
            if len(tds) > 4:
                id = tds[0].string
                if id != None:
                    print 'Import for #', id, '...',
                    sys.stdout.flush()
                    picture = tds[1].img
                    if picture is not None:
                        picture = wikiaImageURL(picture.get('src'))
                    if tds[2].span is not None:
                        tmp = tds[2].span.extract()
                    if tds[2].br is not None:
                        tmp = tds[2].br.extract()
                    name = clean(tds[2].string)
                    defaults = {
                        'japanese_name': name,
                    }
                    if picture is not None:
                        defaults['round_card_url'] = picture
                    card, created = models.Card.objects.update_or_create(
                        id=id, defaults=defaults)
                    if picture and (redownload or not card.round_card_image):
                        print 'Download image...',
                        sys.stdout.flush()
                        card.round_card_image.save(
                            str(card.id) + 'round.jpg', downloadFile(picture))
                    print 'Done'

        f.close()

        idols = getGirls()
        if not local:
            print "### Import idols"
            for (idx, (idol, _)) in enumerate(idols):
                if not idol:
                    continue
                idol, created = models.Idol.objects.get_or_create(name=idol)
                if idol.attribute and not redownload:
                    continue
                print '  Import Idol', idol, '...',
                f = urllib2.urlopen(
                    'http://decaf.kouhi.me/lovelive/index.php?title=' +
                    urllib.quote(idol))
                soup = BeautifulSoup(f.read())
                content = soup.find('div', {'id': 'mw-content-text'})
                if content is not None:
                    content.find('div', {'id', 'toc'}).extract()
                    defaults = {}
                    wikitable = None
                    if idx <= 9:
                        wikitable = content.find('table',
                                                 {'class': 'wikitable'})
                    ul_ = content.find('ul')
                    ul = ul_.find_all('li')
                    for li in ul:
                        if li.b is not None:
                            title = clean(
                                clean(li.b.extract()).replace(':', ''))
                            content = clean(li.text)
                            if title is not None and content is not None and content != '?' and content != ' ?' and content != 'B? / W? / H?' and content != '' and content != '?cm':
                                if title == 'Age':
                                    defaults['age'] = content
                                elif title == 'Birthday':
                                    split = content.replace(')', '').split('(')
                                    birthday = dateutil.parser.parse(
                                        clean(split[0]))
                                    sign = clean(split[-1])
                                    defaults['birthday'] = birthday
                                    defaults['astrological_sign'] = sign
                                elif title == 'Japanese Name':
                                    defaults['japanese_name'] = content
                                elif title == 'Blood Type':
                                    defaults['blood'] = content
                                elif title == 'Height':
                                    defaults['height'] = content.replace(
                                        'cm', '')
                                elif title == 'Three Sizes':
                                    defaults['measurements'] = content
                                elif title == 'Favorite Food' or title == 'Favorite Foods':
                                    defaults['favorite_food'] = content
                                elif title == 'Least Favorite Food' or title == 'Least Favorite Foods':
                                    defaults['least_favorite_food'] = content
                                elif title == 'Hobbies':
                                    defaults['hobbies'] = content
                                elif title == 'Main Attribute':
                                    defaults['attribute'] = content
                                elif title == 'Year':
                                    defaults['year'] = content
                                elif title == 'CV':
                                    defaults['cv'] = content
                                    if li.a:
                                        defaults['cv_url'] = li.a.get('href')
                                else:
                                    print '/!\\ Unknown content', title, content
                    if wikitable is not None:
                        ps = wikitable.find_all('p')
                        if len(ps) >= 2:
                            if ps[0].br is not None:
                                ps[0].br.extract()
                            defaults['summary'] = clean(ps[0].text)
                            if ps[1].a is not None:
                                url = ps[1].a.get('href')
                                defaults['official_url'] = url

                    idol, created = models.Idol.objects.update_or_create(
                        name=idol, defaults=defaults)

                f.close()
                print 'Done'

        print "#### Update raw information"
        for idol in raw_information.keys():
            card = models.Card.objects.filter(name=idol).order_by('id')[0]
            raw_information[idol]['main'] = True
            idol, created = models.Idol.objects.update_or_create(
                name=idol, defaults=raw_information[idol])

        print "#### Update cardsinfo.json"
        j = json.dumps({
            'max_stats': {
                'Smile':
                models.Card.objects.order_by(
                    '-idolized_maximum_statistics_smile')[:1]
                [0].idolized_maximum_statistics_smile,
                'Pure':
                models.Card.objects.order_by(
                    '-idolized_maximum_statistics_pure')[:1]
                [0].idolized_maximum_statistics_pure,
                'Cool':
                models.Card.objects.order_by(
                    '-idolized_maximum_statistics_cool')[:1]
                [0].idolized_maximum_statistics_cool,
            },
            'idols':
            ValuesQuerySetToDict(
                models.Card.objects.values('name').annotate(
                    total=Count('name')).order_by('-total', 'name')),
            'collections':
            ValuesQuerySetToDict(
                models.Card.objects.filter(
                    japanese_collection__isnull=False).exclude(
                        japanese_collection__exact='').values(
                            'japanese_collection').annotate(
                                total=Count('name')).order_by(
                                    '-total', 'japanese_collection')),
            'skills':
            ValuesQuerySetToDict(
                models.Card.objects.filter(
                    skill__isnull=False).values('skill').annotate(
                        total=Count('skill')).order_by('-total'))
        })
        f = open('cardsinfo.json', 'w')
        print >> f, j
        f.close()