Пример #1
0
    def getReleases(url, max=-1, full=False):
        "Find all releases on a given URL"
        if url[:4] != 'http':
            url = baseUrl + '/' + url
        rels = []
        doc = URLGetter(url).getsoup()

        start = doc.find(text=re.compile('MAIN CONTENT')).parent
        if start:
            doc = start

        relp = re.compile('/release/\?id=(\d+)')
        alist = doc.findAll('a', href=relp)
        # print alist
        for a in alist:
            # print a.contents
            for at in a.attrs:
                # print at
                if at[0] == 'href':
                    m = relp.match(at[1])
                    if a.contents:
                        r = Release(int(m.group(1)), a.contents[0])
                    else:
                        r = Release(int(m.group(1)))
                    if full:
                        r.load()
                    rels.append(r)
            if max > 0 and len(rels) >= max:
                break
        return rels
Пример #2
0
    def findGroups(name):
        groups = []
        print '"%s"' % (name)
        doc = URLGetter(baseUrl + '/search/?seinsel=groups&search=%s' % (name)).getsoup()
        # doc = BeautifulSoup(urllib.urlopen('http://noname.c64.org/csdb/search/?seinsel=groups&search=%s' % name))
        try:
            res = doc.ol.findAll('a')
        except:
            print 'Direct result, figuring out ID'
            r = re.compile('/group/\?id=(\d+).*votes')
            x = doc.find(href=r)
            id = int(r.findall(x.attrs[0][1])[0])
            return [(id, name)]

        for r in res:
            id = int(re.compile('id=(\w+)').findall(r.attrs[0][1])[0])
            name = r.string
            groups.append((id, name))
        return groups
Пример #3
0
    def getGroup(id):
        doc = URLGetter(baseUrl + '/group/?id=%s' % (id)).getsoup()
        start = doc.find(text=re.compile('MAIN CONTENT')).parent
        group = start.find('font')
        name = fixhtml(group.string.strip())
        t = re.compile('\((.*)\)').findall(group.next.next)
        tla = ''
        if t:
            tla = t[0]
        reltab = doc.body.find(text=re.compile('Releases')).findNext('table')
        alist = reltab.findAll('tr')
        releases = []
        url = ''
        for tr in alist:

            # print tr

            rname = '???????'
            rid = -1
            url = ''
            event = ''
            year = '?'
            type = '?'

            try:
                f = tr.findAll('font')
                year = f[1].string.strip()
                if len(f) >= 3:
                    type = fixhtml(f[2].string).strip()
            except:
                pass

            t = tr.find('a', href=re.compile('/release/\?'))
            if t:
                rname = fixhtml(t.string).strip()

                print rname

                try:
                    rid = int(re.compile('id=(\w+)').findall(t.attrs[0][1])[0])
                except:
                    print "Could not parse ID"
                    pass

            tds = tr.findAll('td')
            year = tds[2].font.string
            type = tds[3].font.string

            if type.startswith(' '):
                type = type[6:]

            print "       " + type

            if type == 'Crack':
                print "Skipping Crack " + rname
                continue

            t = tr.find('a', href=re.compile('/release/download'))
            if t:
                url = baseUrl + t.attrs[0][1]
                url = url.strip()
            print url
            releases.append(Release(rid, rname, name, type, year, [url]))

        return Group(id, name, tla, releases)
Пример #4
0
    def download_from_url(self, url, targetdir, to_d64=False, to_prg=False):
        # print "Original URL %s" % url
        url = url.rstrip(';')

        u = URLGetter(url)
        # print "Reading from url '%s'" % u.geturl()
        contents = u.read()
        # print "Got %d bytes" % len(contents)
        fname = fixname(os.path.basename(u.geturl()))

        # if self.type in oneFilers :
        #	to_prg = True

        pickOne = False
        print "ARCHIVE " + fname
        if fname in packArcs or 'releases' in fname.lower() or 'graphics' in fname.lower():
            pickOne = True
            to_prg = True

        # if to_prg == Auto :
        #	if self.type in oneFilers :
        #		to_prg = Always
        #	else :
        #		to_prg = Never

        if u.geturl().find('intros.c64.org') >= 0:
            fname = 'intro.prg'
        print "FNAME: " + fname

        filter = None
        # if to_prg and self.name and self.name != '?' :
        # filter = fixname(self.name).lower()

        try:
            path = tempfile.mkdtemp()
            fullname = path + '/' + fname.lstrip('-')
            f = open(fullname, 'w')
            f.write(contents)
            f.close()
            print "Handling file '%s' to '%s'" % (fullname, targetdir)
            if to_d64 and to_prg:
                if 'oicgm'.find(self.tletter) >= 0:
                    unpack(fullname, targetdir, False, True, filter)
                else:
                    unpack(fullname, targetdir, filter)
            else:
                unpack(fullname, targetdir, to_d64, to_prg, filter)
        # except :
        #	print "FAILED to unpack file"
        finally:
            shutil.rmtree(path)

        count = 0
        res = os.listdir(targetdir)

        best = None
        maxl = 0
        for r in res:
            rs = os.path.splitext(r)
            ext = rs[1].upper()
            base = rs[0].lower()

            if ext == '.PRG' or ext == '.D64' or ext == '.TAP' or ext == ".CRT" or ext == ".G64":
                count += 1
            if ext == ".PRG" and pickOne:
                base = re.sub(r'[^\w]', '', base)
                name = re.sub(r'[^\w]', '', self.name.lower())
                creator = re.sub(r'[^\w]', '', self.creator.lower())
                n = longest_common_substring(name, base)
                c = longest_common_substring(creator, base)
                l = len(n) + len(c)
                if l > maxl:
                    if best:
                        os.remove(targetdir + '/' + best)
                    best = r
                    maxl = l
                else:
                    os.remove(targetdir + '/' + r)
        if best:
            print "Matched %s with %s" % (self.name, best)

        if count == 0:
            print "Download did not contain any C64 files."
            for r in res:
                os.remove(targetdir + '/' + r)
            raise IOError
            return
        elif count > 4 and not pickOne and (fname not in possiblePack):
            print "Directory '%s' contains %d files" % (targetdir, count)
            possiblePack.append(fname)
Пример #5
0
    def load(self):
        doc = URLGetter(baseUrl + '/release/?id=%s' % (self.id)).getsoup()
        if not doc:
            return
        start = doc.find(text=re.compile('MAIN CONTENT')).parent

        anyword = re.compile('\w+')
        name = '?'
        # gid = -1
        group = '?'
        type = '?'
        rdate = '?'
        artist = None
        downloads = None
        vote = None
        score = None
        compo = '?'
        party = '?'
        place2 = place = '?'

        print "Loading release"

        try:
            name = start.find('font').string
            release_by = start.find(text=re.compile('Release.*by'))
            # ag = release_by.findNext('a')
            # gid = int(re.compile('id=(\w+)').findall(ag.attrs[0][1])[0])
            group = release_by.findNext(text=anyword)
        except:
            pass

        try:
            vote = start.find(text=re.compile('.*votes.*'))
            if vote:
                #   7.7/10 (62 votes)  
                m = re.match('[^\d]*([\d\.]+)', vote)
                score = m.group(1)
                if len(score) == 1:
                    score = score + '.0'
        except:
            pass

        try:
            credits = start.find(text=re.compile('Credits :'))
            gfx = credits.findNext(text=re.compile('Graphics'))
            artist = gfx.findNext('a').string
        except:
            pass

        try:
            type = start.find(text=re.compile('Type')).findNext(text=anyword)
            rdate = start.find(text=re.compile('Release.*ate')).findNext(text=anyword)
        except:
            pass
        try:
            downloads = start.find(text=re.compile('Download')).findNext('table').findAll(text=re.compile('tp://'))
        except:
            pass
        # print "'%s' by '%s' (%d), Release date '%s'" % (name, group, gid, rdate)
        try:
            rstart = start.find(text=re.compile('Released At'))
            party = rstart.findNext(text=anyword)
            compo = type
        except:
            pass

        try:
            astart = start.find(text=re.compile('Achievements'))
            print astart
            where = astart.findNext(text=anyword)
            if where:
                compo, party = where.split(' at ')
                print party
                place = where.next[where.next.find('#') + 1:]
                print place
        except:
            pass

        creator = group
        cstart = start.find(text=re.compile('Credits'))
        musicBy = codeBy = graphicsBy = None
        try:
            codeBy = cstart.findNext(text='Code').next.next.findNext(text=anyword)
        except:
            pass
        try:
            musicBy = cstart.findNext(text='Music').next.next.findNext(text=anyword)
        except:
            pass
        try:
            graphicsBy = cstart.findNext(text='Graphics').next.next.findNext(text=anyword)
        except:
            pass

        if type == 'C64 Music':
            creator = musicBy
        elif type == 'C64 Graphics':
            creator = graphicsBy

        print downloads

        if not artist:
            print "ARTIST < " + group
            artist = group

        if name != '?':
            self.name = fixhtml(name)
        if artist and artist != '?':
            self.artist = fixhtml(artist)
        if group != '?':
            self.group = fixhtml(group)
        if type != '?':
            self.type = fixhtml(type)
        if rdate != '?':
            self.date = fixhtml(rdate)
        if downloads:
            self.downloads = downloads
        if score:
            self.score = score
        if compo != '?':
            self.compo = fixhtml(compo)
            print self.compo
        if party != '?':
            self.party = fixhtml(party)
        if place != '?':
            self.place = int(place)
            self.place2 = "%02d" % self.place
        if creator:
            self.creator = fixhtml(creator)
        self.loaded = True
        self.update()