예제 #1
0
    def parseItempage(self):

        if not self.hasnext:
            log("%s: no more items" % self.site)
            return []
        
        #print >>sys.stderr,"web2: gs: Self.newpage = true"
        url = self.get('URL_SEARCH') % (self.query, self.pagecount)
        if DEBUG:
            print >>sys.stderr,"web2: gs: Retrieving url", url
        try:
            pageconn = urllib.urlopen(url)
        except:
            return []
        page = pageconn.read().replace('\n','')
        pageconn.close()
        if DEBUG:
            #print >>sys.stderr,'web2: gs: The page:\n%s' % page
            print >>sys.stderr,'web2: gs: Regexp: %s' % self.get('RE_SEARCHITEM')
        items = re.findall(self.get('RE_SEARCHITEM'), page, re.S)

        self.pagecount += 1

        RE_RESULTS_HASNEXT = self.get('RE_RESULTS_HASNEXT')
        if RE_RESULTS_HASNEXT:
            if len(re.findall(RE_RESULTS_HASNEXT, page)) == 0:
                self.hasnext = False

        if DEBUG:
            print >>sys.stderr,'web2: gs: Items found: %s' % str(items)
            #print >>sys.stderr,'web2: gs: Page',page
        return items
예제 #2
0
    def update(self, subject, item):
        self.lock.acquire()

        if item == None:
            log("CompoundSearch: update -> received None")
            subject.detach(self)
            subject.quit()
            self.searches.remove(subject)

            if len(self.searches) == 0:
                log("CompoundSearch: no more items")
                self.notify(None)

       
        else:
            self.total += 1
            log("CompoundSearch: update -> wanted:" + str(self.wanted)+ ", update -> new total:" + str(self.total))

            if self.total == self.wanted:
                log("CompoundSearch: update -> Enough")
                self.giveSearchFeedback(True, self.total)
                for i in range(len(self.searches)):
                    self.searches[i].enough()
                    
            if self.total < self.wanted:
                self.giveSearchFeedback(False, self.total)

            if self.total <= self.wanted:
                if DEBUG:
                    print >>sys.stderr,"web2: db: CompoundSearch: returning an item"
                self.notify(item)
            else:
                self.items.append(item)

        self.lock.release()
예제 #3
0
    def update(self, subject, item):
        self.lock.acquire()

        if item == None:
            log("CompoundSearch: update -> received None")
            subject.detach(self)
            subject.quit()
            self.searches.remove(subject)

            if len(self.searches) == 0:
                log("CompoundSearch: no more items")
                self.notify(None)

        else:
            self.total += 1
            log("CompoundSearch: update -> wanted:" + str(self.wanted) +
                ", update -> new total:" + str(self.total))

            if self.total == self.wanted:
                log("CompoundSearch: update -> Enough")
                self.giveSearchFeedback(True, self.total)
                for i in range(len(self.searches)):
                    self.searches[i].enough()

            if self.total < self.wanted:
                self.giveSearchFeedback(False, self.total)

            if self.total <= self.wanted:
                if DEBUG:
                    print >> sys.stderr, "web2: db: CompoundSearch: returning an item"
                self.notify(item)
            else:
                self.items.append(item)

        self.lock.release()
예제 #4
0
def VideoTranscode(input, output):
    tmpout = os.path.join(os.path.dirname(output), "." + os.path.basename(output))

    cmd = copy.deepcopy(settings.VIDDECODE_CMD)
    cmd[settings.VIDDECODE_CMD_IO[0]] = input
    cmd[settings.VIDDECODE_CMD_IO[1]] = tmpout
    log("Video transcoder: doing: " + str(cmd))
    proc = subprocess.Popen(cmd)
    rcode = proc.wait()

    if rcode == 0 and os.path.exists(tmpout):
        shutil.move(tmpout, output)
    else:
        os.remove(tmpout)
        raise RuntimeError("Transcoding failed")
예제 #5
0
def VideoTranscode(input, output):
    tmpout = os.path.join(os.path.dirname(output),
                          "." + os.path.basename(output))

    cmd = copy.deepcopy(settings.VIDDECODE_CMD)
    cmd[settings.VIDDECODE_CMD_IO[0]] = input
    cmd[settings.VIDDECODE_CMD_IO[1]] = tmpout
    log("Video transcoder: doing: " + str(cmd))
    proc = subprocess.Popen(cmd)
    rcode = proc.wait()

    if rcode == 0 and os.path.exists(tmpout):
        shutil.move(tmpout, output)
    else:
        os.remove(tmpout)
        raise RuntimeError("Transcoding failed")
예제 #6
0
    def run(self):

        if self.rate != None:
            self.notify(self.rate)
            return

        try:
            url = utilsettings.RATINGGET % (self.flatid, settings.INSTALL_ID)
            log(url)
            conn = urllib.urlopen(url)
            response = conn.read()

            log(response)
            rating = re.findall("([^ ]*) ([^ ]*)", response)[0]
            log(str(rating))
            self.notify((int(rating[0]), int(rating[1])))
        except:
            pass
예제 #7
0
    def run(self):

        if self.rate != None:
            self.notify(self.rate)
            return

        try:
            url = utilsettings.RATINGGET % (self.flatid, settings.INSTALL_ID)
            log(url)
            conn = urllib.urlopen(url)
            response = conn.read()

            log(response)
            rating = re.findall("([^ ]*) ([^ ]*)", response)[0]
            log(str(rating))
            self.notify((int(rating[0]), int(rating[1])))
        except:
            pass
예제 #8
0
    def parseItem(self, workitem):
        if DEBUG:
            print >>sys.stderr,'web2: gs: parseItem called with %s' % str(workitem)
            
        item = {}

        id = workitem[0]

        item['infohash'] = id

        url = self.get('URL_WATCH') % id
        if DEBUG:
            print >>sys.stderr,"web2: gs: getting URL", url
        conn = urllib.urlopen(url)
        itempage = conn.read().replace('\n','')
        srcpage = itempage
        conn.close()
        
        # Liveleak needs a separate page to get video url
        url_for_src = self.get('URL_SRC')
        if url_for_src:
            url_for_src = url_for_src % id
            if DEBUG:
                print >>sys.stderr,"web2: gs: getting URL", url_for_src
            conn = urllib.urlopen(url_for_src)
            srcpage = conn.read().replace('\n','')
            conn.close()

        
        trynum = 0
        success = False
        # Either get link by formatting id
        if self.get('VIDEO_URL'):
            src = self.get('VIDEO_URL') % id
        else:
            # Or with a regexp
            while(True):
                regexp = self.get('RE_VIDEO_URL%d' % trynum)
                if not regexp:
                    break
                src = re.findall(regexp, srcpage, re.S | re.I)
                if len(src) == 1:
                    success = True
                    src = src[0]
                    break
                trynum += 1
            if not success:
                if DEBUG:
                    print >>sys.stderr,'web2: gs: Error, src=%s' % src
                return None
        
        

        # Youtube needs id parsed in url
        id2url = self.get('URL_DL_VIDEO')
        if id2url:
            src = self.get('URL_DL_VIDEO') % src
        
        unquote_url = self.get('UNQUOTE')
        if DEBUG:
            print >>sys.stderr,'web2: gs: unquote: %s' % unquote_url
        if unquote_url:
            src = urllib.unquote(src)
            
        if DEBUG:
            print >>sys.stderr,'web2: gs: Got video url: %s' % src
            
        name = re.findall(self.get('RE_NAME'), itempage)
        if len(name) == 0:
            if DEBUG:
                print >>sys.stderr,'web2: gs: Cannot find item name, error'
            return None
        #print name[0]
        ENCODING = self.get('ENCODING')
        name = unicode(name[0], ENCODING)
        name = codec.decodehtml(name)
        #print name

        item['content_name'] = name

#        category = re.findall(RE_CAT, itempage)
#        if len(category) == 0:
#            if DEBUG:
#                print >>sys.stderr,'web2: gs: Youtube: Category error'
#            return None
#        category = unicode(category[0], ENCODING)
#        category = codec.decodehtml(category)
#        
#        item['category'] = category

        tags = re.findall(self.get('RE_TAG'), itempage)
        if len(tags) == 1:
            tags = re.findall(self.get('RE_TAG2'), tags[0])
        else:
            tags = []
        
        unicodetags = []
        for tag in tags:
            unicodetags.append(unicode(tag, ENCODING))

        item['tags'] = tags
        #item = video.VideoItem((site, id), name, YoutubeDownload, unicodetags, category)

        if workitem[1].lower().startswith('http'):
            conn = urllib.urlopen(workitem[1])
            item['preview'] = conn.read()
        

        if item == None:
            log("Generic search: returning None")
        else:
            log("Generic search: returning " + item['content_name'])


        try:
            desc = re.findall(self.get('RE_DESC'), itempage, re.S)[0]
            if desc.strip() == '':
                raise RuntimeError()
            item['description'] = "from %s: %s" % (self.site, desc)
        except:
            item['description'] = "from %s" % self.site
            
        item['description'] = codec.decodehtml(unicode(item['description'], ENCODING))
        item['web2'] = True
        
        assert type(src) == str, "Url of video was not string, but %s (site:%s, name:%s)" % (repr(src), self.site, name)
        
        item['url'] = src
        item['views'] = 'unknown'
        
        if self.get('RE_VIEWS'):
            views = re.findall(self.get('RE_VIEWS'), itempage)
            if views:    
                item['views'] = int(filter(lambda x: x.isdigit(), views[0]))
        
        if self.get('RE_DATE'):
            date = re.findall(self.get('RE_DATE'), itempage, re.S)
            if date:
                d = GenericDateParser(date[0], self)
                if d is None:
                    d = date[0]
                item['info'] = { 'creation date' : d }
                item['date'] = item['info']['creation date']
                
        item['status'] = 'good'
        item['seeder'] = 1
        item['leecher'] = 1
        try:
            item['length'] = workitem[2]
        except:
            item['length'] = 'unknown'

        
        
        return item