def __init__(self): self.video_ext = [ '.avi', '.mkv', '.mp4', '.3gp', '.mpg', '.mpeg', '.wmv', '.ts' ] ##FIXME needs a global option self.subs_ext = ['.srt', '.ass'] self.logger = tvdb_logger_loader('tvren') self.meddler = Meddler()
def __init__(self): self.logger = tvdb_logger('feeds') self.req = req.Session() self.req.headers = {'User-Agent' : 'w2p_tvdb'} self.meddler = Meddler() self.errors = None
class w2p_tvseries_feed(object): def __init__(self): self.logger = tvdb_logger('feeds') self.req = req.Session() self.req.headers = {'User-Agent' : 'w2p_tvdb'} self.meddler = Meddler() self.errors = None def log(self, function, message): log = self.logger log.log(function, message) def error(self, function, message): log = self.logger log.error(function, message) def search(self, show_name, seasonnumber, quality='', minsize=100, maxsize=4780, regex=None, lower_attention='Verified'): self.eps = [] self.calc_url_feed(show_name, seasonnumber, lower_attention) self.parse_feed() self.filter_list(quality, minsize*1024*1024, maxsize*1024*1024, regex, seasonnumber) return self.eps def calc_url_feed(show_name, seasonnumber): pass def get_torrent_from_hash(self, hash): hash = hash.upper() if len(hash) == 40: return "http://torrage.com/torrent/%s.torrent" % (hash) def retrieve_ttl(self, feed): root = etree.fromstring(feed) ttl = root.findtext('ttl') try: ttl = int(ttl)*60 except: ttl = 15*60 ttl = datetime.datetime.utcnow() + datetime.timedelta(seconds=ttl) return ttl def downloader(self, url, inserted_on=None, verbose=False): """manage ttl""" db = current.w2p_tvseries.database ct = db.urlcache cachekey = hashlib.md5(url).hexdigest() timelimit = datetime.datetime.utcnow() - datetime.timedelta(seconds=3*60) cached = db((ct.kkey == cachekey) & (ct.inserted_on > timelimit)).select().first() if cached: if verbose: self.log('downloader', '%s fetched from cache' % (url)) return cached.value else: try: i = 0 while i < 5: try: r = self.req.get(url, timeout=3, verify=False) r.raise_for_status() break except: i += 1 time.sleep(0.2) if i == 5: raise Exception("can't connect") content = r.content if not inserted_on: inserted_on = self.retrieve_ttl(content) ct.update_or_insert(ct.kkey==cachekey, value=content, inserted_on=inserted_on, kkey=cachekey) db.commit() if verbose: self.log('downloader', '%s fetched from internet' % (url)) except: content = None db.rollback() self.error('downloader', '%s failed to fetch from internet' % (url)) return content def parse_feed(self): """return a list of dicts title seasonnumber episodenumber torrenturl magneturl guid """ content = self.downloader(self.feed_url) if not content: self.errors = "No content fetched" self.error('parse_feed', 'Unable to download feed') self.eps = [] return root = etree.fromstring(content) eps = [] for item in root.findall('channel/item'): eps.append(self.parse_item(item)) self.eps = eps def parse_item(self, item): """return a dict title seasonnumber [episodes] torrenturl magneturl guid pubdate size """ def parse_title(self, title): m = self.meddler.analyze(title) return m def filter_list(self, quality, minsize, maxsize, regex, seasonnumber): self.splitr = re.compile(r'[\s\[\]\(\)\.-]*') quality = [q for q in quality.split(',') if q <> ''] qon = [] qoff = [] self.excluded = [] for a in quality: if a.upper().startswith('NO_'): qoff.append(a.upper().replace('NO_', '',1)) else: qon.append(a.upper()) qon = set(qon) qoff = set(qoff) for i, a in enumerate(self.eps): if a.reason: continue if seasonnumber <> 'ALL': seasonnumber = int(seasonnumber) if self.eps[i].seasonnumber <> seasonnumber: self.eps[i].reason = "Season mismatch (%s vs %s)" % (self.eps[i].seasonnumber, seasonnumber) continue m = self.splitr.split(a.filterwith.upper()) m = [s for s in m if s <> ''] m = set(m) if qoff: exclude = m & qoff if exclude: self.eps[i].reason = "contains %s" % ("and ".join(exclude)) continue if qon: include = m & qon if not include: self.eps[i].reason = "not contains %s" % ("and ".join(qon)) continue if a.size >= maxsize: self.eps[i].reason = "exceeded max size (%s>=%s)" % (a.size/1024/1024, maxsize/1024/1024) elif a.size <= minsize: self.eps[i].reason = "exceeded min size (%s<=%s)" % (a.size/1024/1024, minsize/1024/1024) if regex: try: regex_ = re.compile(regex) if not regex_.search(a.filterwith.upper()): self.eps[i].reason = "not matching regex '%s'" % (regex) except: self.eps[i].reason = "invalid regex '%s'" % (regex) grouper = Storage() for i, ep in enumerate(self.eps): episodes = ep.episodes if ep.reason: continue for subep in episodes: if not grouper[subep]: grouper[subep] = [i] else: grouper[subep].append(i) for k,v in grouper.iteritems(): maxsize = 0 for ep in v: size = self.eps[ep].size / len(self.eps[ep].episodes) if self.eps[ep].size >= maxsize: maxsize = self.eps[ep].size status = 0 for ep in v: if not self.eps[ep].size == maxsize: self.eps[ep].reason = "Better quality found in another torrent" else: status = 1 if not status: continue #if same size all better = [] for ep in v: if '720P' in self.eps[ep].filterwith.upper(): better.append(ep) if len(better) > 0: for ep in v: if ep in better: continue self.eps[ep].reason = "Found same episode with 720P quality" if len(v)>1: for i, ep in enumerate(v): if i > 0: self.eps[ep].reason = "Couldn't see the best episode, discarding the older ones" else: self.eps[ep].reason = None #proper checking for k,v in grouper.iteritems(): proper = [] for ep in v: if 'PROPER' in self.eps[ep].filterwith.upper() or 'REPACK' in self.eps[ep].filterwith.upper(): self.eps[ep].reason = None proper.append(ep) if len(proper) > 0: for ep in v: if ep not in proper: self.eps[ep].reason = 'Proper found for the same episode'
def __init__(self): self.video_ext = ['.avi', '.mkv', '.mp4', '.3gp', '.mpg', '.mpeg', '.wmv', '.ts'] ##FIXME needs a global option self.subs_ext = ['.srt', '.ass'] self.logger = tvdb_logger_loader('tvren') self.meddler = Meddler()
class w2p_tvseries_tvren(object): def __init__(self): self.video_ext = ['.avi', '.mkv', '.mp4', '.3gp', '.mpg', '.mpeg', '.wmv', '.ts'] ##FIXME needs a global option self.subs_ext = ['.srt', '.ass'] self.logger = tvdb_logger_loader('tvren') self.meddler = Meddler() def log(self, function, message): log = self.logger log.log(function, message) def error(self, function, message): log = self.logger log.error(function, message) def slugify(self, value): """taken from web2py's urlify""" import unicodedata s = value s = s.decode('utf-8') # to utf-8 s = unicodedata.normalize('NFKD', s) # normalize eg è => e, ñ => n s = s.encode('ASCII', 'ignore') # encode as ASCII s = re.sub('&\w+;', '', s) # strip html entities s = re.sub('[^\w\- ]', '', s) # strip all but alphanumeric/underscore/hyphen/space s = re.sub('[-_][-_]+', '-', s) # collapse strings of hyphens s = s.strip('-') # remove leading and trailing hyphens return s[:150] # 150 chars will be sufficient def check(self, seriesid, seasonnumber, mode='video'): db = current.w2p_tvseries.database se_tb = db.series ep_tb = db.episodes ss_tb = db.seasons_settings gs = w2p_tvseries_settings().global_settings() if mode == 'video': check_exts = self.video_ext elif mode == 'subs': check_exts = self.subs_ext path_format = gs.path_format or '%(seasonnumber).2d' rec = db((se_tb.id == seriesid) & (ss_tb.tracking == True) & (ss_tb.series_id == se_tb.id) & (ss_tb.seasonnumber == seasonnumber) ).select().first() bpath = rec and rec.series.basepath or '' name = rec and rec.series.name or '' if bpath == '': self.error('check', "basepath not found (%s season %s)" % (name, seasonnumber)) return dict(seriesid=seriesid, seasonnumber=seasonnumber, rename=[], missing=[], errors='basepath not found') path = os.path.join(bpath, path_format % dict(seasonnumber = rec.seasons_settings.seasonnumber)) if not os.path.exists(path): self.error('check', "%s path not found (%s season %s)" % (path, name, seasonnumber)) return dict(seriesid=seriesid, seasonnumber=seasonnumber, rename=[], missing=[], errors='path not found') lista = [] for a in os.listdir(path): file = os.path.join(path, a) if os.path.isfile(file): if os.path.splitext(file)[1] in check_exts: lista.append(file) ep_tb = db.episodes se_tb = db.series eplist = db( (se_tb.id == seriesid) & (ep_tb.seriesid == se_tb.seriesid) & (ep_tb.seasonnumber == seasonnumber) & (ep_tb.firstaired < datetime.datetime.utcnow()) & (ep_tb.tracking == True) ).select(ep_tb.seasonnumber, ep_tb.epnumber, ep_tb.name) eplist_dict = {} for row in eplist: eplist_dict[row.epnumber] = row.name self.default_format = "%(seriesname)s - S%(seasonnumber).2d%(number)s - %(name)s%(ext)s" to_rename = [] episodes_matching = [] matching = [] for file_ in lista: file = os.path.split(file_)[1] match = self.meddler.analyze(file) if match.reason: continue matching.append((file_, match)) if len(match.episodes)>0: for ep in match.episodes: episodes_matching.append(("%.2d" % ep)) missing = [] errors = [] for a in eplist_dict: if "%.2d" % (int(a)) not in episodes_matching: missing.append(a) if len(lista) == 0: rtn = dict(seriesid=seriesid, seasonnumber=seasonnumber, rename=[], missing=missing, errors=errors) return rtn for a in matching: file, match = a origext = os.path.splitext(file)[1] origpath, origfile = os.path.split(file) name = [] number = [] if match.reason: errors.append(file) continue if len(match.episodes)>1: for ep in match.episodes: #(we have a episode, we don't have a record for it) name_ = eplist_dict.get(int(ep), 'WEDONTHAVEARECORDFORTHIS') if name == 'WEDONTHAVEARECORDFORTHIS' and file not in errors: errors.append(file) continue name.append(name_) number.append(ep) name = '-'.join(name) number = ''.join(["E%.2d" % i for i in number]) else: #find name, if not, continue (we have a episode, we don't have a record for it) name = eplist_dict.get(int(match.episodes[0]), 'WEDONTHAVEARECORDFORTHIS') if name == 'WEDONTHAVEARECORDFORTHIS': errors.append(file) continue number = "E%.2d" % int(match.episodes[0]) newname = self.default_format % dict(seriesname = self.slugify(rec.series.name), seasonnumber = int(match.seasonnumber), number = number, name = self.slugify(name), ext = origext ) if mode == 'video': db( (ep_tb.epnumber.belongs(match.episodes)) & (ep_tb.seasonnumber == seasonnumber) & (ep_tb.seriesid == rec.series.seriesid) ).update(filename=newname) if newname != origfile: to_rename.append((file, os.path.join(origpath, newname))) self.log('check', "Completed check for %s season %s" % (rec.series.name, seasonnumber)) rtn = dict(seriesid=seriesid, seasonnumber=seasonnumber, rename=to_rename, missing=missing, errors=errors) return rtn def check_path(self, seriesid, seasonnumber, create=False): db = current.w2p_tvseries.database se_tb = db.series ss_tb = db.seasons_settings gs = w2p_tvseries_settings().global_settings() path_format = gs.path_format or '%(seasonnumber).2d' #check path existance and writability season = db( (se_tb.id == seriesid) & (ss_tb.series_id == se_tb.id) & (ss_tb.seasonnumber == seasonnumber) ).select().first() name = season and season.series.name or '' series_basepath = season and season.series.basepath or '' if series_basepath == '': self.error('check_path', "No basepath found for %s season %s" % (name, seasonnumber)) return dict(err='no basepath', series=season.series.name, seasonnumber=seasonnumber) season_path = os.path.join(series_basepath, path_format % dict(seasonnumber=int(seasonnumber))) if os.path.exists(season_path) and os.access(season_path, os.W_OK): return dict(ok='1') else: if create: try: os.makedirs(season_path) self.log('check_path', 'Created folder %s' % (season_path)) return dict(ok='1') except OSError, e: self.error('check_path', 'error creating folder %s (%s)' % (season_path, e)) return dict(err='error creating folder %s (%s)' % (season_path, e) , series=season.series.name, seasonnumber=seasonnumber ) return dict(dir=season_path, message="dir %s doesn't exist, can I create it?" % season_path, series=season.series.name, seasonnumber=seasonnumber, seriesid=seriesid)
class w2p_tvseries_tvren(object): def __init__(self): self.video_ext = [ '.avi', '.mkv', '.mp4', '.3gp', '.mpg', '.mpeg', '.wmv', '.ts' ] ##FIXME needs a global option self.subs_ext = ['.srt', '.ass'] self.logger = tvdb_logger_loader('tvren') self.meddler = Meddler() def log(self, function, message): log = self.logger log.log(function, message) def error(self, function, message): log = self.logger log.error(function, message) def slugify(self, value): """taken from web2py's urlify""" import unicodedata s = value s = s.decode('utf-8') # to utf-8 s = unicodedata.normalize('NFKD', s) # normalize eg è => e, ñ => n s = s.encode('ASCII', 'ignore') # encode as ASCII s = re.sub('&\w+;', '', s) # strip html entities s = re.sub('[^\w\- ]', '', s) # strip all but alphanumeric/underscore/hyphen/space s = re.sub('[-_][-_]+', '-', s) # collapse strings of hyphens s = s.strip('-') # remove leading and trailing hyphens return s[:150] # 150 chars will be sufficient def check(self, seriesid, seasonnumber, mode='video'): db = current.w2p_tvseries.database se_tb = db.series ep_tb = db.episodes ss_tb = db.seasons_settings gs = w2p_tvseries_settings().global_settings() if mode == 'video': check_exts = self.video_ext elif mode == 'subs': check_exts = self.subs_ext path_format = gs.path_format or '%(seasonnumber).2d' rec = db((se_tb.id == seriesid) & (ss_tb.tracking == True) & (ss_tb.series_id == se_tb.id) & (ss_tb.seasonnumber == seasonnumber)).select().first() bpath = rec and rec.series.basepath or '' name = rec and rec.series.name or '' if bpath == '': self.error( 'check', "basepath not found (%s season %s)" % (name, seasonnumber)) return dict(seriesid=seriesid, seasonnumber=seasonnumber, rename=[], missing=[], errors='basepath not found') path = os.path.join( bpath, path_format % dict(seasonnumber=rec.seasons_settings.seasonnumber)) if not os.path.exists(path): self.error( 'check', "%s path not found (%s season %s)" % (path, name, seasonnumber)) return dict(seriesid=seriesid, seasonnumber=seasonnumber, rename=[], missing=[], errors='path not found') lista = [] for a in os.listdir(path): file = os.path.join(path, a) if os.path.isfile(file): if os.path.splitext(file)[1] in check_exts: lista.append(file) ep_tb = db.episodes se_tb = db.series eplist = db((se_tb.id == seriesid) & (ep_tb.seriesid == se_tb.seriesid) & (ep_tb.seasonnumber == seasonnumber) & (ep_tb.firstaired < datetime.datetime.utcnow()) & (ep_tb.tracking == True)).select(ep_tb.seasonnumber, ep_tb.epnumber, ep_tb.name) eplist_dict = {} for row in eplist: eplist_dict[row.epnumber] = row.name self.default_format = "%(seriesname)s - S%(seasonnumber).2d%(number)s - %(name)s%(ext)s" to_rename = [] episodes_matching = [] matching = [] for file_ in lista: file = os.path.split(file_)[1] match = self.meddler.analyze(file) if match.reason: continue matching.append((file_, match)) if len(match.episodes) > 0: for ep in match.episodes: episodes_matching.append(("%.2d" % ep)) missing = [] errors = [] for a in eplist_dict: if "%.2d" % (int(a)) not in episodes_matching: missing.append(a) if len(lista) == 0: rtn = dict(seriesid=seriesid, seasonnumber=seasonnumber, rename=[], missing=missing, errors=errors) return rtn for a in matching: file, match = a origext = os.path.splitext(file)[1] origpath, origfile = os.path.split(file) name = [] number = [] if match.reason: errors.append(file) continue if len(match.episodes) > 1: for ep in match.episodes: #(we have a episode, we don't have a record for it) name_ = eplist_dict.get(int(ep), 'WEDONTHAVEARECORDFORTHIS') if name == 'WEDONTHAVEARECORDFORTHIS' and file not in errors: errors.append(file) continue name.append(name_) number.append(ep) name = '-'.join(name) number = ''.join(["E%.2d" % i for i in number]) else: #find name, if not, continue (we have a episode, we don't have a record for it) name = eplist_dict.get(int(match.episodes[0]), 'WEDONTHAVEARECORDFORTHIS') if name == 'WEDONTHAVEARECORDFORTHIS': errors.append(file) continue number = "E%.2d" % int(match.episodes[0]) newname = self.default_format % dict( seriesname=self.slugify(rec.series.name), seasonnumber=int(match.seasonnumber), number=number, name=self.slugify(name), ext=origext) if mode == 'video': db((ep_tb.epnumber.belongs(match.episodes)) & (ep_tb.seasonnumber == seasonnumber) & (ep_tb.seriesid == rec.series.seriesid)).update( filename=newname) if newname != origfile: to_rename.append((file, os.path.join(origpath, newname))) self.log( 'check', "Completed check for %s season %s" % (rec.series.name, seasonnumber)) rtn = dict(seriesid=seriesid, seasonnumber=seasonnumber, rename=to_rename, missing=missing, errors=errors) return rtn def check_path(self, seriesid, seasonnumber, create=False): db = current.w2p_tvseries.database se_tb = db.series ss_tb = db.seasons_settings gs = w2p_tvseries_settings().global_settings() path_format = gs.path_format or '%(seasonnumber).2d' #check path existance and writability season = db((se_tb.id == seriesid) & (ss_tb.series_id == se_tb.id) & (ss_tb.seasonnumber == seasonnumber)).select().first() name = season and season.series.name or '' series_basepath = season and season.series.basepath or '' if series_basepath == '': self.error( 'check_path', "No basepath found for %s season %s" % (name, seasonnumber)) return dict(err='no basepath', series=season.series.name, seasonnumber=seasonnumber) season_path = os.path.join( series_basepath, path_format % dict(seasonnumber=int(seasonnumber))) if os.path.exists(season_path) and os.access(season_path, os.W_OK): return dict(ok='1') else: if create: try: os.makedirs(season_path) self.log('check_path', 'Created folder %s' % (season_path)) return dict(ok='1') except OSError, e: self.error( 'check_path', 'error creating folder %s (%s)' % (season_path, e)) return dict(err='error creating folder %s (%s)' % (season_path, e), series=season.series.name, seasonnumber=seasonnumber) return dict(dir=season_path, message="dir %s doesn't exist, can I create it?" % season_path, series=season.series.name, seasonnumber=seasonnumber, seriesid=seriesid)