def updateTorrentList(self): # Consolidate torrents in transmission, previously completed torrents and # torrents already in torrentList # Load torrents from data file fileTorrents = BFun.jLoader(self.TORRENT_LIST_PATH, expectList=True) if len(self.torrentList) == 0: self.torrentList = fileTorrents else: consolidatedTorrents = [] hashList = [] for sTorrent in self.torrentList: hashList.append(sTorrent["hashString"]) for fTorrent in fileTorrents: hashList.append(fTorrent["hashString"]) hashSet = set(hashList) hashList = set(hashSet) for hashString in hashList: found = False for sTorrent in self.torrentList: if hashString == sTorrent["hashString"]: consolidatedTorrents.append(sTorrent) found = True break if found == False: for fTorrent in fileTorrents: if hashString == fTorrent["hashString"]: consolidatedTorrents.append(fTorrent) break self.torrentList = consolidatedTorrents BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
def updateTorrentList(self): # Consolidate torrents in transmission, previously completed torrents and # torrents already in torrentList # Load torrents from data file fileTorrents = BFun.jLoader(self.TORRENT_LIST_PATH, expectList=True) if len(self.torrentList) == 0: self.torrentList = fileTorrents else: consolidatedTorrents = [] hashList = [] for sTorrent in self.torrentList: hashList.append(sTorrent['hashString']) for fTorrent in fileTorrents: hashList.append(fTorrent['hashString']) hashSet = set(hashList) hashList = set(hashSet) for hashString in hashList: found = False for sTorrent in self.torrentList: if hashString == sTorrent['hashString']: consolidatedTorrents.append(sTorrent) found = True break if found == False: for fTorrent in fileTorrents: if hashString == fTorrent['hashString']: consolidatedTorrents.append(fTorrent) break self.torrentList = consolidatedTorrents BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
def checkTorrents(self): # global PuxGlobal.globalRecentCompleteTorrents # List of dictionaries # global PuxGlobal.globalCurrentTorrentsProgress # Dictionary of list of dictionaries # Reduce self.requestSleep. if self.requestSleep > 1: self.requestSleep = self.requestSleep - 1 progressList = [] # [{'percent}] for torrentDict in self.torrentList: if torrentDict["completed"] < 1: # refresh the information about the torrent. fileDicts = self.transmissionSession.get_torrent(torrentDict["hashString"]).files() # self.DOWNLOAD_PATH sizeList = [] completeList = [] for n in torrentDict["wantedFileIDs"]: sizeList.append(int(fileDicts[n]["size"])) completeList.append(int(fileDicts[n]["completed"])) totalSize = sum(sizeList) totalComplete = sum(completeList) if totalComplete == totalSize: # Append to globalRecentCompleteTorrents completeDictionary = {} completeDictionary["show"] = self.SHOW_NAME completeDictionary["epNum"] = torrentDict["epNum"] completeDictionary["tic"] = BFun.tic() PuxGlobal.globalRecentCompleteTorrents.append(completeDictionary) # Move files to fileServer # Make sure directory exists try: # Try to get file list from show folder on file server. temp = os.listdir(self.SHOW_PATH) except FileNotFoundError as ex: # Make the directory os.mkdir(self.SHOW_PATH) for n in torrentDict["wantedFileIDs"]: fileName = fileDicts[n]["name"] startPath = self.DOWNLOAD_PATH + "/" + fileName endPath = self.SHOW_PATH print("Copying %s Ep %d" % (completeDictionary["show"], completeDictionary["epNum"])) shutil.copy(startPath, endPath) torrentDict["completed"] = -1 BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH) print("Finished copying") time.sleep(0.1) # Sleep for no reason? # Remove torrent and files from transmission self.transmissionSession.remove_torrent(torrentDict["hashString"], delete_data=True) torrentDict["completed"] = 1 BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH) tempBody = self.SHOW_NAME + " Ep: %d complete" % torrentDict["epNum"] self.pBullet.sendBrowsers(body=tempBody) else: # Add progress to progressDictionary progressDictionary = {} progressDictionary["epNum"] = torrentDict["epNum"] progressDictionary["percent"] = 100.0 * totalComplete / totalSize progressList.append(progressDictionary) PuxGlobal.globalCurrentTorrentsProgress[self.SHOW_NAME] = progressList
# PushBullet module for PuxHelper.py import requests import json import BFun import re import os import time import math import operator from requests.auth import HTTPBasicAuth ''' Notes: Save and load to json with: BFun.jSaver(someJSON,somePath) someJSON = BFun.jLoader(somePath) ''' class PuxBullet: def __init__(self, myName='PuxHelper', sendOnly=True): self.version = 1 self.TILDA = os.environ['HOME'] # Home directory of linux self.CONFIG_DIRECTORY = self.TILDA + '/Mount/fileServer/PUBLIC/Nick/PythonData/Configs' self.getAccessToken() self.myName = myName self.myIden = None self.sleepTime = 1 # Time to sleep between pushes. self.pushList = [] # Need to load push list from local file. self.ratesPerRequest = [5] # Last n ratesPerRequest self.previousRates = -1 self.seenPushIdens = []
def saveData(self): BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
def checkTorrents(self): #global PuxGlobal.globalRecentCompleteTorrents # List of dictionaries #global PuxGlobal.globalCurrentTorrentsProgress # Dictionary of list of dictionaries # Reduce self.requestSleep. if self.requestSleep > 1: self.requestSleep = self.requestSleep - 1 progressList = [] # [{'percent}] for torrentDict in self.torrentList: if torrentDict['completed'] < 1: # refresh the information about the torrent. fileDicts = self.transmissionSession.get_torrent( torrentDict['hashString']).files() #self.DOWNLOAD_PATH sizeList = [] completeList = [] for n in torrentDict['wantedFileIDs']: sizeList.append(int(fileDicts[n]['size'])) completeList.append(int(fileDicts[n]['completed'])) totalSize = sum(sizeList) totalComplete = sum(completeList) if totalComplete == totalSize: # Append to globalRecentCompleteTorrents completeDictionary = {} completeDictionary['show'] = self.SHOW_NAME completeDictionary['epNum'] = torrentDict['epNum'] completeDictionary['tic'] = BFun.tic() PuxGlobal.globalRecentCompleteTorrents.append( completeDictionary) # Move files to fileServer # Make sure directory exists try: # Try to get file list from show folder on file server. temp = os.listdir(self.SHOW_PATH) except FileNotFoundError as ex: # Make the directory os.mkdir(self.SHOW_PATH) for n in torrentDict['wantedFileIDs']: fileName = fileDicts[n]['name'] startPath = self.DOWNLOAD_PATH + '/' + fileName endPath = self.SHOW_PATH print('Copying %s Ep %d' % (completeDictionary['show'], completeDictionary['epNum'])) shutil.copy(startPath, endPath) torrentDict['completed'] = -1 BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH) print('Finished copying') time.sleep(0.1) # Sleep for no reason? # Remove torrent and files from transmission self.transmissionSession.remove_torrent( torrentDict['hashString'], delete_data=True) torrentDict['completed'] = 1 BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH) tempBody = self.SHOW_NAME + ' Ep: %d complete' % torrentDict[ 'epNum'] self.pBullet.sendBrowsers(body=tempBody) else: # Add progress to progressDictionary progressDictionary = {} progressDictionary['epNum'] = torrentDict['epNum'] progressDictionary[ 'percent'] = 100. * totalComplete / totalSize progressList.append(progressDictionary) PuxGlobal.globalCurrentTorrentsProgress[self.SHOW_NAME] = progressList
def newTorrentURLs(self, test=False): # Find the latest torrent URLs, add them (paused) to Transmission and check their size. excludeList = self.getExcludeList() searchDict = {} # Empty dictionary searchDict['URL'] = self.SHOW_URL #searchDict['extension'] = self.EXTENSION foundTorrents = PuxTorrents.torrentSearch(searchDict) urlList = [] tempEpNum = -1 for t in foundTorrents: # tuples in foundTorrents epNum = t[0] url = t[1] alreadyAdded = False if epNum <= self.latestEpisode: print('Already have episode %d' % epNum) alreadyAdded = True else: for torrentDict in self.torrentList: if torrentDict['URL'] == url: print('Already added URL') alreadyAdded = True if not alreadyAdded: print('Processing %s ep %d' % (self.SHOW_NAME, epNum)) #aa = BFun.tic() # add_torrent() will return control once it has finished adding the torrent. print('Getting .torrent...') torrentRetrys = 0 gotTorrent = False while torrentRetrys < 99: try: self.requestSleep += 1.1 print( 'Sleeping %0.1f seconds before retrieving torrent' % self.requestSleep) time.sleep(self.requestSleep) #r = requests.get(url,timeout=10) fileLikeObject = urllib.request.urlopen(url, None, 30) gotTorrent = True break except urllib.error.URLError as ex: if 'error timed out' in str(ex).lower(): torrentRetrys += 1 print('Timed out. Trying again. %d' % torrentRetrys) else: BFun.ezLog( 'Error when retrieving .torrent file. Search for 9217568', ex) except Exception as ex: BFun.ezLog( 'Error when retrieving .torrent file. Search for 3195897', ex) if gotTorrent: # Write data to .torrent file. localURI = self.DATA_PATH + '/lastTorrent.torrent' fo = open(localURI, 'wb') # Gets overwritten. #fo.write(r.content) shutil.copyfileobj(fileLikeObject, fo) fo.close() try: # .torrent file may be bad? torrent = self.transmissionSession.add_torrent( localURI, paused=True) hashString = torrent.hashString torrentContents = self.transmissionSession.get_torrent( hashString).files() except Exception as ex: BFun.ezLog( 'Error when sending .torrent file to transmission. Search for 4523861', ex) # Add bogus torrent to torrentList with URL and Completed. # Skip the URL now but don't mark it so we can try it later. torrentDict = { 'URL': 'a', 'completed': 1, 'epNum': -epNum, 'hashString': 'brokenURL', 'wantedFileIDs': [], 'comment': ['ep %d' % epNum, 'brokenURL', url] } self.torrentList.append(torrentDict) self.saveData() torrentContents = [ ] # Empty list so it won't be processed # Log what happened. BFun.ezLog('Bad .torrent file from %s. (ep %d, %s)' % (url, epNum, localURI)) time.sleep(2) else: print("Didn't get torrent after 99 retrys.") break # Get the unique hash of the torrent so working with torrents is easier. #tSess2.get_torrent(hashString).files() fileInfos = [] wantedFiles = [] unwantedFiles = [] # Check if torrent has files we want. desiredTorrent = False for m in range(len(torrentContents)): desiredFile = True fileName = torrentContents[m]['name'] fileSize = torrentContents[m]['size'] nameSubs = fileName.split('.') if nameSubs[-1].lower() in self.EXTENSION.lower( ): # self.EXTENSION has the '.' print('Processing %s' % fileName) if fileSize > self.MAX_FILE_SIZE: desiredFile = False print('File %d too big' % m) if fileSize < self.MIN_FILE_SIZE: desiredFile = False #print('File %d too small'%m) for exSubString in excludeList: if exSubString.lower() in fileName.lower(): desiredFile = False print('Found %s in %s' % (exSubString, fileName)) else: desiredFile = False #print('File %d has wrong extension'%m) if desiredFile: desiredTorrent = True wantedFiles.append(m) else: unwantedFiles.append(m) if len(wantedFiles) > 8: # If there are more than 8 files, it's a compliation torrent which we don't want. # There may be week's worth of episodes which is ok. desiredTorrent = False print('Found %d files. Rejecting' % len(wantedFiles)) print('unwantedFiles:') print(unwantedFiles) # If torrentContents has no items, it's because it wasn't added. Check "unique hash" for human readable description if len(torrentContents) > 0: self.transmissionSession.change_torrent( hashString, files_unwanted=unwantedFiles) self.transmissionSession.change_torrent( hashString, files_wanted=wantedFiles) if desiredTorrent: urlList.append(url) # If it's a behind the scenes or something, don't change the actual episode number. if epNum != 9999999: print('Found episode %d' % epNum) tempEpNum = max([ epNum, tempEpNum ]) # Largest number in the found torrents. else: epNum = -1 if test: # We're just testing so remove the torrent. self.transmissionSession.remove_torrent( hashString, delete_data=True) else: # Start the torrent self.transmissionSession.start_torrent(hashString) self.pendingTorrents = True # Add to self.torrentList torrentDict = {} torrentDict['hashString'] = hashString torrentDict['wantedFileIDs'] = wantedFiles # 0 = still downloading. -1 = complete on Transmission, not moved to fileserver. 1 = moved to fileserver, removed from transmission torrentDict['completed'] = 0 torrentDict['URL'] = url torrentDict['epNum'] = epNum self.torrentList.append(torrentDict) BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH) else: # Not a torrent we want so remove from transmission. if len(torrentContents) > 0: self.transmissionSession.remove_torrent( hashString, delete_data=True) # and Add bogus torrent to torrentList with URL and Completed so we don't download it again. torrentDict = { 'URL': url, 'completed': 1, 'epNum': -epNum, 'hashString': 'noFiles', 'wantedFileIDs': [] } self.torrentList.append(torrentDict) self.saveData() if tempEpNum > self.latestEpisode: # Update latest episode number print('tempEpNum = %d' % tempEpNum) self.latestEpisode = tempEpNum #dataFile = open(self.LATEST_DATA_PATH,'w') #dataFile.write('%d'%tempEpNum) #dataFile.close() print('Updated data file') if test: return urlList else: return urlList
# PushBullet module for PuxHelper.py import requests import json import BFun import re import os import time import math import operator from requests.auth import HTTPBasicAuth ''' Notes: Save and load to json with: BFun.jSaver(someJSON,somePath) someJSON = BFun.jLoader(somePath) ''' class PuxBullet: def __init__(self,myName='PuxHelper',sendOnly=True): self.version = 1 self.TILDA = os.environ['HOME'] # Home directory of linux self.CONFIG_DIRECTORY = self.TILDA+'/Mount/fileServer/PUBLIC/Nick/PythonData/Configs' self.getAccessToken() self.myName = myName self.myIden = None self.sleepTime = 1 # Time to sleep between pushes. self.pushList = [] # Need to load push list from local file. self.ratesPerRequest = [5] # Last n ratesPerRequest self.previousRates = -1
def newTorrentURLs(self, test=False): # Find the latest torrent URLs, add them (paused) to Transmission and check their size. excludeList = self.getExcludeList() searchDict = {} # Empty dictionary searchDict["URL"] = self.SHOW_URL # searchDict['extension'] = self.EXTENSION foundTorrents = PuxTorrents.torrentSearch(searchDict) urlList = [] tempEpNum = -1 for t in foundTorrents: # tuples in foundTorrents epNum = t[0] url = t[1] alreadyAdded = False if epNum <= self.latestEpisode: print("Already have episode %d" % epNum) alreadyAdded = True else: for torrentDict in self.torrentList: if torrentDict["URL"] == url: print("Already added URL") alreadyAdded = True if not alreadyAdded: print("Processing %s ep %d" % (self.SHOW_NAME, epNum)) # aa = BFun.tic() # add_torrent() will return control once it has finished adding the torrent. print("Getting .torrent...") torrentRetrys = 0 gotTorrent = False while torrentRetrys < 99: try: self.requestSleep += 1.1 print("Sleeping %0.1f seconds before retrieving torrent" % self.requestSleep) time.sleep(self.requestSleep) # r = requests.get(url,timeout=10) fileLikeObject = urllib.request.urlopen(url, None, 30) gotTorrent = True break except urllib.error.URLError as ex: if "error timed out" in str(ex).lower(): torrentRetrys += 1 print("Timed out. Trying again. %d" % torrentRetrys) else: BFun.ezLog("Error when retrieving .torrent file. Search for 9217568", ex) except Exception as ex: BFun.ezLog("Error when retrieving .torrent file. Search for 3195897", ex) if gotTorrent: # Write data to .torrent file. localURI = self.DATA_PATH + "/lastTorrent.torrent" fo = open(localURI, "wb") # Gets overwritten. # fo.write(r.content) shutil.copyfileobj(fileLikeObject, fo) fo.close() try: # .torrent file may be bad? torrent = self.transmissionSession.add_torrent(localURI, paused=True) hashString = torrent.hashString torrentContents = self.transmissionSession.get_torrent(hashString).files() except Exception as ex: BFun.ezLog("Error when sending .torrent file to transmission. Search for 4523861", ex) # Add bogus torrent to torrentList with URL and Completed. # Skip the URL now but don't mark it so we can try it later. torrentDict = { "URL": "a", "completed": 1, "epNum": -epNum, "hashString": "brokenURL", "wantedFileIDs": [], "comment": ["ep %d" % epNum, "brokenURL", url], } self.torrentList.append(torrentDict) self.saveData() torrentContents = [] # Empty list so it won't be processed # Log what happened. BFun.ezLog("Bad .torrent file from %s. (ep %d, %s)" % (url, epNum, localURI)) time.sleep(2) else: print("Didn't get torrent after 99 retrys.") break # Get the unique hash of the torrent so working with torrents is easier. # tSess2.get_torrent(hashString).files() fileInfos = [] wantedFiles = [] unwantedFiles = [] # Check if torrent has files we want. desiredTorrent = False for m in range(len(torrentContents)): desiredFile = True fileName = torrentContents[m]["name"] fileSize = torrentContents[m]["size"] nameSubs = fileName.split(".") if nameSubs[-1].lower() in self.EXTENSION.lower(): # self.EXTENSION has the '.' print("Processing %s" % fileName) if fileSize > self.MAX_FILE_SIZE: desiredFile = False print("File %d too big" % m) if fileSize < self.MIN_FILE_SIZE: desiredFile = False # print('File %d too small'%m) for exSubString in excludeList: if exSubString.lower() in fileName.lower(): desiredFile = False print("Found %s in %s" % (exSubString, fileName)) else: desiredFile = False # print('File %d has wrong extension'%m) if desiredFile: desiredTorrent = True wantedFiles.append(m) else: unwantedFiles.append(m) if len(wantedFiles) > 8: # If there are more than 8 files, it's a compliation torrent which we don't want. # There may be week's worth of episodes which is ok. desiredTorrent = False print("Found %d files. Rejecting" % len(wantedFiles)) print("unwantedFiles:") print(unwantedFiles) # If torrentContents has no items, it's because it wasn't added. Check "unique hash" for human readable description if len(torrentContents) > 0: self.transmissionSession.change_torrent(hashString, files_unwanted=unwantedFiles) self.transmissionSession.change_torrent(hashString, files_wanted=wantedFiles) if desiredTorrent: urlList.append(url) # If it's a behind the scenes or something, don't change the actual episode number. if epNum != 9999999: print("Found episode %d" % epNum) tempEpNum = max([epNum, tempEpNum]) # Largest number in the found torrents. else: epNum = -1 if test: # We're just testing so remove the torrent. self.transmissionSession.remove_torrent(hashString, delete_data=True) else: # Start the torrent self.transmissionSession.start_torrent(hashString) self.pendingTorrents = True # Add to self.torrentList torrentDict = {} torrentDict["hashString"] = hashString torrentDict["wantedFileIDs"] = wantedFiles # 0 = still downloading. -1 = complete on Transmission, not moved to fileserver. 1 = moved to fileserver, removed from transmission torrentDict["completed"] = 0 torrentDict["URL"] = url torrentDict["epNum"] = epNum self.torrentList.append(torrentDict) BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH) else: # Not a torrent we want so remove from transmission. if len(torrentContents) > 0: self.transmissionSession.remove_torrent(hashString, delete_data=True) # and Add bogus torrent to torrentList with URL and Completed so we don't download it again. torrentDict = { "URL": url, "completed": 1, "epNum": -epNum, "hashString": "noFiles", "wantedFileIDs": [], } self.torrentList.append(torrentDict) self.saveData() if tempEpNum > self.latestEpisode: # Update latest episode number print("tempEpNum = %d" % tempEpNum) self.latestEpisode = tempEpNum # dataFile = open(self.LATEST_DATA_PATH,'w') # dataFile.write('%d'%tempEpNum) # dataFile.close() print("Updated data file") if test: return urlList else: return urlList
import PuxGlobal import BFun import transmissionrpc import re import os import time import PuxTorrents import shutil import urllib.request import requests from PuxBulletf import PuxBullet """ Notes: Save and load self.torrentList to a json with: BFun.jSaver(someJSON,somePath) someJSON = BFun.jLoader(somePath) """ # global PuxGlobal.globalRecentCompleteTorrents # List of dictionaries # global PuxGlobal.globalCurrentTorrentsProgress # Dictionary of list of dictionaries class PuxShow: # Pre define some functions so things work? # def findLatestEpisode(self): # pass # Class to handle show torrents, files on server etc. def __init__(self, torrentLine, PuxBulletObject): # Initialise PuxShow class