def updateTorrentList(self):
# Consolidate torrents in transmission, previously completed torrents and
# torrents already in torrentList
# Load torrents from data file
fileTorrents = BFun.jLoader(self.TORRENT_LIST_PATH, expectList=True)
if len(self.torrentList) == 0:
self.torrentList = fileTorrents
else:
consolidatedTorrents = []
hashList = []
for sTorrent in self.torrentList:
hashList.append(sTorrent["hashString"])
for fTorrent in fileTorrents:
hashList.append(fTorrent["hashString"])
hashSet = set(hashList)
hashList = set(hashSet)
for hashString in hashList:
found = False
for sTorrent in self.torrentList:
if hashString == sTorrent["hashString"]:
consolidatedTorrents.append(sTorrent)
found = True
break
if found == False:
for fTorrent in fileTorrents:
if hashString == fTorrent["hashString"]:
consolidatedTorrents.append(fTorrent)
break
self.torrentList = consolidatedTorrents
BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
def updateTorrentList(self):
# Consolidate torrents in transmission, previously completed torrents and
# torrents already in torrentList
# Load torrents from data file
fileTorrents = BFun.jLoader(self.TORRENT_LIST_PATH, expectList=True)
if len(self.torrentList) == 0:
self.torrentList = fileTorrents
else:
consolidatedTorrents = []
hashList = []
for sTorrent in self.torrentList:
hashList.append(sTorrent['hashString'])
for fTorrent in fileTorrents:
hashList.append(fTorrent['hashString'])
hashSet = set(hashList)
hashList = set(hashSet)
for hashString in hashList:
found = False
for sTorrent in self.torrentList:
if hashString == sTorrent['hashString']:
consolidatedTorrents.append(sTorrent)
found = True
break
if found == False:
for fTorrent in fileTorrents:
if hashString == fTorrent['hashString']:
consolidatedTorrents.append(fTorrent)
break
self.torrentList = consolidatedTorrents
BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
def checkTorrents(self):
# global PuxGlobal.globalRecentCompleteTorrents # List of dictionaries
# global PuxGlobal.globalCurrentTorrentsProgress # Dictionary of list of dictionaries
# Reduce self.requestSleep.
if self.requestSleep > 1:
self.requestSleep = self.requestSleep - 1
progressList = [] # [{'percent}]
for torrentDict in self.torrentList:
if torrentDict["completed"] < 1:
# refresh the information about the torrent.
fileDicts = self.transmissionSession.get_torrent(torrentDict["hashString"]).files()
# self.DOWNLOAD_PATH
sizeList = []
completeList = []
for n in torrentDict["wantedFileIDs"]:
sizeList.append(int(fileDicts[n]["size"]))
completeList.append(int(fileDicts[n]["completed"]))
totalSize = sum(sizeList)
totalComplete = sum(completeList)
if totalComplete == totalSize:
# Append to globalRecentCompleteTorrents
completeDictionary = {}
completeDictionary["show"] = self.SHOW_NAME
completeDictionary["epNum"] = torrentDict["epNum"]
completeDictionary["tic"] = BFun.tic()
PuxGlobal.globalRecentCompleteTorrents.append(completeDictionary)
# Move files to fileServer
# Make sure directory exists
try:
# Try to get file list from show folder on file server.
temp = os.listdir(self.SHOW_PATH)
except FileNotFoundError as ex:
# Make the directory
os.mkdir(self.SHOW_PATH)
for n in torrentDict["wantedFileIDs"]:
fileName = fileDicts[n]["name"]
startPath = self.DOWNLOAD_PATH + "/" + fileName
endPath = self.SHOW_PATH
print("Copying %s Ep %d" % (completeDictionary["show"], completeDictionary["epNum"]))
shutil.copy(startPath, endPath)
torrentDict["completed"] = -1
BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
print("Finished copying")
time.sleep(0.1) # Sleep for no reason?
# Remove torrent and files from transmission
self.transmissionSession.remove_torrent(torrentDict["hashString"], delete_data=True)
torrentDict["completed"] = 1
BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
tempBody = self.SHOW_NAME + " Ep: %d complete" % torrentDict["epNum"]
self.pBullet.sendBrowsers(body=tempBody)
else:
# Add progress to progressDictionary
progressDictionary = {}
progressDictionary["epNum"] = torrentDict["epNum"]
progressDictionary["percent"] = 100.0 * totalComplete / totalSize
progressList.append(progressDictionary)
PuxGlobal.globalCurrentTorrentsProgress[self.SHOW_NAME] = progressList
# PushBullet module for PuxHelper.py
import requests
import json
import BFun
import re
import os
import time
import math
import operator
from requests.auth import HTTPBasicAuth
'''
Notes:
Save and load to json with:
BFun.jSaver(someJSON,somePath)
someJSON = BFun.jLoader(somePath)
'''
class PuxBullet:
def __init__(self, myName='PuxHelper', sendOnly=True):
self.version = 1
self.TILDA = os.environ['HOME'] # Home directory of linux
self.CONFIG_DIRECTORY = self.TILDA + '/Mount/fileServer/PUBLIC/Nick/PythonData/Configs'
self.getAccessToken()
self.myName = myName
self.myIden = None
self.sleepTime = 1 # Time to sleep between pushes.
self.pushList = [] # Need to load push list from local file.
self.ratesPerRequest = [5] # Last n ratesPerRequest
self.previousRates = -1
self.seenPushIdens = []
def saveData(self):
BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
def checkTorrents(self):
#global PuxGlobal.globalRecentCompleteTorrents # List of dictionaries
#global PuxGlobal.globalCurrentTorrentsProgress # Dictionary of list of dictionaries
# Reduce self.requestSleep.
if self.requestSleep > 1:
self.requestSleep = self.requestSleep - 1
progressList = [] # [{'percent}]
for torrentDict in self.torrentList:
if torrentDict['completed'] < 1:
# refresh the information about the torrent.
fileDicts = self.transmissionSession.get_torrent(
torrentDict['hashString']).files()
#self.DOWNLOAD_PATH
sizeList = []
completeList = []
for n in torrentDict['wantedFileIDs']:
sizeList.append(int(fileDicts[n]['size']))
completeList.append(int(fileDicts[n]['completed']))
totalSize = sum(sizeList)
totalComplete = sum(completeList)
if totalComplete == totalSize:
# Append to globalRecentCompleteTorrents
completeDictionary = {}
completeDictionary['show'] = self.SHOW_NAME
completeDictionary['epNum'] = torrentDict['epNum']
completeDictionary['tic'] = BFun.tic()
PuxGlobal.globalRecentCompleteTorrents.append(
completeDictionary)
# Move files to fileServer
# Make sure directory exists
try:
# Try to get file list from show folder on file server.
temp = os.listdir(self.SHOW_PATH)
except FileNotFoundError as ex:
# Make the directory
os.mkdir(self.SHOW_PATH)
for n in torrentDict['wantedFileIDs']:
fileName = fileDicts[n]['name']
startPath = self.DOWNLOAD_PATH + '/' + fileName
endPath = self.SHOW_PATH
print('Copying %s Ep %d' %
(completeDictionary['show'],
completeDictionary['epNum']))
shutil.copy(startPath, endPath)
torrentDict['completed'] = -1
BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
print('Finished copying')
time.sleep(0.1) # Sleep for no reason?
# Remove torrent and files from transmission
self.transmissionSession.remove_torrent(
torrentDict['hashString'], delete_data=True)
torrentDict['completed'] = 1
BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
tempBody = self.SHOW_NAME + ' Ep: %d complete' % torrentDict[
'epNum']
self.pBullet.sendBrowsers(body=tempBody)
else:
# Add progress to progressDictionary
progressDictionary = {}
progressDictionary['epNum'] = torrentDict['epNum']
progressDictionary[
'percent'] = 100. * totalComplete / totalSize
progressList.append(progressDictionary)
PuxGlobal.globalCurrentTorrentsProgress[self.SHOW_NAME] = progressList
def newTorrentURLs(self, test=False):
# Find the latest torrent URLs, add them (paused) to Transmission and check their size.
excludeList = self.getExcludeList()
searchDict = {} # Empty dictionary
searchDict['URL'] = self.SHOW_URL
#searchDict['extension'] = self.EXTENSION
foundTorrents = PuxTorrents.torrentSearch(searchDict)
urlList = []
tempEpNum = -1
for t in foundTorrents: # tuples in foundTorrents
epNum = t[0]
url = t[1]
alreadyAdded = False
if epNum <= self.latestEpisode:
print('Already have episode %d' % epNum)
alreadyAdded = True
else:
for torrentDict in self.torrentList:
if torrentDict['URL'] == url:
print('Already added URL')
alreadyAdded = True
if not alreadyAdded:
print('Processing %s ep %d' % (self.SHOW_NAME, epNum))
#aa = BFun.tic()
# add_torrent() will return control once it has finished adding the torrent.
print('Getting .torrent...')
torrentRetrys = 0
gotTorrent = False
while torrentRetrys < 99:
try:
self.requestSleep += 1.1
print(
'Sleeping %0.1f seconds before retrieving torrent'
% self.requestSleep)
time.sleep(self.requestSleep)
#r = requests.get(url,timeout=10)
fileLikeObject = urllib.request.urlopen(url, None, 30)
gotTorrent = True
break
except urllib.error.URLError as ex:
if 'error timed out' in str(ex).lower():
torrentRetrys += 1
print('Timed out. Trying again. %d' %
torrentRetrys)
else:
BFun.ezLog(
'Error when retrieving .torrent file. Search for 9217568',
ex)
except Exception as ex:
BFun.ezLog(
'Error when retrieving .torrent file. Search for 3195897',
ex)
if gotTorrent:
# Write data to .torrent file.
localURI = self.DATA_PATH + '/lastTorrent.torrent'
fo = open(localURI, 'wb') # Gets overwritten.
#fo.write(r.content)
shutil.copyfileobj(fileLikeObject, fo)
fo.close()
try:
# .torrent file may be bad?
torrent = self.transmissionSession.add_torrent(
localURI, paused=True)
hashString = torrent.hashString
torrentContents = self.transmissionSession.get_torrent(
hashString).files()
except Exception as ex:
BFun.ezLog(
'Error when sending .torrent file to transmission. Search for 4523861',
ex)
# Add bogus torrent to torrentList with URL and Completed.
# Skip the URL now but don't mark it so we can try it later.
torrentDict = {
'URL': 'a',
'completed': 1,
'epNum': -epNum,
'hashString': 'brokenURL',
'wantedFileIDs': [],
'comment': ['ep %d' % epNum, 'brokenURL', url]
}
self.torrentList.append(torrentDict)
self.saveData()
torrentContents = [
] # Empty list so it won't be processed
# Log what happened.
BFun.ezLog('Bad .torrent file from %s. (ep %d, %s)' %
(url, epNum, localURI))
time.sleep(2)
else:
print("Didn't get torrent after 99 retrys.")
break
# Get the unique hash of the torrent so working with torrents is easier.
#tSess2.get_torrent(hashString).files()
fileInfos = []
wantedFiles = []
unwantedFiles = []
# Check if torrent has files we want.
desiredTorrent = False
for m in range(len(torrentContents)):
desiredFile = True
fileName = torrentContents[m]['name']
fileSize = torrentContents[m]['size']
nameSubs = fileName.split('.')
if nameSubs[-1].lower() in self.EXTENSION.lower(
): # self.EXTENSION has the '.'
print('Processing %s' % fileName)
if fileSize > self.MAX_FILE_SIZE:
desiredFile = False
print('File %d too big' % m)
if fileSize < self.MIN_FILE_SIZE:
desiredFile = False
#print('File %d too small'%m)
for exSubString in excludeList:
if exSubString.lower() in fileName.lower():
desiredFile = False
print('Found %s in %s' %
(exSubString, fileName))
else:
desiredFile = False
#print('File %d has wrong extension'%m)
if desiredFile:
desiredTorrent = True
wantedFiles.append(m)
else:
unwantedFiles.append(m)
if len(wantedFiles) > 8:
# If there are more than 8 files, it's a compliation torrent which we don't want.
# There may be week's worth of episodes which is ok.
desiredTorrent = False
print('Found %d files. Rejecting' % len(wantedFiles))
print('unwantedFiles:')
print(unwantedFiles)
# If torrentContents has no items, it's because it wasn't added. Check "unique hash" for human readable description
if len(torrentContents) > 0:
self.transmissionSession.change_torrent(
hashString, files_unwanted=unwantedFiles)
self.transmissionSession.change_torrent(
hashString, files_wanted=wantedFiles)
if desiredTorrent:
urlList.append(url)
# If it's a behind the scenes or something, don't change the actual episode number.
if epNum != 9999999:
print('Found episode %d' % epNum)
tempEpNum = max([
epNum, tempEpNum
]) # Largest number in the found torrents.
else:
epNum = -1
if test:
# We're just testing so remove the torrent.
self.transmissionSession.remove_torrent(
hashString, delete_data=True)
else:
# Start the torrent
self.transmissionSession.start_torrent(hashString)
self.pendingTorrents = True
# Add to self.torrentList
torrentDict = {}
torrentDict['hashString'] = hashString
torrentDict['wantedFileIDs'] = wantedFiles
# 0 = still downloading. -1 = complete on Transmission, not moved to fileserver. 1 = moved to fileserver, removed from transmission
torrentDict['completed'] = 0
torrentDict['URL'] = url
torrentDict['epNum'] = epNum
self.torrentList.append(torrentDict)
BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
else:
# Not a torrent we want so remove from transmission.
if len(torrentContents) > 0:
self.transmissionSession.remove_torrent(
hashString, delete_data=True)
# and Add bogus torrent to torrentList with URL and Completed so we don't download it again.
torrentDict = {
'URL': url,
'completed': 1,
'epNum': -epNum,
'hashString': 'noFiles',
'wantedFileIDs': []
}
self.torrentList.append(torrentDict)
self.saveData()
if tempEpNum > self.latestEpisode: # Update latest episode number
print('tempEpNum = %d' % tempEpNum)
self.latestEpisode = tempEpNum
#dataFile = open(self.LATEST_DATA_PATH,'w')
#dataFile.write('%d'%tempEpNum)
#dataFile.close()
print('Updated data file')
if test:
return urlList
else:
return urlList
# PushBullet module for PuxHelper.py import requests import json import BFun import re import os import time import math import operator from requests.auth import HTTPBasicAuth ''' Notes: Save and load to json with: BFun.jSaver(someJSON,somePath) someJSON = BFun.jLoader(somePath) ''' class PuxBullet: def __init__(self,myName='PuxHelper',sendOnly=True): self.version = 1 self.TILDA = os.environ['HOME'] # Home directory of linux self.CONFIG_DIRECTORY = self.TILDA+'/Mount/fileServer/PUBLIC/Nick/PythonData/Configs' self.getAccessToken() self.myName = myName self.myIden = None self.sleepTime = 1 # Time to sleep between pushes. self.pushList = [] # Need to load push list from local file. self.ratesPerRequest = [5] # Last n ratesPerRequest self.previousRates = -1
def saveData(self):
BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
def newTorrentURLs(self, test=False):
# Find the latest torrent URLs, add them (paused) to Transmission and check their size.
excludeList = self.getExcludeList()
searchDict = {} # Empty dictionary
searchDict["URL"] = self.SHOW_URL
# searchDict['extension'] = self.EXTENSION
foundTorrents = PuxTorrents.torrentSearch(searchDict)
urlList = []
tempEpNum = -1
for t in foundTorrents: # tuples in foundTorrents
epNum = t[0]
url = t[1]
alreadyAdded = False
if epNum <= self.latestEpisode:
print("Already have episode %d" % epNum)
alreadyAdded = True
else:
for torrentDict in self.torrentList:
if torrentDict["URL"] == url:
print("Already added URL")
alreadyAdded = True
if not alreadyAdded:
print("Processing %s ep %d" % (self.SHOW_NAME, epNum))
# aa = BFun.tic()
# add_torrent() will return control once it has finished adding the torrent.
print("Getting .torrent...")
torrentRetrys = 0
gotTorrent = False
while torrentRetrys < 99:
try:
self.requestSleep += 1.1
print("Sleeping %0.1f seconds before retrieving torrent" % self.requestSleep)
time.sleep(self.requestSleep)
# r = requests.get(url,timeout=10)
fileLikeObject = urllib.request.urlopen(url, None, 30)
gotTorrent = True
break
except urllib.error.URLError as ex:
if "error timed out" in str(ex).lower():
torrentRetrys += 1
print("Timed out. Trying again. %d" % torrentRetrys)
else:
BFun.ezLog("Error when retrieving .torrent file. Search for 9217568", ex)
except Exception as ex:
BFun.ezLog("Error when retrieving .torrent file. Search for 3195897", ex)
if gotTorrent:
# Write data to .torrent file.
localURI = self.DATA_PATH + "/lastTorrent.torrent"
fo = open(localURI, "wb") # Gets overwritten.
# fo.write(r.content)
shutil.copyfileobj(fileLikeObject, fo)
fo.close()
try:
# .torrent file may be bad?
torrent = self.transmissionSession.add_torrent(localURI, paused=True)
hashString = torrent.hashString
torrentContents = self.transmissionSession.get_torrent(hashString).files()
except Exception as ex:
BFun.ezLog("Error when sending .torrent file to transmission. Search for 4523861", ex)
# Add bogus torrent to torrentList with URL and Completed.
# Skip the URL now but don't mark it so we can try it later.
torrentDict = {
"URL": "a",
"completed": 1,
"epNum": -epNum,
"hashString": "brokenURL",
"wantedFileIDs": [],
"comment": ["ep %d" % epNum, "brokenURL", url],
}
self.torrentList.append(torrentDict)
self.saveData()
torrentContents = [] # Empty list so it won't be processed
# Log what happened.
BFun.ezLog("Bad .torrent file from %s. (ep %d, %s)" % (url, epNum, localURI))
time.sleep(2)
else:
print("Didn't get torrent after 99 retrys.")
break
# Get the unique hash of the torrent so working with torrents is easier.
# tSess2.get_torrent(hashString).files()
fileInfos = []
wantedFiles = []
unwantedFiles = []
# Check if torrent has files we want.
desiredTorrent = False
for m in range(len(torrentContents)):
desiredFile = True
fileName = torrentContents[m]["name"]
fileSize = torrentContents[m]["size"]
nameSubs = fileName.split(".")
if nameSubs[-1].lower() in self.EXTENSION.lower(): # self.EXTENSION has the '.'
print("Processing %s" % fileName)
if fileSize > self.MAX_FILE_SIZE:
desiredFile = False
print("File %d too big" % m)
if fileSize < self.MIN_FILE_SIZE:
desiredFile = False
# print('File %d too small'%m)
for exSubString in excludeList:
if exSubString.lower() in fileName.lower():
desiredFile = False
print("Found %s in %s" % (exSubString, fileName))
else:
desiredFile = False
# print('File %d has wrong extension'%m)
if desiredFile:
desiredTorrent = True
wantedFiles.append(m)
else:
unwantedFiles.append(m)
if len(wantedFiles) > 8:
# If there are more than 8 files, it's a compliation torrent which we don't want.
# There may be week's worth of episodes which is ok.
desiredTorrent = False
print("Found %d files. Rejecting" % len(wantedFiles))
print("unwantedFiles:")
print(unwantedFiles)
# If torrentContents has no items, it's because it wasn't added. Check "unique hash" for human readable description
if len(torrentContents) > 0:
self.transmissionSession.change_torrent(hashString, files_unwanted=unwantedFiles)
self.transmissionSession.change_torrent(hashString, files_wanted=wantedFiles)
if desiredTorrent:
urlList.append(url)
# If it's a behind the scenes or something, don't change the actual episode number.
if epNum != 9999999:
print("Found episode %d" % epNum)
tempEpNum = max([epNum, tempEpNum]) # Largest number in the found torrents.
else:
epNum = -1
if test:
# We're just testing so remove the torrent.
self.transmissionSession.remove_torrent(hashString, delete_data=True)
else:
# Start the torrent
self.transmissionSession.start_torrent(hashString)
self.pendingTorrents = True
# Add to self.torrentList
torrentDict = {}
torrentDict["hashString"] = hashString
torrentDict["wantedFileIDs"] = wantedFiles
# 0 = still downloading. -1 = complete on Transmission, not moved to fileserver. 1 = moved to fileserver, removed from transmission
torrentDict["completed"] = 0
torrentDict["URL"] = url
torrentDict["epNum"] = epNum
self.torrentList.append(torrentDict)
BFun.jSaver(self.torrentList, self.TORRENT_LIST_PATH)
else:
# Not a torrent we want so remove from transmission.
if len(torrentContents) > 0:
self.transmissionSession.remove_torrent(hashString, delete_data=True)
# and Add bogus torrent to torrentList with URL and Completed so we don't download it again.
torrentDict = {
"URL": url,
"completed": 1,
"epNum": -epNum,
"hashString": "noFiles",
"wantedFileIDs": [],
}
self.torrentList.append(torrentDict)
self.saveData()
if tempEpNum > self.latestEpisode: # Update latest episode number
print("tempEpNum = %d" % tempEpNum)
self.latestEpisode = tempEpNum
# dataFile = open(self.LATEST_DATA_PATH,'w')
# dataFile.write('%d'%tempEpNum)
# dataFile.close()
print("Updated data file")
if test:
return urlList
else:
return urlList
import PuxGlobal
import BFun
import transmissionrpc
import re
import os
import time
import PuxTorrents
import shutil
import urllib.request
import requests
from PuxBulletf import PuxBullet
"""
Notes:
Save and load self.torrentList to a json with:
BFun.jSaver(someJSON,somePath)
someJSON = BFun.jLoader(somePath)
"""
# global PuxGlobal.globalRecentCompleteTorrents # List of dictionaries
# global PuxGlobal.globalCurrentTorrentsProgress # Dictionary of list of dictionaries
class PuxShow:
# Pre define some functions so things work?
# def findLatestEpisode(self):
# pass
# Class to handle show torrents, files on server etc.
def __init__(self, torrentLine, PuxBulletObject):
# Initialise PuxShow class
