def checkFirstSvtPage(firstPage, verbose):
haltOnError = True
if verbose:
printInfo2("Parsing page...")
soup = BeautifulSoup(firstPage)
items = soup.findAll(attrs={'data-json-href' : True})
try:
firstLink = items[0]['data-json-href']
except:
xmlCode = "Error"
return xmlCode
if verbose:
printInfo1("Found first link:")
print firstLink
else:
sys.stdout.write(".")
sys.stdout.flush()
linkOK, linkError = checkLink(firstLink, haltOnError, verbose)
if not linkOK:
onError(55,linkError)
else:
if verbose:
printInfo1("Link OK")
xmlCode = checkSecondSvtPage(firstLink, verbose)
return xmlCode
def rtmpdumpDownloadCommand(line, verbose):
if verbose:
printInfo2("Composing download command...")
if resolveHost:
url = domainToIPno(line['address'], verbose)
else:
url = line['address']
part1 = url.partition(' playpath=')
part2 = part1[2].partition(' swfVfy=1 swfUrl=')
if "kanal5play" in part2[2]:
if verbose:
printInfo1("This is from kanal5play\nAdding --live option to download command")
rtmpdumpOptions = "--realtime"
else:
rtmpdumpOptions = ""
cmd = (
"rtmpdump -o '%s.%s'"
" -r %s"
" -y %s"
" -W %s"
" %s"
% (line['name'].rstrip(), line['suffix'],
part1[0],
part2[0],
part2[2],
rtmpdumpOptions)
)
if verbose:
printInfo1("rtmpdump command: %s" % cmd)
return cmd
def parseXML(xmlRoot, url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose):
vidBitRate = 0
vidWidth = 0
currentQuality = 0
lastQuality = 0
downloads = []
if name == "null":
trys = 0
printInfo2("Getting page title to use as file name...")
while True:
trys += 1
if trys > maxTrys:
onError(10, "Tried connecting %s times. Giving up..." % (trys - 1))
if verbose:
printInfo1("%s%s try" % (trys, numbering(trys, verbose)))
try:
html = urllib2.urlopen(url)
except urllib2.HTTPError, e:
onError(35, "HTTPError\n %s\n Trying again...\n" % str(e.code))
sleep(waitTime)
except urllib2.URLError, e:
onError(36, "URLError\n %s\n Trying again...\n" % str(e.reason))
sleep(waitTime)
except:
def handleXML(xmlRoot, verbose):
for xmlChild in xmlRoot:
if verbose:
print "%s: %s" % (xmlChild.tag, xmlChild.text)
if 'language' in xmlChild.tag.lower():
language = xmlChild.text
printInfo1("\nLanguage: %s" % language)
if 'font' in xmlChild.tag.lower():
print "Here comes the subs"
print xmlChild
for line in xmlChild:
if 'subtitle' in line.tag.lower():
print line.attrib['SpotNumber']
print "%s --> %s" % (line.attrib['TimeIn'],
line.attrib['TimeOut'])
for text in line:
if 'text' in text.tag.lower():
print text.text
print
def retrievePiratePlayXML(apiBaseUrl, url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose):
xmlCode = ""
trys = 0
if verbose:
printInfo1("Using %s as source for getting XML" % apiBaseUrl)
if verbose:
printInfo2("Parsing the response from pirateplay.se API...")
parseUrl = "%s/%s%s" % (apiBaseUrl, getStreamsXML, url)
printInfo2("\nGetting streams for %s ..." % parseUrl)
printScores()
while True:
trys += 1
if trys > maxTrys:
onError(10, "Tried connecting %s times. Giving up..." % (trys - 1))
break
if verbose:
printInfo1("%s%s try" % (trys, numbering(trys, verbose)))
try:
piratePlayXML = urllib2.urlopen(parseUrl)
except urllib2.HTTPError, e:
onError(35, "HTTPError\n %s\n Trying again...\n" % str(e.code))
sleep(waitTime)
except urllib2.URLError, e:
onError(36, "URLError\n %s\n Trying again...\n" % str(e.reason))
sleep(waitTime)
def internalXMLGenerator(url, verbose):
haltOnError = True
linkOK, linkError = checkLink(url, haltOnError, verbose)
if not linkOK:
onError(58,linkError)
else:
if verbose:
printInfo1("Link OK")
firstPage = getWebPage(url, verbose)
if firstPage:
if "svtplay" in url.lower():
xmlCode = svtPlayXML(firstPage, verbose)
if xmlCode == "Error":
onError(66, "Not able to find link with internal XML-generator")
xmlCode = ""
else:
xmlCode = '\n'.join(xmlCode)
if verbose:
printInfo1("XML code:")
print xmlCode
return xmlCode
else:
onError(64, "Not able to run local python XML generator on this address")
xmlCode = ""
else:
onError(59, "Could not download webpage")
if not verbose:
print "\n"
return xmlCode
def parseURL(url, name, verbose):
printInfo1("\nSearching for '%s' in %s ..." % (videoText, url))
videos = getPages(url, verbose)
if videos:
makeList(videos, name, verbose)
else:
onError(19, "Could not find videos")
def getInfo(line, argument, verbose):
cmd = "mediainfo %s '%s.%s'" % (argument, line['name'].rstrip(), line['suffix'])
if verbose:
printInfo1("Command: %s" % cmd)
args = shlex.split(cmd)
process = Popen(args, stdout=PIPE, stderr=PIPE)
output, error = process.communicate()
if verbose:
printInfo1(output.rstrip())
return output.rstrip()
def ffmpegDownloadCommand(line, verbose):
if verbose:
printInfo2("Composing download command...")
ffmpeg = getffmpegPath(verbose)
if resolveHost:
url = domainToIPno(line['address'], verbose)
else:
url = line['address']
if ffmpeg == ffmpegPath:
if verbose:
cmd = (
"%s -i %s"
" -loglevel verbose -acodec copy -vcodec copy -absf aac_adtstoasc -timeout 1000"
" '%s.%s'"
% (ffmpeg,
url,
line['name'].rstrip(), line['suffix'])
)
else:
cmd = (
"%s -i %s"
" -stats -loglevel fatal -acodec copy -vcodec copy -absf aac_adtstoasc -timeout 1000"
" '%s.%s'"
% (ffmpeg,
url,
line['name'].rstrip(), line['suffix'])
)
elif ffmpeg == avconvPath:
if verbose:
cmd = (
"%s -i %s"
" -loglevel verbose -acodec copy -vcodec copy" # -absf aac_adtstoasc"
" '%s.%s'"
% (ffmpeg,
url,
line['name'].rstrip(), line['suffix'])
)
else:
cmd = (
"%s -i %s"
" -acodec copy -vcodec copy" # -absf aac_adtstoasc"
" '%s.%s'"
% (ffmpeg,
url,
line['name'].rstrip(), line['suffix'])
)
else:
onError(16, "You do not have either ffmpeg or avconv on the paths set in your config")
if verbose:
printInfo1("ffmpeg command: %s" % cmd)
return cmd
def readXML(xmlCode, verbose):
print
try:
xmlRoot = ET.fromstring(xmlCode)
except:
onError(6, "Not a valid XML")
else:
printInfo1("XML is valid")
handleXML(xmlRoot, verbose)
def makeList(videos, name, verbose):
printInfo1("\nCreating %s.%s..." % (name, listSuffix))
if continueWithProcess(name, listSuffix, True, False,
"Will redownload\n", "Keeping old file. No download\n", verbose):
listFile = codecs.open("%s.%s" % (name, listSuffix), "w", 'utf-8')
for video in videos:
if verbose:
printInfo1("\nURL: %s" % video['url'])
printInfo1("Season: %s" % video['season'])
printInfo1("Episode title: %s" % video['episodeTitle'])
listFile.write("%s\n" % video['url'])
listFile.write("%s.%s.%s\n\n" % (name,
video['season'],
video['episodeTitle']))
listFile.close()
def getPages(url, verbose):
gotAnswer = False
sourceCode = ""
pageNo = 1
videos = []
parsed_uri = urlparse( url )
domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
if verbose:
printInfo1("Domain is: %s" % domain)
while True:
printInfo1("\nGetting page %s..." % pageNo)
printInfo1("URL: %s" % "%s/?sida=%s" % (url, pageNo))
responseCode = getResponseCode("%s/?sida=%s" % (url, pageNo), verbose)
#if pageNo == 2:
# responseCode = 404
if responseCode != 404:
trys = 0
while True:
trys += 1
if verbose:
printInfo1("%s%s try" % (trys, numbering(trys, verbose)))
if trys > maxTrys:
onError(10, "Tried connecting %s times. Giving up..." % (trys - 1))
try:
source = urllib2.urlopen("%s/?sida=%s" % (url, pageNo))
except urllib2.HTTPError, e:
onError(23, "HTTPError\n %s\n Trying again...\n" % str(e.code))
sleep(waitTime)
except urllib2.URLError, e:
onError(24, "URLError\n %s\n Trying again...\n" % str(e.reason))
sleep(waitTime)
except:
def compareDurations(expectedDuration, actualDuration, verbose):
if verbose:
printInfo2("Comparing durations...")
printInfo1("Expected duration: %s s" % expectedDuration)
printInfo1("Actual duration: %s s" % actualDuration)
if expectedDuration == 0:
durationsMatch = True
printWarning("Expected duration was 0\nSkipping checking...")
else:
if actualDuration + 2 > expectedDuration and actualDuration - 2 < expectedDuration:
durationsMatch = True
else:
durationsMatch = False
return durationsMatch
def getSubSize(subAddress, checkDuration, verbose):
subSize = "0"
trys = 0
gotAnswer = False
if checkDuration:
if verbose:
printInfo2("Probing for size of subtitle file...")
while True:
trys += 1
if trys > maxTrys:
onError(26, "Giving up after %s trys" % (trys - 1))
printWarning("Setting subtitle size to %s" % subSize)
gotAnswer = True
break
try:
sub = urllib2.urlopen(subAddress)
except:
printInfo2("Could not get subtitle size")
onError(41, "Undefined error")
printInfo2("Trying again...")
else:
if verbose:
printInfo1("Got an answer")
meta = sub.info()
if meta.getheaders:
subSize = meta.getheaders("Content-Length")[0]
else:
onError(21, "Could not get headers")
printWarning("Setting subsize to %s", subSize)
gotAnswer = True
break
if gotAnswer:
break
else:
printWarning("Subsize check disabled")
printWarning("Setting subsize to %s" % subSize)
printInfo1("Sub size: %s B" % subSize)
return subSize
def composeXML(videos, subtitleLink, verbose):
xmlCode = []
if verbose:
printInfo2("Generating XML...")
printInfo1("Adding %s streams" % len(videos))
else:
sys.stdout.write(".")
sys.stdout.flush()
xmlCode.append("<streams>")
for index in range(0, len(videos)):
if verbose:
printInfo2("Adding video stream #%s..." % index)
print "Bitrate: %s" % videos[index]['reportedBitrate']
print "Video link: %s" % videos[index]['videoLink']
if subtitleLink:
print "Subtitle link: %s" % subtitleLink
print "Suffix hint: %s" % videos[index]['suffixHint']
else:
sys.stdout.write(".")
sys.stdout.flush()
if subtitleLink:
xmlCode.append(('<stream quality="%s kbps" subtitles="%s" suffix-hint="%s" required-player-version="0">') %
(videos[index]['reportedBitrate'],
subtitleLink,
videos[index]['suffixHint'])
)
else:
xmlCode.append(('<stream quality="%s kbps" suffix-hint="%s" required-player-version="0">') %
(videos[index]['reportedBitrate'],
videos[index]['suffixHint'])
)
xmlCode.append(videos[index]['videoLink'])
xmlCode.append('</stream>')
xmlCode.append('</streams>')
return xmlCode
def wgetDownloadCommand(line, verbose):
if verbose:
printInfo2("Composing download command...")
if resolveHost:
url = domainToIPno(line['subs'], verbose)
else:
url = line['subs']
cmd = (
"wget -O '%s.srt'"
" %s"
% (line['name'].rstrip(),
url)
)
if verbose:
printInfo1("wget command: %s" % cmd)
return cmd
def convertVideo(videoInFile, convertTo, reEncode, verbose):
keepOld = False
reDownload = False
fileAccepted = False
fileName, fileExtension = os.path.splitext(videoInFile)
fileExtension = fileExtension.lstrip(".")
if verbose:
printInfo1("File name: %s" % fileName)
printInfo1("File extension: %s" % fileExtension)
for extension in videoExtensions:
if extension == fileExtension.lower():
fileAccepted = True
if not fileAccepted:
onError(15, "%s is probably not a video file" % videoInFile)
if fileExtension.lower() == convertTo:
printWarning("Same out format chosen as existing\nWill not convert")
else:
if verbose:
printInfo2("Converting %s to %s format" % (videoInFile, convertTo))
printInfo2("Renaming %s to %s.bak" % (videoInFile, videoInFile))
os.rename(videoInFile, "%s.bak" % videoInFile)
if fileExtension.lower() == "flv" and convertTo == "mp4":
ffmpeg = getffmpegPath(verbose)
if reEncode:
if verbose:
printInfo2("Reencoding video...")
cmd = ("%s"
" -i %s"
" -qscale 0 -ar 22050 -vcodec %s"
" '%s.%s'"
% (ffmpeg,
"%s.bak" % videoInFile,
videoCodec,
fileName, convertTo)
)
else:
cmd = ("%s"
" -i %s -vcodec copy -acodec copy"
" '%s.%s'"
% (ffmpeg,
"%s.bak" % videoInFile,
fileName, convertTo)
)
while True:
printInfo1("Will convert")
if continueWithProcess(fileName, fileExtension, keepOld, reDownload,
"Will re convert\n", "Keeping old file\nNo converting\n", verbose):
process = runProcess(cmd, verbose)
def checkFileSize(line, verbose):
printScores()
expectedFileSize = int(line['subSize'])
downloadedFileSize = os.path.getsize("%s.srt" % line['name'].rstrip())
printInfo1("Expected file size: %d B" % (expectedFileSize))
printInfo1("Downloaded file size: %d B" % (downloadedFileSize))
if expectedFileSize == 0:
FileSizesMatch = True
printWarning("Expected file size was 0\nSkipping checking...")
else:
if downloadedFileSize + 2 > expectedFileSize and downloadedFileSize - 2 < expectedFileSize:
FileSizesMatch = True
if verbose:
printInfo1("File sizes match")
else:
FileSizesMatch = False
if verbose:
printWarning("File sizes does not match")
return FileSizesMatch
def getDownloadCommands(line, verbose):
subCmd = ""
if verbose:
printInfo2("Composing download commands for line...")
print line
if line['address'].startswith("http"):
if verbose:
printInfo1("This should be downloaded with ffmpeg")
videoCmd = ffmpegDownloadCommand(line, verbose)
elif line['address'].startswith("rtmp"):
printInfo1("This should be downloaded with rtmpdump")
videoCmd = rtmpdumpDownloadCommand(line, verbose)
if line['subs']:
if verbose:
printInfo1("This should be downloaded with wget")
subCmd = wgetDownloadCommand(line, verbose)
return videoCmd, subCmd
def checkDurations(line, verbose):
printScores()
expectedDuration = round(float(line['duration']), 2)
downloadedDuration = float(getInfo(line, '--Inform="General;%Duration%"', verbose)) / 1000
printInfo1("Expected duration: %d s (%s)" % (expectedDuration, str(datetime.timedelta(seconds=expectedDuration))))
printInfo1("Downloaded duration: %d s (%s)" % (downloadedDuration, str(datetime.timedelta(seconds=downloadedDuration))))
if expectedDuration == 0:
durationsMatch = True
printWarning("Expected duration was 0\nSkipping checking...")
else:
if downloadedDuration + 2 > expectedDuration and downloadedDuration - 2 < expectedDuration:
durationsMatch = True
if verbose:
printInfo1("Durations match")
else:
durationsMatch = False
if verbose:
printWarning("Durations does not match")
return durationsMatch
def getResponseCode(url, verbose):
trys = 0
responseCode = ""
if verbose:
printInfo1("Getting response code...")
while True:
trys += 1
if verbose:
printInfo1("%s%s try" % (trys, numbering(trys, verbose)))
if trys > maxTrys:
onError(22, "Tried %s times" % (trys - 1))
try:
r = requests.head(url)
except requests.ConnectionError:
onError(20, "Failed to connect\nTrying again...")
else:
responseCode = r.status_code
if verbose:
printInfo1("Response: %s" % responseCode)
break
return responseCode
def checkVideoLink(videoLink, verbose):
index = 0
linkOK = False
checkQuality = True
haltOnError = False
videos = []
resolution = "0 x 0"
bitrate = 0
correctLinkSuffix = "/manifest.f4m"
oldPattern = "/z/"
newPattern = "/i/"
reportedBitrates = findReportedBitrates(videoLink, verbose)
if verbose:
printInfo2("Checking video link...")
else:
sys.stdout.write(".")
sys.stdout.flush()
if videoLink.endswith(correctLinkSuffix):
LinkOK = True
if verbose:
printInfo1("Link ending OK")
printInfo2("Stripping off suffix %s" % correctLinkSuffix)
else:
sys.stdout.write(".")
sys.stdout.flush()
videoLink = videoLink.rstrip(correctLinkSuffix)
if verbose:
printInfo1("New link:")
print videoLink
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
onError(61, "Link did not end with %s" % correctLinkSuffix)
if verbose:
printInfo2("Looking for valid video links...")
else:
sys.stdout.write(".")
sys.stdout.flush()
while True:
testLink = videoLink.replace(oldPattern, newPattern)
testLink = "%s/index_%s_av.m3u8?null=0" % (testLink, index)
if verbose:
printInfo2("Checking video link #%s..." % index)
print "%s/index_%s_av.m3u8?null=0" % (testLink, index)
else:
sys.stdout.write(".")
sys.stdout.flush()
linkOK , linkError = checkLink(testLink, haltOnError, verbose)
if not linkOK:
if verbose:
printWarning("Did not get an answer for link #%s" % index)
else:
sys.stdout.write(".")
sys.stdout.flush()
break
else:
if checkQuality:
resolution, bitrate, codecLongName = findQuality(testLink, verbose)
if "mpeg-4" in codecLongName.lower() or "h.264" in codecLongName.lower():
suffixHint = "mp4"
else:
suffixHint = "unknown_suffix"
if verbose:
printInfo1("Resolution:")
print resolution
printInfo1("Bit rate: ")
print bitrate
printInfo1("Codec long name: ")
print codecLongName
printInfo1("Suffix hint: ")
print suffixHint
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
printWarning("Not checking quality")
if verbose:
printInfo2("Adding video...")
else:
sys.stdout.write(".")
sys.stdout.flush()
videos.append({"videoLink" : testLink,
"resolution" : resolution,
"bitrate" : bitrate,
"reportedBitrate" : reportedBitrates[index],
"codecLongName" : codecLongName,
"suffixHint": suffixHint})
index += 1
return videos
def findQuality(url, verbose):
width = 0
height = 0
bitrate = 0
codecLongName = ""
trys = 0
noFFmpeg = False
ffprobe = getffprobePath(verbose)
if verbose:
printInfo2("Looking up quality for stream with %s..." % ffprobe)
else:
sys.stdout.write(".")
sys.stdout.flush()
if ffprobe == ffprobePath:
if verbose:
printInfo1("Using %s to get video information" % ffprobePath)
cmd = "%s -loglevel error -show_format -show_streams %s -print_format xml" % (ffprobe, url)
if verbose:
printInfo1("Command: %s\n" % cmd)
args = shlex.split(cmd)
while True:
trys += 1
if trys > maxTrys:
onError(38, "Giving up after %s trys" % (trys - 1))
printWarning("Setting bitrate to %s" % bitrate)
gotAnswer = True
gotXML = True
break
while True:
try:
process = Popen(args, stdout=PIPE, stderr=PIPE)
except OSError as e:
onError(39, "%s\nYou are probably missing ffmpeg" % e)
noFFmpeg = True
break
else:
if verbose:
printInfo1("Got an answer")
output, error = process.communicate()
gotAnswer = True
break
if not noFFmpeg:
try:
xmlRoot = ET.fromstring(output)
except:
onError(43, "Did not receive a valid XML")
printInfo2("Trying again...")
else:
if verbose:
printInfo1("Downloaded a valid XML:")
print output
for xmlChild in xmlRoot:
if 'bit_rate' in xmlChild.attrib:
bitrate = xmlChild.attrib['bit_rate']
for innerChild in xmlChild:
if 'codec_long_name' in innerChild.attrib and not codecLongName:
codecLongName = innerChild.attrib['codec_long_name']
if 'width' in innerChild.attrib and not width:
width = innerChild.attrib['width']
if 'height' in innerChild.attrib and not height:
height = innerChild.attrib['height']
gotXML = True
else:
onError(40, "Can not detect duration")
printWarning("Setting bitrate to %s" % bitrate)
gotAnswer = True
gotXML = True
if gotAnswer and gotXML:
break
else:
if verbose:
printInfo1("Using %s to get video information" % avprobePath)
cmd = "%s -loglevel error -show_format -show_streams %s -of json" % (ffprobe, url)
args = shlex.split(cmd)
while True:
trys += 1
if trys > maxTrys:
onError(38, "Giving up after % trys" % (trys - 1))
gotAnswer = True
gotXML = True
break
while True:
try:
process = Popen(args, stdout=PIPE, stderr=PIPE)
except OSError as e:
onError(39, "%s\nYou are probably missing ffmpeg" % e)
noFFmpeg = True
break
else:
if verbose:
printInfo1("Got an answer")
else:
sys.stdout.write(".")
sys.stdout.flush()
output, error = process.communicate()
gotAnswer = True
break
if gotAnswer:
break
if gotAnswer:
jsonString = json.loads(output)
if verbose:
print "Full json: %s" % json.dumps(jsonString, sort_keys=True, indent=2)
else:
sys.stdout.write(".")
sys.stdout.flush()
if json.dumps(jsonString['streams'][0]['width']):
width = json.dumps(jsonString['streams'][0]['width'])
if verbose:
print "Width: %s" % width
else:
sys.stdout.write(".")
sys.stdout.flush()
if json.dumps(jsonString['streams'][0]['height']):
height = json.dumps(jsonString['streams'][0]['height'])
if verbose:
print "Height: %s" % height
else:
sys.stdout.write(".")
sys.stdout.flush()
if json.dumps(jsonString['format']['bit_rate']):
bitrate = json.dumps(jsonString['format']['bit_rate'])
if verbose:
print "Bitrate: %s" % bitrate
else:
sys.stdout.write(".")
sys.stdout.flush()
if json.dumps(jsonString['streams'][0]['codec_long_name']):
codecLongName = json.dumps(jsonString['streams'][0]['codec_long_name'])
if verbose:
print "Codec long name: %s" % codecLongName
else:
sys.stdout.write(".")
sys.stdout.flush()
if width and height:
if verbose:
printInfo1("Found both width and height")
print "%s x %s" % (width, height)
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
if verbose:
printWarning("Could not find width and height")
else:
printWarning("\nCould not find width and height")
if bitrate:
if verbose:
printInfo1("Found bitrate")
print bitrate
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
if verbose:
printWarning("Could not find bitrate")
else:
printWarning("\nCould not find bitrate")
if codecLongName:
if verbose:
printInfo1("Found codec long name")
print codecLongName
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
if verbose:
printWarning("Could not find codec long name")
else:
printWarning("\nCould not codec long name")
return ("%s x %s" % (width, height), bitrate, codecLongName)
def findVideos(domain, sourceCode, videos, verbose):
gotSeason = False
gotURL = False
printInfo1("\nSearching for links in code with the word '%s' in the url..." % videoText)
soup = BeautifulSoup(sourceCode)
for item in soup.fetch(['h2', 'a']):
if verbose:
printInfo1("\nParsing line: %s\n..." % item)
if item.contents:
if item.name == "h2" and seasonText in item.contents[0]:
season = HTMLParser().unescape(item.contents[0])
if verbose:
printInfo2("Found season text")
printInfo1("Season: %s" % season)
gotSeason = True
if item.name == "a" and videoText in item['href']:
episodeTitle = HTMLParser().unescape(item['title'])
url = item['href']
if verbose:
printInfo2("Found link to video")
printInfo1("Episode title: %s" % episodeTitle)
printInfo1("URL: %s" % url)
gotURL = True
if not gotSeason and not gotURL:
if verbose:
printInfo2("No valuable info in this item")
if gotURL:
if not gotSeason:
season = "None"
url = urljoin(domain, url)
if verbose:
printInfo1("Adding...")
printInfo1("URL: %s" % url)
printInfo1("Season: %s" % season)
printInfo1("Episode title: %s" % episodeTitle)
videos.append({'url': url,
'season': season,
'episodeTitle': episodeTitle})
gotSeason = False
gotURL = False
printInfo1("Found %s videos" % len(videos))
return videos
if trys > maxTrys:
onError(10, "Tried connecting %s times. Giving up..." % (trys - 1))
try:
source = urllib2.urlopen("%s/?sida=%s" % (url, pageNo))
except urllib2.HTTPError, e:
onError(23, "HTTPError\n %s\n Trying again...\n" % str(e.code))
sleep(waitTime)
except urllib2.URLError, e:
onError(24, "URLError\n %s\n Trying again...\n" % str(e.reason))
sleep(waitTime)
except:
onError(25, "Error\n Trying again...\n")
sleep(waitTime)
else:
if verbose:
printInfo1("Got answer")
gotAnswer = True
break
pageNo += 1
if gotAnswer:
sourceCode = source.read()
videos = findVideos(url, sourceCode, videos, verbose)
else:
printInfo1("\nNo more videos")
break
return videos
def findVideos(domain, sourceCode, videos, verbose):
gotSeason = False
gotURL = False
def generateDownloads(url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose):
xmlCode = ""
xmlRoot = ""
if name != name.replace("'", "").replace('"', '').replace(":", ""):
name = name.replace("'", "").replace('"', '').replace(":", "")
if verbose:
printWarning("Removed quotes , double quotes and colons in out file name")
if verbose:
printInfo2("Getting XML...")
s = 0
printInfo2("Trying these sources one by one:")
for xmlSource in xmlPriorityOrder:
s += 1
print "%s: %s" % (s, xmlSource)
for xmlSource in xmlPriorityOrder:
##### try internal XML generator #####
if xmlSource == "internal":
if verbose:
printInfo2("Getting XML from this programs python xml generator...")
xmlCode = internalXMLGenerator(url, verbose)
if xmlCode:
break
##### try local pirateplay's API #####
elif xmlSource == "localPiratePlay":
printInfo2("Getting XML from local pirateplay's API...")
xmlCode = retrievePiratePlayXML(apiBaseUrlLocal, url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose)
if xmlCode:
break
##### try pirateplay's API #####
elif xmlSource == "piratePlay":
printInfo2("Getting XML from pirateplay.se's API...")
xmlCode = retrievePiratePlayXML(apiBaseUrlPiratePlay, url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose)
if xmlCode:
break
##### try svtplay-dl #####
elif xmlSource == "svtPlayDl":
printInfo2("Getting XML from svtplay-dl...")
xmlCode = svtplaydlXML(url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose)
if xmlCode:
break
##### giving up if no XML #####
if not xmlCode:
onError(69, "Could not find any way to get XML\nGiving up\nExiting!")
if verbose:
printInfo1("XML code:")
print xmlCode
try:
xmlRoot = ET.fromstring(xmlCode)
except:
onError(42, "Did not receive a valid XML")
#printInfo2("Trying again...")
else:
if verbose:
printInfo1("Downloaded a valid XML")
downloads = parseXML(xmlRoot, url, name, fileInfo, downloadAll, setQuality, bestQuality, checkDuration, verbose)
if len(downloads) == 0:
onError(67, "Did not find any suitable streams")# \nTrying another method...")
sys.exit()
return downloads
def checkSecondSvtPage(url, verbose):
secondTag = ""
secondPage = getWebPage(url, verbose)
if verbose:
printInfo2("Parsing page...")
else:
sys.stdout.write(".")
sys.stdout.flush()
soup = BeautifulSoup(secondPage)
items = soup.findAll("embed", attrs={'attr' : True})
secondTag = items[0]['attr']
if secondTag:
if verbose:
printInfo1("Found second tag:")
print secondTag
printInfo2("Decoding...")
else:
sys.stdout.write(".")
sys.stdout.flush()
secondTag = urllib.unquote(secondTag.encode('utf8')).decode('utf8')
if verbose:
printInfo1("Decoded tag:")
print secondTag
secondTag = secondTag.split('=', 1)[-1]
else:
printError("Did not find second tag")
if verbose:
printInfo2("Converting to json...")
else:
sys.stdout.write(".")
sys.stdout.flush()
jsonString = json.loads(secondTag)
if verbose:
printInfo1("JSON string:")
print json.dumps(jsonString['video'], sort_keys=True, indent=2)
printInfo2("Extracting video link...")
else:
sys.stdout.write(".")
sys.stdout.flush()
videoLink = jsonString['video']['videoReferences'][0]['url']
if verbose:
printInfo1("Found video link:")
print videoLink
else:
sys.stdout.write(".")
sys.stdout.flush()
videos = checkVideoLink(videoLink, verbose)
if verbose:
printInfo2("Extracting subtitle link...")
else:
sys.stdout.write(".")
sys.stdout.flush()
if "url" in jsonString['video']['subtitleReferences'][0]:
subtitleLink = jsonString['video']['subtitleReferences'][0]['url']
checkSubtitleLink(subtitleLink, verbose)
if verbose:
printInfo1("Found subtitle link:")
print subtitleLink
else:
sys.stdout.write(".")
sys.stdout.flush()
else:
if verbose:
printWarning("No subtitles found")
else:
sys.stdout.write(".")
sys.stdout.flush()
subtitleLink = ""
if verbose:
printInfo1("Found videos:")
for video in videos:
print video
else:
sys.stdout.write(".")
sys.stdout.flush()
xmlCode = composeXML(videos, subtitleLink, verbose)
return xmlCode
if verbose:
printInfo1("%s%s try" % (trys, numbering(trys, verbose)))
try:
html = urllib2.urlopen(url)
except urllib2.HTTPError, e:
onError(35, "HTTPError\n %s\n Trying again...\n" % str(e.code))
sleep(waitTime)
except urllib2.URLError, e:
onError(36, "URLError\n %s\n Trying again...\n" % str(e.reason))
sleep(waitTime)
except:
onError(37, "Error\n Trying again...\n")
sleep(waitTime)
else:
if verbose:
printInfo1("Got answer")
soup = BeautifulSoup(html)
name = soup.title.string.encode('utf-8')
if verbose:
printInfo1("Setting name to %s" % name)
break
for xmlChild in xmlRoot:
if 'quality' in xmlChild.attrib:
quality = xmlChild.attrib['quality']
printInfo1("\nQuality: %s" % quality)
else:
quality = "null"
currentQuality = 1
printWarning("No quality stated: %s" % quality)
def finish(downloads, keepOld, reDownload, checkDuration, listOnly, convertTo, bashOutFile, verbose):
shouldBeDeleted = []
if not listOnly:
if downloads:
infoDownloaded = getVideos(downloads, keepOld, reDownload, checkDuration, verbose)
if convertTo:
if verbose:
printInfo2("Converting downloads...")
convertDownloads(downloads, convertTo, verbose)
else:
infoDownloaded = ""
onError(17, "Could not find any streams to download")
else:
infoDownloaded = ""
printInfo1("\nListing only")
printScores()
if bashOutFile:
if continueWithProcess(bashOutFile, bashSuffix, True, False,
"Will redownload\n", "Keeping old file. No download\n", verbose):
bashFile = open("%s.%s" % (bashOutFile, bashSuffix), "w")
bashFile.write("#!/bin/bash\n\n")
if downloads:
printInfo1("These files would have been downloaded:")
for line in downloads:
# print line
printInfo1("\nVideo name: %s.%s" % (line['name'].rstrip(), line['suffix']))
printInfo1("Video quality: %s" % line['quality'])
printInfo1("Video address: %s" % line['address'])
if line['subs']:
printInfo1("Subtitles name: %s.srt" % line['name'].rstrip())
printInfo1("Subtitles address: %s" % line['subs'])
else:
printInfo1("No subtitles found")
print "Duration: %s s" % line['duration']
if bashOutFile:
if line['address'].startswith("http"):
cmd = ffmpegDownloadCommand(line, verbose)
bashFile.write("%s\n\n" % cmd)
elif line['address'].startswith("rtmp"):
cmd = rtmpdumpDownloadCommand(line, verbose)
bashFile.write("%s\n\n" % cmd)
if line['subs']:
cmd = wgetDownloadCommand(line, verbose)
bashFile.write("%s\n\n" % cmd)
if bashOutFile:
bashFile.close()
st = os.stat("%s.sh" % bashOutFile)
os.chmod("%s.sh" % bashOutFile, st.st_mode | stat.S_IEXEC)
else:
printWarning("Could not find anything that would have been downloaded")
for line in infoDownloaded:
printInfo1("\nVideo: %s" % line['videoName'])
printScores()
if line['expectedDuration'] != "0.000":
printInfo1("Expected duration: %s" % (str(datetime.timedelta(seconds=round(float(line['expectedDuration']), 2)))))
if verbose:
printInfo1("Duration: %s ms" % line['duration'])
printInfo1("Duration: %s" % line['durationFormatted'])
if line['videoDlComment'] == dlCommentError:
printError(line['videoDlComment'])
shouldBeDeleted.append(line['videoName'])
else:
printInfo2(line['videoDlComment'])
if verbose:
printInfo1("Expected duration: %s s" % line['expectedDuration'])
printInfo1("Actual duration: %s s" % line['duration'])
if not compareDurations(float(line['expectedDuration']), float(line['duration']) / 1000, verbose):
printError("Durations does not match")
shouldBeDeleted.append(line['videoName'])
if verbose:
printInfo1("File size: %s b" % line['fileSize'])
printInfo1("File size: %s" % line['fileSizeMeasure'])
if verbose:
printInfo1("Overall bit rate: %s bps" % line['overallBitRate'])
printInfo1("Overall bit rate: %s" % line['overallBitRateMeasure'])
print
if verbose:
printInfo1("Video format: %s" % line['videoFormat'])
printInfo1("Video codec ID: %s" % line['videoCodecId'])
printInfo1("Video bit rate: %s bps" % line['videoBitRate'])
printInfo1("Video bit rate: %s" % line['videoBitRateMeasure'])
printInfo1("Width: %s px" % line['width'])
printInfo1("Height: %s px" % line['height'])
printInfo1("Frame rate: %s fps" % line['frameRate'])
if verbose:
printInfo1("Frame count: %s" % line['frameCount'])
print
printInfo1("Audio format: %s" % line['audioFormat'])
printInfo1("Audio codec ID: %s" % line['audioCodecId'])
printInfo1("Audio bit rate: %s bps" % line['audioBitRate'])
printInfo1("Audio bit rate: %s" % line['audioBitRateMeasure'])
if line['subLines'] != 'na':
printInfo1("\nSubtitles: %s" % line['subName'])
printScores()
if line['subDlComment'] == dlCommentError:
printError(line['subDlComment'])
shouldBeDeleted.append(line['subName'])
else:
printInfo2(line['subDlComment'])
if line['expectedSubSize'] != "0":
printInfo1("Expected file size: %s B" % line['expectedSubSize'])
printInfo1("File size: %s B" % line['subSize'])
printInfo1("Number of lines: %s" % line['subLines'])
else:
printWarning("\nNo subtitles downloaded")
if shouldBeDeleted:
print
printError("\nThese files should be deleted and re downloaded")
printScores()
for line in shouldBeDeleted:
printWarning(line)
print
def getDuration(stream, checkDuration, verbose):
duration = "0.000"
gotAnswer = False
gotXML = False
noFFmpeg = False
trys = 0
ffprobe = getffprobePath(verbose)
if not ffprobe:
if verbose:
printWarning("Disabling checking of duration")
checkDuration = False
if checkDuration:
if verbose:
printScores()
printInfo2("Probing for duration of stream...")
if ffprobe == ffprobePath:
if verbose:
printInfo2("Using ffprope to determin duration...")
cmd = "%s -loglevel error -show_format -show_streams %s -print_format xml" % (ffprobe, stream)
if verbose:
printInfo1("Command: %s\n" % cmd)
args = shlex.split(cmd)
while True:
trys += 1
if trys > maxTrys:
onError(38, "Giving up after %s trys" % (trys - 1))
printWarning("Setting duration to %s" % duration)
gotAnswer = True
gotXML = True
break
while True:
try:
process = Popen(args, stdout=PIPE, stderr=PIPE)
except OSError as e:
onError(39, "%s\nYou are probably missing ffmpeg" % e)
noFFmpeg = True
break
else:
if verbose:
printInfo1("Got an answer")
output, error = process.communicate()
gotAnswer = True
break
if not noFFmpeg:
try:
xmlRoot = ET.fromstring(output)
except:
onError(43, "Did not receive a valid XML")
printInfo2("Trying again...")
else:
if verbose:
printInfo1("Downloaded a valid XML")
print output
for xmlChild in xmlRoot:
if 'duration' in xmlChild.attrib:
duration = xmlChild.attrib['duration']
if verbose:
printInfo1("Found duration in XML: %s" % duration)
gotXML = True
if not duration and verbose:
printWarning("Could not find duration in XML")
else:
onError(40, "Can not detect duration")
printWarning("Setting duration to %s" % duration)
gotAnswer = True
gotXML = True
if gotAnswer and gotXML:
break
if verbose:
printScores()
else:
printInfo2("Using avprobe to determine duration...")
cmd = "%s -loglevel error -show_format -show_streams %s -of json" % (ffprobe, stream)
if verbose:
printInfo1("Command: %s\n" % cmd)
args = shlex.split(cmd)
while True:
trys += 1
if trys > maxTrys:
onError(38, "Giving up after % trys" % (trys - 1))
gotAnswer = True
gotXML = True
break
while True:
try:
process = Popen(args, stdout=PIPE, stderr=PIPE)
except OSError as e:
onError(39, "%s\nYou are probably missing ffmpeg" % e)
noFFmpeg = True
break
else:
if verbose:
printInfo1("Got an answer")
output, error = process.communicate()
gotAnswer = True
break
if gotAnswer:
break
if gotAnswer:
jsonString = json.loads(output)
if json.dumps(jsonString['format']['duration']):
duration = (json.dumps(jsonString['format']['duration']).strip('"'))
if verbose:
print "Duration: %s" % duration
else:
printWarning("Duration check disabled")
printWarning("Setting duration to %s" % duration)
printInfo1("Duration: %s s (%s)" % (duration.rstrip("0").rstrip("."),
str(datetime.timedelta(seconds=round(float(duration), 2))))
)
return duration
def usage(exitCode):
printInfo1("\nUsage:")
printScores()
printInfo1("%s -u <url>" % sys.argv[0])
printInfo1(" Parse <url> for streamed video")
def getVideos(downloads, keepOld, reDownload, checkDuration, verbose):
oldReDownload = reDownload
printInfo2("\nStarting downloads")
for line in downloads:
trys = 0
videoCmd, subCmd = getDownloadCommands(line, verbose)
while True:
trys += 1
if trys > maxTrys:
onError(29, "Tried to download video %s times\nSkipping..." % (trys - 1))
videoComment = dlCommentError
break
print
printInfo2("Downloading stream to %s.%s ..." % (line['name'].rstrip(), line['suffix']))
expectedDuration = round(float(line['duration']), 2)
printInfo1("Expected duration: %s" % str(datetime.timedelta(seconds=expectedDuration)))
printInfo1("%s%s try" % (trys, numbering(trys, verbose)))
printScores()
if continueWithProcess(line['name'].rstrip(), line['suffix'], keepOld, reDownload,
"Will redownload\n", "Keeping old file. No download\n", verbose):
if verbose:
printInfo2("Executing download command...")
print videoCmd
exitCode = runProcess(videoCmd, verbose)
if exitCode != 0:
printScores()
onError(30, "Failed. Process exited on %s" % exitCode)
printInfo2("Trying again...")
reDownload = True
else:
if checkDuration and float(line['duration']) > 0:
durationOK = checkDurations(line, verbose)
else:
if verbose:
printWarning("Not checking duration")
durationOK = True
if os.path.isfile("%s.%s" % (line['name'].rstrip(), line['suffix'])) and durationOK:
printScores()
printInfo1("Finished downloading video")
setPerms("%s.%s" % (line['name'].rstrip(), line['suffix']), verbose)
reDownload = oldReDownload
videoComment = dlCommentSuccess
break
else:
printScores()
if not durationOK:
onError(46, "Durations does not match")
printInfo2("Trying again")
else:
onError(31, "Failed. Video file does not exist")
printInfo2("Trying again...")
reDownload = True
else:
videoComment = dlCommentExist
break
if subCmd:
trys = 0
oldReDownload = reDownload
while True:
trys += 1
if trys > maxTrys:
onError(32, "Tried to download subtitles %s times\nSkipping..." % (trys - 1))
subComment = dlCommentError
break
print
printInfo2("Downloading subtitles %s.srt ..." % line['name'].rstrip())
if verbose:
printInfo1("from %s" % line['subs'])
printInfo1("%s%s try" % (trys, numbering(trys, verbose)))
printScores()
if continueWithProcess(line['name'].rstrip(), "srt", keepOld, reDownload,
"Will redownload\n", "Keeping old file. No download\n", verbose):
if verbose:
printInfo2("Downloading file...")
print line['subs']
print videoCmd
result = downloadFile(line['subs'], "%s.%s" % (line['name'].rstrip(), "srt"), verbose)
if not result:
printScores()
onError(33, "Failed to download subtitles")
printInfo2("Trying again...")
reDownload = True
else:
fileSizeOK = checkFileSize(line, verbose)
if os.path.isfile("%s.srt" % line['name'].rstrip()) and fileSizeOK:
printScores()
printInfo1("Finished downloading subtitles")
setPerms("%s.srt" % line['name'].rstrip(), verbose)
reDownload = oldReDownload
subComment = dlCommentSuccess
break
else:
printScores()
if not fileSizeOK:
onError(47, "Failed. File sizes does not match")
printInfo2("Trying again")
else:
onError(34, "Failed. Subtitle file does not exist")
printInfo2("Trying again")
reDownload = True
else:
subComment = dlCommentExist
break
else:
subComment = dlCommentNoSub
printInfo2("\nGetting file info...")
fileSize = getInfo(line, '--Inform="General;%FileSize%"', verbose)
fileSizeMeasure = getInfo(line, '--Inform="General;%FileSize/String%"', verbose)
duration = getInfo(line, '--Inform="General;%Duration%"', verbose)
durationFormatted = getInfo(line, '--Inform="General;%Duration/String3%"', verbose)
overallBitRate = getInfo(line, '--Inform="General;%OverallBitRate%"', verbose)
overallBitRateMeasure = getInfo(line, '--Inform="General;%OverallBitRate/String%"', verbose)
videoFormat = getInfo(line, '--Inform="Video;%Format%"', verbose)
videoCodecId = getInfo(line, '--Inform="Video;%CodecID%"', verbose)
videoBitRate = getInfo(line, '--Inform="Video;%BitRate%"', verbose)
videoBitRateMeasure = getInfo(line, '--Inform="Video;%BitRate/String%"', verbose)
width = getInfo(line, '--Inform="Video;%Width%"', verbose)
height = getInfo(line, '--Inform="Video;%Height%"', verbose)
frameRate = getInfo(line, '--Inform="Video;%FrameRate%"', verbose)
frameCount = getInfo(line, '--Inform="Video;%FrameCount%"', verbose)
audioFormat = getInfo(line, '--Inform="Audio;%Format%"', verbose)
audioCodecId = getInfo(line, '--Inform="Audio;%CodecID%"', verbose)
audioBitRate = getInfo(line, '--Inform="Audio;%BitRate%"', verbose)
audioBitRateMeasure = getInfo(line, '--Inform="Audio;%BitRate/String%"', verbose)
if line['subs']:
subSize = os.path.getsize("%s.srt" % line['name'].rstrip())
with open("%s.srt" % line['name'].rstrip()) as myfile:
subLines = sum(1 for line in myfile) # number of lines in file
myfile.close() # close file
else:
subSize = "na"
subLines = "na"
infoDownloaded.append({'videoName': "%s.%s" % (line['name'].rstrip(), line['suffix']),
'expectedDuration': line['duration'],
'videoDlComment': videoComment,
'fileSize': fileSize,
'fileSizeMeasure': fileSizeMeasure,
'duration': duration,
'durationFormatted': durationFormatted,
'overallBitRate': overallBitRate,
'overallBitRateMeasure': overallBitRateMeasure,
'videoFormat': videoFormat,
'videoCodecId': videoCodecId,
'videoBitRate': videoBitRate,
'videoBitRateMeasure': videoBitRateMeasure,
'width': width,
'height': height,
'frameRate': frameRate,
'frameCount': frameCount,
'audioFormat': audioFormat,
'audioCodecId': audioCodecId,
'audioBitRate': audioBitRate,
'audioBitRateMeasure': audioBitRateMeasure,
'subName': "%s.srt" % line['name'].rstrip(),
'expectedSubSize': line['subSize'],
'subDlComment': subComment,
'subSize': subSize,
'subLines': subLines})
printScores()
print
return infoDownloaded
