def download(db, grabber, downType):
progress = Utils.getProgress()
name = Utils.httpFilename(RAIUrls.info)
folder = Config.tgFolder
localName = os.path.join(folder, name)
f = Utils.download(grabber, progress, RAIUrls.info, localName, downType, "utf-8", True)
process(grabber, progress, downType, f, db)
def download(db, grabber, downType):
progress = Utils.getProgress()
name = Utils.httpFilename(RAIUrls.info)
folder = Config.tgFolder
localName = os.path.join(folder, name)
f = Utils.download(grabber, progress, RAIUrls.info, localName, downType,
"utf-8", True)
process(grabber, progress, downType, f, db)
def download(db, grabber, downType):
page = Utils.httpFilename(RAIUrls.junior)
folder = Config.juniorFolder
localFilename = os.path.join(folder, page)
progress = Utils.getProgress()
f = Utils.download(grabber, progress, RAIUrls.junior, localFilename, downType, None, True)
process(grabber, progress, folder, f, db, downType)
def download(db, grabber, downType):
page = Utils.httpFilename(RAIUrls.onDemand)
folder = Config.demandFolder
localFilename = os.path.join(folder, page)
progress = Utils.getProgress()
f = Utils.download(grabber, progress, RAIUrls.onDemand, localFilename, downType, "raw-unicode-escape", True)
process(grabber, f, db)
def download(db, grabber, downType):
progress = Utils.getProgress()
for channel in channels:
url = getCatalogueUrl(channel)
name = Utils.httpFilename(url) + "." + channel
folder = Config.m6Folder
localName = os.path.join(folder, name)
f = Utils.download(grabber, progress, url, localName, downType, "utf-8", True)
if (f):
process(grabber, downType, f, channel, db)
def download(db, grabber, downType):
progress = Utils.getProgress()
folder = Config.tf1Folder
localName = os.path.join(folder, "news.json")
f = Utils.download(grabber, progress, newsUrl, localName, downType, "utf-8", True)
processNews(grabber, f, folder, progress, downType, db)
localName = os.path.join(folder, "programs.json")
f = Utils.download(grabber, progress, programsUrl, localName, downType, "utf-8", True)
processPrograms(grabber, f, folder, progress, downType, db)
def download(db, grabber, downType):
progress = Utils.getProgress()
for channel in channels:
url = getCatalogueUrl(channel)
name = Utils.httpFilename(url) + "." + channel
folder = Config.m6Folder
localName = os.path.join(folder, name)
f = Utils.download(grabber, progress, url, localName, downType,
"utf-8", True)
if (f):
process(grabber, downType, f, channel, db)
def download(db, grabber, downType):
progress = Utils.getProgress()
folder = Config.tf1Folder
localName = os.path.join(folder, "news.json")
f = Utils.download(grabber, progress, newsUrl, localName, downType,
"utf-8", True)
processNews(grabber, f, folder, progress, downType, db)
localName = os.path.join(folder, "programs.json")
f = Utils.download(grabber, progress, programsUrl, localName, downType,
"utf-8", True)
processPrograms(grabber, f, folder, progress, downType, db)
def download(db, grabber, downType):
progress = Utils.getProgress()
name = Utils.httpFilename(infoUrl)
folder = Config.pluzzFolder
localName = os.path.join(folder, name)
f = Utils.download(grabber, progress, infoUrl, localName, downType, None, True)
z = zipfile.ZipFile(f, "r")
decoder = codecs.getreader("ascii")
for a in z.namelist():
if a.find("catch_up_") == 0:
with z.open(a) as f:
process(grabber, decoder(f), db)
def getTS(self):
if self.ts:
return self.ts
folder = Config.m6Folder
name = Utils.httpFilename(self.url)
localName = os.path.join(folder, name)
progress = Utils.getProgress()
f = Utils.download(self.grabber, progress, self.url, localName, self.downType, "utf-8", True)
if (f):
root = ElementTree.parse(f).getroot()
asset = root.find("asset")
for v in asset.findall("assetItem"):
u = v.find("url").text
self.ts = getTSUrl(u)
return self.ts
def download(db, grabber, downType):
progress = Utils.getProgress()
name = Utils.httpFilename(infoUrl)
folder = Config.pluzzFolder
localName = os.path.join(folder, name)
f = Utils.download(grabber, progress, infoUrl, localName, downType, None,
True)
z = zipfile.ZipFile(f, "r")
decoder = codecs.getreader("ascii")
for a in z.namelist():
if a.find("catch_up_") == 0:
with z.open(a) as f:
process(grabber, decoder(f), db)
def getTS(self):
if self.ts:
return self.ts
folder = Config.m6Folder
name = Utils.httpFilename(self.url)
localName = os.path.join(folder, name)
progress = Utils.getProgress()
f = Utils.download(self.grabber, progress, self.url, localName,
self.downType, "utf-8", True)
if (f):
root = ElementTree.parse(f).getroot()
asset = root.find("asset")
for v in asset.findall("assetItem"):
u = v.find("url").text
self.ts = getTSUrl(u)
return self.ts
def download(db, grabber, downType, mediasetType):
progress = Utils.getProgress()
name = Utils.httpFilename(configUrl)
folder = Config.mediasetFolder
localName = os.path.join(folder, name)
f = Utils.download(grabber, progress, configUrl, localName, downType, None, True)
s = f.read().strip()
root = ElementTree.fromstring(s)
conf = parseConfig(root)
if mediasetType == "tg5":
url = conf["FullVideoRequestUrl"].replace("http://ww.", "http://www.")
downloadItems(grabber, url, FULL_VIDEO, conf, folder, progress, downType, db)
else:
url = conf["ProgramListRequestUrl"]
downloadItems(grabber, url, PROGRAM_LIST, conf, folder, progress, downType, db)
def download(db, grabber, downType):
progress = Utils.getProgress()
today = datetime.date.today()
folder = Config.replayFolder
for x in range(1, 8):
day = today - datetime.timedelta(days = x)
strDate = day.strftime("_%Y_%m_%d")
for channel in channels.values():
filename = channel + strDate + ".html"
url = RAIUrls.replay + "/" + filename
localName = os.path.join(folder, filename)
f = Utils.download(grabber, progress, url, localName, downType, "utf-8")
if f:
process(grabber, f, db)
def download(db, grabber, downType):
progress = Utils.getProgress()
today = datetime.date.today()
folder = Config.replayFolder
for x in range(1, 8):
day = today - datetime.timedelta(days=x)
strDate = day.strftime("_%Y_%m_%d")
for channel in channels.values():
filename = channel + strDate + ".html"
url = RAIUrls.replay + "/" + filename
localName = os.path.join(folder, filename)
f = Utils.download(grabber, progress, url, localName, downType,
"utf-8")
if f:
process(grabber, f, db)
def follow(self, db, downType):
folder = Config.juniorFolder
progress = Utils.getProgress()
again = True
url = self.url
while again:
name = Utils.httpFilename(url)
localFilename = os.path.join(folder, name)
f = Utils.download(self.grabber, progress, url, localFilename, downType, None, True)
(index, total) = processSet(self.grabber, progress, folder, f, db, downType)
if index + 1 == total:
again = False
else:
# replace -V-0.xml -> -V-1.xml and so on
pos = url.rfind("-V-")
if pos == -1:
again = False
else:
base = url[:pos]
url = "{0}-V-{1}.xml".format(base, index + 1)
