def _saveCSS(self, cssHref):
# Try to do css files only once.
if cssHref in self.downloadedHrefs:
return
fileName = self._generateResourceName(self.cssDir, cssHref,
IZResourceDependencyTypes.CSS)
download = ZSimpleTextHTTPRequest(cssHref, cookieReg=self.cookieReg)
self.listener.onWebpageGetterResourceStarted(
cssHref, IZResourceDependencyTypes.CSS)
if download.send(None):
cssContent = download.getResponse()
self._saveFileContent(fileName, cssContent)
self.listener.onWebpageGetterResourceComplete(cssHref)
finder = ZCSSDependencyFinder(cssContent)
dependencies = finder.findDependencies()
self.listener.onWebpageGetterFoundDependencies(
cssHref, dependencies)
for dependency in dependencies:
href = urlparse.urljoin(cssHref, dependency.getHref())
resourceFile = None
if dependency.getType() == IZResourceDependencyTypes.IMAGE:
resourceFile = self._saveImage(href)
elif dependency.getType == IZResourceDependencyTypes.CSS:
resourceFile = self._saveCSS(href)
self._addResourceToFixup(resourceFile)
if resourceFile:
self.downloadedHrefs.add(href)
self.resourceMap[dependency.getHref()] = resourceFile
else:
return None
# raise ZException(_extstr(u"httpdownload.FailedToDownloadCSS") % cssHref) #$NON-NLS-1$
return fileName
def _saveCSS(self, cssHref):
# Try to do css files only once.
if cssHref in self.downloadedHrefs:
return
fileName = self._generateResourceName(self.cssDir, cssHref, IZResourceDependencyTypes.CSS)
download = ZSimpleTextHTTPRequest(cssHref, cookieReg=self.cookieReg)
self.listener.onWebpageGetterResourceStarted(cssHref, IZResourceDependencyTypes.CSS)
if download.send(None):
cssContent = download.getResponse()
self._saveFileContent(fileName, cssContent)
self.listener.onWebpageGetterResourceComplete(cssHref)
finder = ZCSSDependencyFinder(cssContent)
dependencies = finder.findDependencies()
self.listener.onWebpageGetterFoundDependencies(cssHref, dependencies)
for dependency in dependencies:
href = urlparse.urljoin(cssHref, dependency.getHref())
resourceFile = None
if dependency.getType() == IZResourceDependencyTypes.IMAGE:
resourceFile = self._saveImage(href)
elif dependency.getType == IZResourceDependencyTypes.CSS:
resourceFile = self._saveCSS(href)
self._addResourceToFixup(resourceFile)
if resourceFile:
self.downloadedHrefs.add(href)
self.resourceMap[dependency.getHref()] = resourceFile
else:
return None
# raise ZException(_extstr(u"httpdownload.FailedToDownloadCSS") % cssHref) #$NON-NLS-1$
return fileName
def _sendHttpPostData(self, pingUrl, postDataMap):
try:
request = ZSimpleTextHTTPRequest(pingUrl)
if request.send(postDataMap):
return request.getResponse()
except:
pass
return None
def _sendHttpPostData(self, pingUrl, postDataMap):
try:
request = ZSimpleTextHTTPRequest(pingUrl)
if request.send(postDataMap):
return request.getResponse()
except:
pass
return None
def _saveScript(self, href):
# Try to do scripts only once.
if href in self.downloadedHrefs:
return
fileName = self._generateResourceName(self.otherDir, href, IZResourceDependencyTypes.SCRIPT)
download = ZSimpleTextHTTPRequest(href, cookieReg=self.cookieReg)
self.listener.onWebpageGetterResourceStarted(href, IZResourceDependencyTypes.SCRIPT)
if download.send(None):
self._saveFileContent(fileName, download.getResponse())
self.listener.onWebpageGetterResourceComplete(href)
return fileName
else:
return None
def _saveAsWebpage(self):
rootFile = self._generateResourceName(self.outputDir,
self.url) #$NON-NLS-1$
self.resourceMap[self.url] = rootFile
self.resourcesToFixup.add(rootFile)
# Download the root URL
download = ZSimpleTextHTTPRequest(self.url, cookieReg=self.cookieReg)
self.listener.onWebpageGetterResourceStarted(
self.url, u"text/xhtml") #$NON-NLS-1$
if download.send(None):
# Now find all dependencies, and download those too.
content = download.getResponse()
self._saveFileContent(rootFile, content)
self.listener.onWebpageGetterResourceComplete(self.url)
finder = ZXhtmlDependencyFinder(content)
dependencies = finder.findDependencies()
baseHref = finder.getBaseHref()
if baseHref is None:
baseHref = self.url
self.listener.onWebpageGetterFoundDependencies(
self.url, dependencies)
for dependency in dependencies:
href = urlparse.urljoin(baseHref, dependency.getHref())
# Skip this dependency if it is not an http or https guy.
if href is None or not href.lower().startswith(
u"http"): #$NON-NLS-1$
continue
resourceFile = None
if dependency.getType() == IZResourceDependencyTypes.IMAGE:
resourceFile = self._saveImage(href)
elif dependency.getType() == IZResourceDependencyTypes.SCRIPT:
resourceFile = self._saveScript(href)
self._addResourceToFixup(resourceFile)
elif dependency.getType() == IZResourceDependencyTypes.CSS:
resourceFile = self._saveCSS(href)
self._addResourceToFixup(resourceFile)
if resourceFile:
self.downloadedHrefs.add(href)
self.resourceMap[dependency.getHref()] = resourceFile
# Now fix up all of the downloaded resources by replacing the hrefs in
# them with the new local hrefs found in the resourceMap.
self._fixupAllHrefs()
return rootFile
def _saveScript(self, href):
# Try to do scripts only once.
if href in self.downloadedHrefs:
return
fileName = self._generateResourceName(self.otherDir, href,
IZResourceDependencyTypes.SCRIPT)
download = ZSimpleTextHTTPRequest(href, cookieReg=self.cookieReg)
self.listener.onWebpageGetterResourceStarted(
href, IZResourceDependencyTypes.SCRIPT)
if download.send(None):
self._saveFileContent(fileName, download.getResponse())
self.listener.onWebpageGetterResourceComplete(href)
return fileName
else:
return None
def _saveAsWebpage(self):
rootFile = self._generateResourceName(self.outputDir, self.url) # $NON-NLS-1$
self.resourceMap[self.url] = rootFile
self.resourcesToFixup.add(rootFile)
# Download the root URL
download = ZSimpleTextHTTPRequest(self.url, cookieReg=self.cookieReg)
self.listener.onWebpageGetterResourceStarted(self.url, u"text/xhtml") # $NON-NLS-1$
if download.send(None):
# Now find all dependencies, and download those too.
content = download.getResponse()
self._saveFileContent(rootFile, content)
self.listener.onWebpageGetterResourceComplete(self.url)
finder = ZXhtmlDependencyFinder(content)
dependencies = finder.findDependencies()
baseHref = finder.getBaseHref()
if baseHref is None:
baseHref = self.url
self.listener.onWebpageGetterFoundDependencies(self.url, dependencies)
for dependency in dependencies:
href = urlparse.urljoin(baseHref, dependency.getHref())
# Skip this dependency if it is not an http or https guy.
if href is None or not href.lower().startswith(u"http"): # $NON-NLS-1$
continue
resourceFile = None
if dependency.getType() == IZResourceDependencyTypes.IMAGE:
resourceFile = self._saveImage(href)
elif dependency.getType() == IZResourceDependencyTypes.SCRIPT:
resourceFile = self._saveScript(href)
self._addResourceToFixup(resourceFile)
elif dependency.getType() == IZResourceDependencyTypes.CSS:
resourceFile = self._saveCSS(href)
self._addResourceToFixup(resourceFile)
if resourceFile:
self.downloadedHrefs.add(href)
self.resourceMap[dependency.getHref()] = resourceFile
# Now fix up all of the downloaded resources by replacing the hrefs in
# them with the new local hrefs found in the resourceMap.
self._fixupAllHrefs()
return rootFile
def _uploadStream(self, fileName, fileStream, metaData): #@UnusedVariable
data = {
u"xml" : u"yes", #$NON-NLS-1$ #$NON-NLS-2$
u"fileupload" : fileStream #$NON-NLS-1$
}
code = self._getRegistrationCode()
if code:
data[u"cookie"] = code #$NON-NLS-1$
imageShackUrl = u"http://www.imageshack.us/index.php" #$NON-NLS-1$
request = ZSimpleTextHTTPRequest(imageShackUrl)
ok = request.send( data )
if ok:
respData = request.getResponse()
dom = ZDom()
dom.loadXML( respData )
url = dom.selectSingleNodeText(u"/links/image_link") #$NON-NLS-1$
return ZUploadResponse(url)
else:
code = unicode( request.getHttpStatusCode() )
msg = unicode( request.getHttpStatusMessage() )
raise ZException(u"ImageShackMedia upload failed. HTTP response: %s %s" % (code, msg)) #$NON-NLS-1$
def _uploadStream(self, fileName, fileStream, metaData): #@UnusedVariable
data = {
u"xml": u"yes", #$NON-NLS-1$ #$NON-NLS-2$
u"fileupload": fileStream #$NON-NLS-1$
}
code = self._getRegistrationCode()
if code:
data[u"cookie"] = code #$NON-NLS-1$
imageShackUrl = u"http://www.imageshack.us/index.php" #$NON-NLS-1$
request = ZSimpleTextHTTPRequest(imageShackUrl)
ok = request.send(data)
if ok:
respData = request.getResponse()
dom = ZDom()
dom.loadXML(respData)
url = dom.selectSingleNodeText(u"/links/image_link") #$NON-NLS-1$
return ZUploadResponse(url)
else:
code = unicode(request.getHttpStatusCode())
msg = unicode(request.getHttpStatusMessage())
raise ZException(
u"ImageShackMedia upload failed. HTTP response: %s %s" %
(code, msg)) #$NON-NLS-1$
def _downloadHtmlContent(self, url):
html = None
request = ZSimpleTextHTTPRequest(url)
if request.send():
html = request.getResponse()
return html
def _downloadHtmlContent(self, url):
html = None
request = ZSimpleTextHTTPRequest(url)
if request.send():
html = request.getResponse()
return html
def _processResponseData(self, resp, txt):
txt = ZSimpleTextHTTPRequest._processResponseData(self, resp, txt)
txt = string.lstrip(txt)
# FIXME (PJ) check resp content-type. If text/plain, then return txt as is isstead of a zdom.
xhtmlDoc = loadXhtmlDocumentFromString(txt)
return xhtmlDoc
def _createRequest(self, url):
return ZSimpleTextHTTPRequest(url)
def _processResponseData(self, resp, txt):
txt = ZSimpleTextHTTPRequest._processResponseData(self, resp, txt)
txt = string.lstrip(txt)
# FIXME (PJ) check resp content-type. If text/plain, then return txt as is isstead of a zdom.
xhtmlDoc = loadXhtmlDocumentFromString(txt)
return xhtmlDoc
def _sendFile(self, url, params, fileStream):
params[u"photo"] = fileStream #$NON-NLS-1$
request = ZSimpleTextHTTPRequest(url)
request.send(params)
rspXML = request.getResponse()
return self.parse_xmlnode(rspXML)