def get_rss2(url): try: with XmlReader.Create(url) as reader: return [ RSSItem( i.Title.Text, i.Summary.Text, i.Links[0].Uri.AbsoluteUri if i.Links.Count > 0 else "") for i in SyndicationFeed.Load(reader).Items ] except XmlException: wc = WebClient() wc.Encoding = UTF8 xmlstr = wc.DownloadString(url) xdoc = XmlDocument() xdoc.LoadXml(xmlstr) xelem = xdoc.DocumentElement titles = [ i.InnerText.Replace("\n", "").Replace("\r", "") for i in xelem.SelectNodes("//item//title") ] links = [i.InnerText for i in xelem.SelectNodes("//item//link")] descriptions = [ i.InnerText for i in xelem.SelectNodes("//item//description") ] return [ RSSItem(t, d, l) for t, d, l in zip(titles, descriptions, links) ]
def process(self): empty = False try: client = WebClient() client.Encoding = Encoding.UTF8 client.Headers['Accept'] = 'text/html' client.Headers[ 'User-Agent'] = 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)' body = client.DownloadString( 'http://search.twitter.com/search/thread/%d' % self.status.Id) divs = re.findall(r'<div class="msg">(.*?)</div>', body, re.S) if divs: for div in divs: match = re.search( r'<a[^>]*>(.*?)</a>.*<span[^>]*>(.*?)</span>', div) name = match.group(1) text = re.sub(r'<[^>]*>', '', match.group(2)) self.notice(text, nick=name) else: empty = True except WebException, e: if e.Response.StatusCode == 404: # クロールされていないかプロテクトか empty = True else: raise
def _get_htmldoc(url, encode=UTF8): """指定したURLのHTMLテキストを取得。失敗するとNoneを返す""" wc = WebClient() wc.Encoding = encode try: html = wc.DownloadString(url) except WebException as ex: return None htmlDoc = HtmlDocument() htmlDoc.LoadHtml(html) return htmlDoc
def get_prj_text_from_EPSG(EPSG): """Get prj string by epsg.io using .Net.""" path = "https://epsg.io/" + str(EPSG) + ".wkt" web_client = WebClient() try: web_client.Headers.Add( "user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)" ) prj_text = web_client.DownloadString(path) except: raise Exception( "Query failed. Please, copy and paste https://epsg.io/?{0}.wkt on your browser, check if text is showed. If yes, use it with gh." .format(EPSG)) return prj_text
def _getCsrf(self): ''' Get csrf token from c4c ''' Log.Write("reloading csrf token") client = WebClient() client.Headers.Add("Authorization", "Basic " + self.odataCredentials) client.Headers.Add("x-csrf-token", "fetch") client.Headers.Add("Content-Type", "application/json") result = client.DownloadString( "https://my{0}.crm.ondemand.com/sap/c4c/odata/v1/c4codataapi/$metadata" .format(self.sapId)) csrf = Objects.Dynamic() csrf.token = client.ResponseHeaders["x-csrf-token"] csrf.cookies = client.ResponseHeaders["set-cookie"].split(",") return csrf
def ShortenUrl(long_url): request_url = "http://api.bit.ly/shorten?version=2.0.1&format=xml&longUrl=%s&login=%s&apiKey=%s" % ( Utility.UrlEncode(long_url), username, apikey) client = WebClient() response = client.DownloadString(request_url) return re_shorten.search(response).group(1)
import time from Spotfire.Dxp.Data.Import import TextFileDataSource, TextDataReaderSettings clr.AddReference('System.Web.Extensions') from System.Web.Script.Serialization import JavaScriptSerializer from System.Net import WebClient # Create a web client client = WebClient() # Download the results of that URL results = client.DownloadString("http://localhost:8888/spotfireFramework/assetMarketPlace/customLibs/visTimeline/dxp/eventsGroup.csv") # print these results print results stream = MemoryStream() writer = StreamWriter(stream) writer.Write(results) writer.Flush() stream.Seek(0, SeekOrigin.Begin) readerSettings = TextDataReaderSettings() readerSettings.Separator = "," readerSettings.AddColumnNameRow(0) readerSettings.SetDataType(0, DataType.String)
def ShortenUrl(long_url): request_url = "http://bit.ly/?url=%s" % (Utility.UrlEncode(long_url)) client = WebClient() response = client.DownloadString(request_url) return re_shorten.search(response).group(1)