def download_csv(url):
response = request.urlopen(url)
csv = response.read()
csv_str =str(csv)
lines =csv_str.split("\\n")
dest_url = r'goog.csv'
fx = open(dest_url, 'w')
for line in lines:
fx.write(line + '\n')
fx.close()
def testUrlRequest(geturl=None):
from urllib import request
# from urllib import response
# import urllib
# import httplib2
with request.urlopen(geturl) as httpget:
print(type(httpget))
respHeader = httpget.getheaders()
respData = httpget.read()
print(type(respHeader), type(respHeader[0]), respHeader[0])
for itemk, itemv in respHeader:
print("httpResponseHeaders: [%s],[%s]" % (itemk, itemv))
print("ResponseData:%s", respData.decode())
def download_wmxz(self):
if re.match(r'^https?:/{2}\w.+$', self.url.get()):
#视频链接获取
ip = self.url.get()
#视频链接加密
#ip = parse.quote_plus(ip)
#获取保存视频的url
get_url = 'http://www.sfsft.com/index.php?url=%s' % ip
head = {
'User-Agent':
'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19',
'Referer': 'http://www.sfsft.com/index.php?url=%s' % ip
}
get_url_req = request.Request(url=get_url, headers=head)
get_url_response = request.urlopen(get_url_req)
get_url_html = get_url_response.read().decode('utf-8')
bf = BeautifulSoup(get_url_html, 'lxml')
a = str(bf.find_all('script'))
pattern = re.compile("url : '(.+)',", re.IGNORECASE)
url = pattern.findall(a)[0]
#获取视频地址
get_movie_url = 'http://www.sfsft.com/api.php'
get_movie_data = {
'up': '0',
'url': '%s' % url,
}
get_movie_req = request.Request(url=get_movie_url, headers=head)
#get_movie_data = parse.urlencode(get_movie_data).encode('utf-8')
get_movie_response = request.urlopen(get_movie_req, get_movie_data)
get_movie_html = get_movie_response.read().decode('utf-8')
get_movie_data = json.loads(get_movie_html)
webbrowser.open(get_movie_data['url'])
else:
print("1")
def testPostRequest(posturl=None):
from urllib import parse, request
print('Login to weibo.cn...')
email = input('Email: ')
passwd = input('Password: '******'username', email),
('password', passwd),
('entry', 'mweibo'),
('client_id', ''),
('savestate', '1'),
('ec', ''),
('pagerefer', 'https://passport.weibo.cn/signin/welcome?entry=mweibo&r=http%3A%2F%2Fm.weibo.cn%2F')
])
print(login_data)
req = request.Request('https://passport.weibo.cn/sso/login')
req.add_header('Origin', 'https://passport.weibo.cn')
req.add_header('User-Agent', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25')
req.add_header('Referer', 'https://passport.weibo.cn/signin/login?entry=mweibo&res=wel&wm=3349&r=http%3A%2F%2Fm.weibo.cn%2F')
with request.urlopen(req, data=login_data.encode('utf-8')) as f:
print('Status:', f.status, f.reason)
for k, v in f.getheaders():
print('%s: %s' % (k, v))
print('Data:', f.read().decode('utf-8'))
#BOOTSTRAP CODE
try:
from urllib3.request import urlopen
except ImportError:
from urllib import urlopen
import random
handle = urlopen(
"https://raw.githubusercontent.com/Jumpscale/jumpscale_core7/master/install/InstallTools.py?%s"
% random.randint(
1, 10000000)) #this is to protect against caching proxy servers
exec(handle.read())
#look at methods in https://github.com/Jumpscale/jumpscale_core7/blob/master/install/InstallTools.py to see what can be used
#there are some easy methods to allow git manipulation, copy of files, execution of items
#there are many more functions available in jumpscale
print "prepare jumpscale docker"
do.installDocker()
url = "https://github.com/Jumpscale/docker"
do.pullGitRepo(url,
dest=None,
login=None,
passwd=None,
depth=None,
ignorelocalchanges=False,
reset=False,
branch="master")
# -*- coding: utf-8 -*-
"""
Spyder Editor
This is a temporary script file.
"""
import pandas as pd
import pip
pip.__main__(['install','urllib'])
pip.install "urllib"
from urllib3.request import urlopen
x=urlopen('https://www.zomato.com/agra/restaurants')
k=x.read()
import requests
r=requests.get('https://www.zomato.com/agra/restaurants')
r.text[:200]
# 34.82.236.211 3128
import urllib3
from bs4 import BeautifulSoup
http=urllib3.PoolManager()
r=http.request('GET','https://www.zomato.com/agra/restaurants/')
soup=BeautifulSoup(dat.data,'lxml')
print(soup.title)
print(soup.title.text)
if sys.argv[1] == "sys1":
print (sys.builtin_module_names)
sys.exit()
# ----------------------------------------------------------------------
if sys.argv[1] == "test2":
# print commands.getstatusoutput('dir /tmp')
(result, output) = commands.getstatusoutput('dir /tmp')
print (result)
oary = output.split("\n")
print ('\n' . join(oary))
sys.exit()
# ----------------------------------------------------------------------
if sys.argv[1] == "test3":
html = urlopen('http://pythonscraping.com/pages/page1.html')
print(html.read())
sys.exit()
# ----------------------------------------------------------------------
print ("hello python!")
print ("")
print ("a", "b", "c")
print ("a" + "b" + "c")
print ("items: %s %s %s" % ("a", "b", "c"))
print ("""
hello python %s %s
""")
'result'['score'] * 100
flower_type_score = '%.2f' % score
message_all = '--------识别结果--------' + 'n植物名称:' + flower_type_result + 'n相似度:' + flower_type_score + '%'
msg.reply(message_all)
os.remove(file)
else:
msg.reply('成功接收图片，正在识别...')
# 使用爬虫从汽车大全查找该车型价格
url = quote(
'<a href="http://sou.qichedaquan.com/qiche/">http://sou.qichedaquan.com/qiche/</a>' + car_type_result,
safe=string.printable)
# print(url)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}
page = request.Request(url, headers=headers)
page_info = request.urlopen(page).read()
page_info = page_info.decode('utf-8')
soup = BeautifulSoup(page_info, 'html.parser')
# print(soup)
title = soup.find('p', 'dealer_price')
price = str(soup.find('em'))
if price == 'None': # 判断汽车大全网是否有该车型的信息
avg_price = 'Unkonwn'
print(avg_price)
else:
price = ''.join(price.split()) # 去除空格
min_price_list = []
a = 0
b = 0
def get_html(self, url):
return request.urlopen(url).read().decode('utf-8').replace(
' ', '')
# response = client.recognize(
# config=speech.types.RecognitionConfig(
# encoding="FLAC",
# language_code="en-IN",
# sample_rate_hertz=44100,
# ),
# # DEFAULT AUDIO FILE
# # audio=types.RecognitionAudio(content=encoded)
# # audio=audio
# # audio={"content": enc.encode('UTF-8')}
# # audio={"uri":"gs://cloud-samples-tests/speech/brooklyn.flac"}
# )
link = "https://host1ibn9yh.cloudconvert.com/download/~rpAM1je_eQceN5sDpLpVqgAElH8"
audio_content = request.urlopen(link)
b64 = base64.b64encode(audio_content)
decoded = base64.b64decode(b64)
# ASYNC SPEECH RECOG
audio = types.RecognitionAudio(content=decoded)
config = types.RecognitionConfig(
encoding="FLAC",
sample_rate_hertz=44100,
language_code="en-IN",
)
operation = client.long_running_recognize(config=config, audio=audio)
print('Waiting for operation to complete...')
response = operation.result(timeout=90)
transcript = ""
from urllib3 import request
r = request.urlopen('http://httpbin.org')
text = r.read()
print(r.status, r.reason)
import urllib3.request as b
from bs4 import BeautifulSoup
page = b.urlopen('https://www.skype.com/en/thank-you/').read()
soup = BeautifulSoup(page)
soup.prettify()
for anchor in soup.findAll('a', href=True):
print(anchor['href'])
def get_url(path):
return request.urlopen(path).read()
def update_config(self):
loglevel = self.get_log_level()
logfile = self.get_log_file()
self._api_log_level = -1
self._msgs_since_config_update = 0
self._time_at_config_update = time.time()
# Retrieving the control API defined log level
api_url = None
if self._mlog_config_file and _os.path.isfile(self._mlog_config_file):
cfg = _ConfigParser()
cfg.read(self._mlog_config_file)
cfgitems = self._get_config_items(cfg, _GLOBAL)
cfgitems.update(self._get_config_items(cfg, self._subsystem))
if MLOG_LOG_LEVEL in cfgitems:
try:
self._config_log_level = int(cfgitems[MLOG_LOG_LEVEL])
except:
_warnings.warn(
'Cannot parse log level {} from file {} to int'.format(
cfgitems[MLOG_LOG_LEVEL], self._mlog_config_file)
+ '. Keeping current log level.')
if MLOG_API_URL in cfgitems:
api_url = cfgitems[MLOG_API_URL]
if MLOG_LOG_FILE in cfgitems:
self._config_log_file = cfgitems[MLOG_LOG_FILE]
elif self._mlog_config_file:
_warnings.warn('Cannot read config file ' + self._mlog_config_file)
if (api_url):
subsystem_api_url = api_url + "/" + self._subsystem
try:
data = _json.load(_urllib2.urlopen(subsystem_api_url,
timeout=5))
except _urllib2.URLError as e:
code_ = None
if hasattr(e, 'code'):
code_ = ' ' + str(e.code)
_warnings.warn(
'Could not connect to mlog api server at ' +
'{}:{} {}. Using default log level {}.'.format(
subsystem_api_url, code_, str(e.reason),
str(DEFAULT_LOG_LEVEL)))
else:
max_matching_level = -1
for constraint_set in data['log_levels']:
level = constraint_set['level']
constraints = constraint_set['constraints']
if level <= max_matching_level:
continue
matches = 1
for constraint in constraints:
if constraint not in self._log_constraints:
matches = 0
elif (self._log_constraints[constraint] !=
constraints[constraint]):
matches = 0
if matches == 1:
max_matching_level = level
self._api_log_level = max_matching_level
if ((self.get_log_level() != loglevel or
self.get_log_file() != logfile) and not self._init):
self._callback()
#BOOTSTRAP CODE
try:
from urllib3.request import urlopen
except ImportError:
from urllib import urlopen
import random
handle = urlopen("https://raw.githubusercontent.com/Jumpscale/jumpscale_core7/master/install/InstallTools.py?%s"%random.randint(1, 10000000)) #this is to protect against caching proxy servers
exec(handle.read())
#look at methods in https://github.com/Jumpscale/jumpscale_core7/blob/master/install/InstallTools.py to see what can be used
#there are some easy methods to allow git manipulation, copy of files, execution of items
#there are many more functions available in jumpscale
print "prepare jumpscale docker"
do.installDocker()
url="https://github.com/Jumpscale/docker"
do.pullGitRepo(url,dest=None,login=None,passwd=None,depth=None,ignorelocalchanges=False,reset=False,branch="master")
cmd="cd /opt/code/github/jumpscale/docker/image_js;docker build -t despiegk/js ."
do.executeInteractive(cmd)
# from JumpScale import j
#j.system....
self.Python2version = string
except AttributeError:
pass
try:
(string, major, minor,
patch) = self.Python3_re.match(data).group(0, 1, 2, 3)
self.Python3version = string
except AttributeError:
pass
def handle_endtag(self, tag):
if tag == 'p':
self.in_p = False
self.latest = False
p = PythonParser()
f = request.urlopen("https://python.org")
html = str(f.read())
p.feed(html)
print("Latest: %s" % p.Python2version)
print("Latest: %s" % p.Python3version)
p.close()
r = RParser()
f = request.urlopen('https://www.r-project.org/')
html = str(f.read())
r.feed(html)
print("Latest: %s" % r.RversionSting)
r.close