def DownloadUpdate(self, file):
		self.log('Downloading: %s' % file)
		dirfile = os.path.join(self.UpdateTempDir,file)
		dirname, filename = os.path.split(dirfile)
		if not os.path.isdir(dirname):
			try:
				os.makedirs(dirname)
			except:
				self.log('Error creating directory: '  +dirname)
		url = self.SVNPathAddress+urllib.quote(file)
		try:
			if re.findall(".xbt",url):
				self.totalsize = int(re.findall("File length: ([0-9]*)",urllib2.urlopen(url+"?view=log").read())[0])
				urllib.urlretrieve( url.decode("utf-8"), dirfile.decode("utf-8"))
			else: urllib.urlretrieve( url.decode("utf-8"), dirfile.decode("utf-8") )
			self.DownloadedFiles.append(urllib.unquote(url))
			return 1
		except:
			try:
				time.sleep(2)
				if re.findall(".xbt",url):
					self.totalsize = int(re.findall("File length: ([0-9]*)",urllib2.urlopen(url+"?view=log").read())[0])
					urllib.urlretrieve(url.decode("utf-8"), dirfile.decode("utf-8"))
				else: urllib.urlretrieve(url.decode("utf-8"), dirfile.decode("utf-8") )
				urllib.urlretrieve(url.decode("utf-8"), dirfile.decode("utf-8"))
				self.DownloadedFiles.append(urllib.unquote(url))
				return 1
			except:
				self.log("Download failed: %s" % url)
				self.DownloadFailedFiles.append(urllib.unquote(url))
				return 0
Beispiel #2
0
 def IsRunning(self):
   """Returns whether the server is up and running."""
   try:
     urllib2.urlopen(self.GetUrl() + '/status')
     return True
   except urllib2.URLError:
     return False
Beispiel #3
0
def download_mango(url, path):   
    """
        Function: download_mango(url, path)
        Usage: download_mango('http://www.mangareader.net/poyopoyo-kansatsu-nikki/1', os.getcwd())
        Added in version: 0.1 Beta
    """
    if path != os.getcwd():
        pathchange(os.getcwd(), path)
    urlContent = urllib2.urlopen(url).read()
    imgUrls = re.findall('img .*?src="(.*?.jpg)"', urlContent)


    for imgUrl in imgUrls:
        try:
            imgData = urllib2.urlopen(imgUrl).read()
            fileName = basename(urlsplit(imgUrl)[2])
            output = open(fileName, 'wb')
            output.write(imgData)
            output.close()
        except IOError: 
            print "File not found or full disk. Try again."
            sys.exit(1)
        except KeyboardInterrupt:
            print "Operation aborted manually."
            sys.exit(1)
        except:
            print "Unknown error. If this persists, contact the author or create a ticket in the bugtracker."
            sys.exit(1)
Beispiel #4
0
def registration(request):
    form = AuthRegistrationForm(request.POST or None)

    if request.method == 'POST' and form.is_valid():
        user_id = User.objects.all().aggregate(Max('id'))['id__max'] + 1
        username = '******' + str(user_id)
        try:
            user = User.objects.create_user(username=username,
                                            password=generate_user_password(16),
                                            email=form.cleaned_data['email'],
                                            first_name=form.cleaned_data['first_name'],
                                            last_name=form.cleaned_data['last_name'])
        except IntegrityError:
            user_id = User.objects.all().aggregate(Max('id'))['id__max'] + 1
            username = '******' + str(user_id)
            user = User.objects.create_user(username=username,
                                            password=generate_user_password(16),
                                            email=form.cleaned_data['email'],
                                            first_name=form.cleaned_data['first_name'],
                                            last_name=form.cleaned_data['last_name'])
        user_profile = UserProfile.objects.create(user=user, phone=request.GET.get('phone'), sms_code=random.randint(100000, 999999), account_type=form.cleaned_data['account_type'])
        urllib2.urlopen('http://smsc.ru/sys/send.php?login=Jango.kz&psw=AcEMXtLGz042Fc1ZJUSl&phones=+' + user_profile.phone + '&mes=Access code: ' + str(user_profile.sms_code))
        Client.objects.create(profile=user_profile)
        return redirect('/account/login/?phone='+user_profile.phone)
    else:
        return render(request, 'accounts/registration.html', {'form': form})
Beispiel #5
0
 def check_proxy(self, specific={}):
     """ Checks if proxy settings are set on the OS
     Returns:
     -- 1 when direct connection works fine
     -- 2 when direct connection fails and any proxy is set in the OS
     -- 3 and settings when direct connection fails but a proxy is set
     see: https://docs.python.org/2/library/urllib.html#urllib.getproxies
     """
     os_proxies = getproxies()
     if len(os_proxies) == 0 and self.check_internet_connection:
         logging.info("No proxy needed nor set. Direct connection works.")
         return 1
     elif len(os_proxies) == 0 and not self.check_internet_connection:
         logging.error("Proxy not set in the OS. Needs to be specified")
         return 2
     else:
         #
         env['http_proxy'] = os_proxies.get("http")
         env['https_proxy'] = os_proxies.get("https")
         #
         proxy = ProxyHandler({
                              'http': os_proxies.get("http"),
                              'https': os_proxies.get("https")
                              })
         opener = build_opener(proxy)
         install_opener(opener)
         urlopen('http://www.google.com')
         return 3, os_proxies
Beispiel #6
0
def recognise_eHentai(link, path):
    url = str(link)
    page = urllib2.urlopen(url).read()
    soup = BeautifulSoup(page)
    name = soup.findAll('title')
    name = name[0].get_text().encode('utf-8')
    name = str(name)
    path = path + '\\' + name
    download_eHentai(link, path)

    pages = soup.find_all('span')
    pages = pages[1].get_text()
    pages = int(pages)
    z = 0

    while (pages > z):
        z = z + 1
        sopa = soup.find('div', 'sn')
        sopa = sopa.find_all('a')
        sopa = sopa[2].get('href')

        url = str(sopa)
        download_eHentai(url, path)
        page = urllib2.urlopen(url).read()

        soup = BeautifulSoup(page)

        sopa = soup.find('div', 'sn')
        sopa = sopa.find_all('a')
        sopa = sopa[2].get('href')
        download_eHentai(sopa, path)
Beispiel #7
0
def update_lyrics(request):  
    b = open('./artistList.txt', 'r') 
    bb = b.read()
    b.close() 
    bbb = bb.split(chr(10))

    for ar in bbb: 
        if ar.split('=')[1] == '1':
            return index(request)

        furl = "/"+ar.split('=')[1]+".htm"
        ar = ar.split('=')[0]
        artxt = ''
        
        #req = urllib2.Request(u"http://mojim.com/"+ar+".html?t1")
        #print "connected >> http://mojim.com/"+ar+".html?t1"
        #response = urllib2.urlopen(req) 
        #result = response.read()     
        print '--',furl,'--'

        if len(furl) > 0:           
            req2 = urllib2.Request("http://mojim.com"+furl) 


            response2 = urllib2.urlopen(req2)
            result2 = response2.read()     
            
            furl2 = re.findall('/tw[0-9x]*.htm', result2)
            iii = -1
            if len(furl2) > 0:        
                for furl3 in furl2: 
                    iii = iii + 1
                    if iii % 2 == 0: continue
                    try: 
                        req3 = urllib2.Request("http://mojim.com"+furl3) 
                        
                        response3 = urllib2.urlopen(req3)
                        result3 = response3.read()   
                        
                        lasturl = re.findall('<dl><dt><br /><br />[^^]*</div>', result3)
                        #a = raw_input()
                        artxt = lasturl[0].replace('更多更詳盡歌詞','').replace(u'在 ','').replace(u'Mojim.com','').replace(u'※','').replace('魔鏡歌詞網','')
  
                        aaaaaaaa = re.findall(u'title="歌詞(.*)">', artxt)
                        bbbbbbbb = re.findall('<dd><br />(.*)</dd>', artxt) 
     
                        bCnt = len(bbbbbbbb)
                        for bi in range(0, bCnt): 
                            if len(bbbbbbbb[bi]) > 22: 
                                lv = LyricsView()
                                ll = striphtml(bbbbbbbb[bi].encode('Shift_JIS').replace('<br />', '\r'))
                                ll = ll[:len(ll)-24]
                                lv.setParams({'artist':ar,'title':aaaaaaaa[bi],'lyrics':ll})
                                lv.save() 
                    except:
                        pass
        '''a = open(u''+ar+'.html', 'w')
        a.write(artxt)
        a.close()'''
    return index(request)
Beispiel #8
0
def tag_to_server(scanid, tagid):
	try:
		myurl = tag_url % (scanid, tagid)
		urlopen(myurl)
	except:
		print 'error'
	print 'sent to server'
Beispiel #9
0
def scrap_items():
	for itemlist in ITEMLIST:
		soup = BS(urllib2.urlopen(''.join([LOLWIKI, itemlist])).read())
		item_table = soup.find('table', class_='stdt sortable')

		for tr in item_table.find_all('tr'):
			tds = tr.find_all('td')
			if len(tds) < 1:
				continue
			if tr.find('p') == None:
				continue

			item_name = tr.find('p').text.strip()
			item_url = tr.find('img')['src']

			if item_url.split(':')[0] == 'data':
				item_url = tr.find('img')['data-src']

			if not HOOKED:
				continue

			#store item in database
			d_item = Item()
			d_item.name = item_name

			t_img = NamedTemporaryFile(delete=True)
			t_img.write(urllib2.urlopen(item_url).read())
			t_img.flush()
			t_img.name = '.'.join([item_name, 'jpg'])

			d_item.picture = File(t_img)
			d_item.save()
Beispiel #10
0
def sendMessage(subject, content, chanel, mobile) :
    if content :
        content = subject + content
        subject = "时时彩计划方案"
        if chanel == "serverChan" :
            key = "SCU749Tfa80c68db4805b9421f52d360f6614cb565696559f19e"
            url = "http://sc.ftqq.com/" + key +".send"
            parameters = {
            "text" : subject, "desp" : content,
            "key"  : key
            }
        elif chanel == "pushBear" :
            url = "http://api.pushbear.com/smart"
            parameters = {
                "sendkey" : "96-d296f0cdb565bae82a833fabcd860309",
                "text" : subject,
                "mobile" : mobile,
                "desp" : content
            }
        if chanel == "mail" :
            sendMail("smtp.126.com", "*****@*****.**", ["*****@*****.**", "*****@*****.**"],
                     subject, content, "126.com", "dhysgzs*211", format='plain')
            return

        postData = urllib.urlencode(parameters)
        request = urllib2.Request(url, postData)
        urllib2.urlopen(request)
Beispiel #11
0
def synopsis_mode_video():
	#####	Check for Preview files.
	PreviewFile = "0"
	for root, dirs, files in os.walk( _Resources_Preview ):
		for filename in files:
			PreviewFile = root + '\\' + filename
			Current_Window.setProperty( 'Synopsis_Video_Preview_Path', PreviewFile )
			Current_Window.setProperty( 'Synopsis_Video_Preview_Name', "Found "+filename )
	if PreviewFile == "0":
		log('|   No preview video found')
		xbmc.executebuiltin('Skin.Reset(SynopsisPreviewThere)')
	else:
		if PreviewFile.endswith('.xmv'):
			Current_Window.setProperty( 'Player_Type','DVDPlayer' )
			xbmc.executebuiltin('Skin.SetBool(SynopsisPreviewThere)')
		elif PreviewFile.endswith('.strm'):
			try:
				urllib2.urlopen('http://www.google.com', timeout=1)
				Current_Window.setProperty( 'Player_Type','MPlayer' )
				xbmc.executebuiltin('Skin.SetBool(SynopsisPreviewThere)')
			except urllib2.URLError as err:
				xbmc.executebuiltin('Skin.Reset(SynopsisPreviewThere)')
		else:
			Current_Window.setProperty( 'Player_Type','MPlayer' )
			xbmc.executebuiltin('Skin.SetBool(SynopsisPreviewThere)')
		log('|   Found ' + PreviewFile)
Beispiel #12
0
def flight_search_results(sid, searchid):
    # 删除开头的$和逗号,并把数字转化成浮点类型
    def parse_price(p):
        return float(p[1:].replace(',', ''))

    # 遍历检测
    while 1:
        time.sleep(2)

        # 构造检测所用的 URL
        url = 'http://www.kayak.com/s/basic/flight?'
        url += 'searchid=%s&c=5&apimode=1&_sid_=%s&version=1' % (searchid, sid)
        doc = xml.dom.minidom.parseString(urllib2.urlopen(url).read())

        # 寻找 morepending 标签,并等待其不再为 true
        more_pending = doc.getElementsByTagName('morepending')[0].firstChild
        if more_pending is None or more_pending.data == 'false':
            break

    # 现在,下载完整列表
    url = 'http://www.kayak.com/s/basic/flight?'
    url += 'searchid=%s&c=999&apimode=1&_sid_=%s&version=1' % (searchid, sid)
    doc = xml.dom.minidom.parseString(urllib2.urlopen(url).read())

    # 得到不同元素组成的列表
    prices = doc.getElementsByTagName('price')
    departures = doc.getElementsByTagName('depart')
    arrivals = doc.getElementsByTagName('arrive')

    # 用 zip 将它们连在一起
    return zip([p.firstChild.data.split(' ')[1] for p in departures],
               [p.firstChild.data.split(' ')[1] for p in arrivals],
               [parse_price(p.firstChild.data) for p in prices])
def get_proportional_hash_area(period):
	""" 	Takes in periods accepted by P2Pool - hour, day, week, month or year,
		then gets hash_data from the server running on localhost, parses it, 
		and calculates each miner's hash power against the total during that time. 
	"""
	import urllib2, json
	path1 = 'http://localhost:9332/web/graph_data/miner_hash_rates/last_'+period
	result1 = json.load(urllib2.urlopen(path1))
	path2 = 'http://localhost:9332/web/graph_data/miner_dead_hash_rates/last_'+period
	result2 = json.load(urllib2.urlopen(path2))
	
	hash_areas = {}
	total_hash_area = 0
	for row in result1:
		for address in row[1]:
			try:
				hash_areas[address] += row[1][address] * row[2]
			except KeyError:
				hash_areas[address] = row[1][address] * row[2]
			finally:
				total_hash_area += row[1][address] * row[2]
	
	for row in result2:
		for address in row[1]:
			hash_areas[address] -= row[1][address]*row[2]
			total_hash_area -= row[1][address] * row[2]

	proportions = {}	
	for address in hash_areas.keys():
		proportions[address] = hash_areas[address] / total_hash_area
		hash_areas[address] /= 1000000000000000
	
	return hash_areas, proportions
Beispiel #14
0
def main():

    #for p in range(1,intGetMaxPage +1):
    #soup = BeautifulSoup()
    try:
        resp = urllib2.urlopen(getUrl,timeout=10)
        soup = BeautifulSoup(resp)
        soup = soup.find('div' ,{'id':'prodlist'})

    
        #for k in soup.findAll("div", {'class': 'p-name'}): # 抓< div class='p=name'>...< /div>
        for k in soup.findAll('a', href=True): 
            try:
            
                url = k.get('href') 
                print k.text
                print url 
        
                page_url = homeUrl + url
                print page_url
                resp_text_page = urllib2.urlopen(homeUrl + url, timeout=10)
            
                soup_text_page = BeautifulSoup(resp_text_page)
                contextPageUrl(soup_text_page,page_url)    
            except:
                print "Unexpected error:", sys.exc_info()[0]
                print "Unexpected error:", sys.exc_info()[1]
                continue
    except:
        #continue
        print "Unexpected error:", sys.exc_info()[0]
        print "Unexpected error:", sys.exc_info()[1]
        pass
Beispiel #15
0
 def getUserAnswers(self, all):
     # 获取最新的文件的qID和aID
     latestFile = self.getLatestAnswerFileName()
     latestQID = 0
     latestAID = 0
     if latestFile is None:  # 没有符合格式的文件,需要全抓
         all = True
     else:  # 计算出最新的questionID和answerID
         pattern = re.compile('^\[\d{4}-\d{2}-\d{2}\].*-q(\d{1,50})-a(\d{1,50}).html$')
         match = pattern.findall(latestFile)
         for pp in match:
             latestQID = pp[0]
             latestAID = pp[1]
     # 默认是要抓第一页的,顺便计算回答的总页数
     pageContent = urllib2.urlopen("{}?page={}".
                                       format(self.answerURL, self.startPage)).read()
     d = pq(pageContent)
     pageMax = self.getMaxPageNumber(d)
     currentPage = self.startPage
     ret = False
     while True:
         self.logging("parsing page {} of {}".format(currentPage, pageMax), True)
         # 如果不是需要全部抓取,那么看看现在抓够了没有
         # 遇到老答案之后,再向前寻找10个老答案,并更新
         ret = self.parseAnswerAndSave(d, latestQID, latestAID, all)
         if not all and ret:  # 不用全抓,而且发现了重复 
             return
         if currentPage >= pageMax:  # 已经是最后一页
             break
         # 计算下一页的pq值
         currentPage += 1
         pageContent = urllib2.urlopen("{}?page={}".
                                       format(self.answerURL, currentPage)).read()
         d = pq(pageContent)
def login():  # 模拟登录程序
    postdata = {
    'entry': 'weibo',
    'gateway': '1',
    'from': '',
    'savestate': '7',
    'userticket': '1',
    'ssosimplelogin': '******',
    'vsnf': '1',
    'vsnval': '',
    'su': '',
    'service': 'miniblog',
    'servertime': '',
    'nonce': '',
    'pwencode': 'rsa2', #'wsse',
    'sp': '',
    'encoding': 'UTF-8',
        ####
    'prelt':'115',
    'rsakv':'',
    ####
    'url':'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
        #'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
    'returntype': 'META'
}
    global account
    username = '******'%(account)
    pwd = '1161895575'
    url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.5)'
    try:#主要获取servertime和nonce这两个随机的内容
        servertime, nonce, pubkey, rsakv = get_servertime()
    except:
        return
    #global postdata
    postdata['servertime'] = servertime
    postdata['nonce'] = nonce
    postdata['rsakv']= rsakv
    postdata['su'] = get_user(username)#对用户名进行加密
    postdata['sp'] = get_pwd(pwd, servertime, nonce, pubkey)#对密码进行加密
    postdata = urllib.urlencode(postdata)
    #headers = {'User-Agent':'Mozilla/5.0 (X11; Linux i686; rv:8.0) Gecko/20100101 Firefox/8.0'}#设置post头部,根据不同的应用平台进行设定
    headers = {'User-Agent':'Mozilla/5.0 (X11; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0'}
    #headers = {'User-Agent':'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)'}
    req  = urllib2.Request(
        url = url,
        data = postdata,
        headers = headers
    )
   
    result = urllib2.urlopen(req)
    
    text = result.read()
    p = re.compile('location\.replace\(\'(.*?)\'\)')
    try:
        login_url = p.search(text).group(1)
        ###print login_url
        urllib2.urlopen(login_url)
        print "Login successful!"
    except:
        print 'Login error!'
 def checkNetConnection(self):
     try:
         urllib2.urlopen('http://www.google.com',timeout=7)
         return True
     except urllib2.URLError as err:
         pass
     return False
Beispiel #18
0
    def get_tags():
        socket_to = None
        try:
            socket_to = socket.getdefaulttimeout()
            socket.setdefaulttimeout(EC2.TIMEOUT)
        except Exception:
            pass

        try:
            iam_role = urllib2.urlopen(EC2.URL + "/iam/security-credentials").read().strip()
            iam_params = json.loads(urllib2.urlopen(EC2.URL + "/iam/security-credentials" + "/" + unicode(iam_role)).read().strip())
            from checks.libs.boto.ec2.connection import EC2Connection
            connection = EC2Connection(aws_access_key_id=iam_params['AccessKeyId'], aws_secret_access_key=iam_params['SecretAccessKey'], security_token=iam_params['Token'])
            instance_object = connection.get_only_instances([EC2.metadata['instance-id']])[0]

            EC2_tags = [u"%s:%s" % (tag_key, tag_value) for tag_key, tag_value in instance_object.tags.iteritems()]

        except Exception:
            log.exception("Problem retrieving custom EC2 tags")
            EC2_tags = []

        try:
            if socket_to is None:
                socket_to = 3
            socket.setdefaulttimeout(socket_to)
        except Exception:
            pass

        return EC2_tags
def fetchVideo_DBpedia(videoName):

    def is_person(url, response):
        try:
            for item in response[url.replace('data', 'resource')[:-5]]['http://www.w3.org/1999/02/22-rdf-syntax-ns#type']:
                if item['value'] == 'http://dbpedia.org/ontology/Person':
                    return True
            return False
        except:
            return False

    def find_disambiguates(url, response):
        ret = []
        try:
            for item in response[url.replace('data', 'resource')[:-5]]['http://dbpedia.org/ontology/wikiPageDisambiguates']:
                ret.append(item['value'])
        except:
            pass
        return ret

    try:
        url="http://dbpedia.org/"
        videoName='_'.join(word[0] + word[1:] for word in videoName.title().split())
        titleUrl = url+"data/"+videoName+".json"
        response = json.loads(urllib2.urlopen(titleUrl).read())
        if is_person(titleUrl, response):
            return True
        ds = find_disambiguates(titleUrl, response)
        for d in ds:
            d = d.replace('resource', 'data') + ".json"
            if is_person(d, json.loads(urllib2.urlopen(d).read())):
                return True
    except:
        return False
    return False
    def test_enketo_remote_server_responses(self):
        #just in case if we want to shift the testing back to the main server
        testing_enketo_url = settings.ENKETO_URL
        #testing_enketo_url = 'http://enketo-dev.formhub.org'
        form_id = "test_%s" % re.sub(re.compile("\."), "_", str(time()))
        server_url = "%s/%s" % (self.base_url, self.user.username)
        enketo_url = '%slaunch/launchSurvey' % testing_enketo_url

        values = {
            'format': 'json',
            'form_id': form_id,
            'server_url': server_url
        }
        data = urllib.urlencode(values)
        req = urllib2.Request(enketo_url, data)
        try:
            response = urllib2.urlopen(req)
            response = json.loads(response.read())
            success = response['success']
            if not success and 'reason' in response:
                fail_msg = "This enketo installation is for use by "\
                    "formhub.org users only."
                if response['reason'].startswith(fail_msg):
                    raise SkipTest
            return_url = response['url']
            success = response['success']
            self.assertTrue(success)
            enketo_base_url = urlparse(settings.ENKETO_URL).netloc
            return_base_url = urlparse(return_url).netloc
            self.assertIn(enketo_base_url, return_base_url)
        except urllib2.URLError:
            self.assertTrue(False)

        #second time
        req2 = urllib2.Request(enketo_url, data)
        try:
            response2 = urllib2.urlopen(req2)
            response2 = json.loads(response2.read())
            return_url_2 = response2['url']
            success2 = response2['success']
            reason2 = response2['reason']
            self.assertEqual(return_url, return_url_2)
            self.assertFalse(success2)
            self.assertEqual(reason2, "existing")
        except urllib2.URLError:
            self.assertTrue(False)

        #error message
        values['server_url'] = ""
        data = urllib.urlencode(values)
        req3 = urllib2.Request(enketo_url, data)
        try:
            response3 = urllib2.urlopen(req3)
            response3 = json.loads(response3.read())
            success3 = response3['success']
            reason3 = response3['reason']
            self.assertFalse(success3)
            self.assertEqual(reason3, "empty")
        except urllib2.URLError:
            self.assertTrue(False)
Beispiel #21
0
 def check_url(self, url):
     try:
         urllib2.urlopen(url).headers.getheader('Content-Length')
     except urllib2.HTTPError:
         print("404 error checking url: " + url)
         return False
     return True
Beispiel #22
0
def getSCLeg(partyDict):
    houseSoup = BeautifulSoup(urllib2.urlopen('http://www.scstatehouse.gov/member.php?chamber=H&order=D').read())
    senateSoup = BeautifulSoup(urllib2.urlopen('http://www.scstatehouse.gov/member.php?chamber=S&order=D').read())
    houseTable = houseSoup.find('div', {'class': 'mainwidepanel'}).find_all('div', {'style': 'width: 325px; height: 135px; margin: 0 0 0 20px; text-align: left; float: left;'})
    senateTable = senateSoup.find('div', {'class': 'mainwidepanel'}).find_all('div', {'style': 'width: 325px; height: 135px; margin: 0 0 0 20px; text-align: left; float: left;'})
    dictList = []

    for item in houseTable:
        repInfo = {}
        link = item.find('a')
        if link is not None:
            repInfo['Website'] = 'http://www.scstatehouse.gov' + link.get('href')
            repInfo['Name'] = re.sub(r'\[.*$', '', link.string.strip()).strip().replace('   ', ' ').replace('  ', ' ')
            repInfo['Party'] = partyDict[str(re.sub(r'^.*\[(.*)\].*$', r'\1', link.string.strip()))]
        else:
            repInfo['Name'] = 'VACANT'
        repInfo['District'] = 'SC State House ' + re.sub(r'^.*(District [0-9]*).*$', r'\1', item.get_text())
        dictList.append(repInfo)

    for item in senateTable:
        repInfo = {}
        link = item.find('a')
        if link is not None:
            repInfo['Website'] = 'http://www.scstatehouse.gov' + link.get('href')
            repInfo['Name'] = re.sub(r'\[.*$', '', link.string.strip()).strip().replace('   ', ' ').replace('  ', ' ')
            repInfo['Party'] = partyDict[str(re.sub(r'^.*\[(.*)\].*$', r'\1', link.string.strip()))]
        else:
            repInfo['Name'] = 'VACANT'
        repInfo['District'] = 'SC State Senate ' + re.sub(r'^.*(District [0-9]*).*$', r'\1', item.get_text())
        dictList.append(repInfo)

    return dictList
Beispiel #23
0
	def parse(self,response):
		#Get Access Token for Microsoft Translate
	
		
		atrequest = urllib2.Request('https://datamarket.accesscontrol.windows.net/v2/OAuth2-13')
		atrequest.add_data(atdata)
		atresponse = urllib2.urlopen(atrequest)
		access_token = json.loads(atresponse.read())['access_token']
		
		hxs = HtmlXPathSelector(response) 
		sites = hxs.select('//span[contains(@class, "productsAzLink")]/a/text()').extract()
		items = []
		
		for site in sites:
			text = []
			item = IkeaItem()
			item['name'],_,item['thing'] = unicode(site).partition(' ')
			
			tosend = {'text': unicode(item['name']), 'from' : 'sv' , 'to' : 'en' }
			request = urllib2.Request('http://api.microsofttranslator.com/v2/Http.svc/Translate?'+urllib.urlencode(tosend))
			request.add_header('Authorization', 'Bearer '+access_token)
			response = urllib2.urlopen(request)
			doc = etree.fromstring(response.read())
			
			for elem in doc.xpath('/foo:string', namespaces={'foo': 'http://schemas.microsoft.com/2003/10/Serialization/'}):
				if elem.text:
					elem_text = ' '.join(elem.text.split())
					if len(elem_text) > 0:
						text.append(elem_text)
		
			item['translation'] = ' '.join(text)
			items.append(item)
		return items
def post(user,passwd):
    fp = open("Score.txt", "w")
    login_url="http://www.dean.gxnu.edu.cn/jwxt/index.php/api/user/login"

    data={}
    data['phone']="+8613512345678"
    data['username']=user
    data['password']=passwd
    post_data=urllib.urlencode(data)
    req=urllib2.urlopen(login_url,post_data)
    content=req.read()
    sid=content[56:82]
    data2={}
    data2['session_id']=sid
    url2="http://www.dean.gxnu.edu.cn/jwxt/index.php/api/chengji/getyxcj"
    sessionid="PHPSESSID="+sid
    post_data2=urllib.urlencode(data2)
    req2=urllib2.Request(url2,post_data2)
    req2.add_header('Cookie',sessionid)
    resp=urllib2.urlopen(req2)
    content2=json.loads(resp.read().encode('utf-8'))
    print u"课程名称\t\t成绩\t\t年度/学期\t\tbk\t\tcx\t\t绩点"
    fp.writelines("课程名称\t\t成绩\t\t年度/学期\t\tbk\t\tcx\t\t绩点\n")
    for subject  in content2['msg']:
        print subject['kcmc'] + "\t\t" + subject['cj'] + "\t\t" + subject['ndxq'][:-1] + "/" + subject['ndxq'][-1] + "\t\t" + subject['bk'] + "\t\t" + subject['cx'] + "\t\t" + subject['jd']
#        print "%-40s\t%-10s" % (subject['kcmc'], subject['cj'])
        fp.write(subject['kcmc'] + "\t\t" + subject['cj'] + "\t\t" + subject['ndxq'][:-1] + "/" + subject['ndxq'][-1] + "\t\t" + subject['bk'] + "\t\t" + subject['cx'] + "\t\t" + subject['jd'] + "\n")
    fp.close()
def pullPhotos(query):

    print "looking for", query
    url1 = "https://www.google.com/search?biw=1309&bih=704&sei=bsHjUbvaEILqrQeA-YCYDw&tbs=itp:lineart&tbm=isch&"
    query2 = urllib.urlencode({"q": query})

    req = urllib2.Request(url1 + query2, headers={"User-Agent": "Chrome"})
    response = urllib2.urlopen(req).read()
    parser = MyHTMLParser()
    parser.feed(response)

    print image_lib + "\\buffer\\" + query

    if not os.path.exists(image_lib + "\\buffer"):
        os.mkdir(image_lib + "\\buffer")  # make directory to put them in

    if not os.path.exists(image_lib + "\\buffer\\" + query):
        os.mkdir(image_lib + "\\buffer\\" + query)  # make directory to put them in

    for i in xrange(5):
        req_cat = urllib2.Request(cat_urls[i], headers={"User-Agent": "Chrome"})
        response_cat = urllib2.urlopen(req_cat).read()
        name = query + os.sep + query + str(i) + ".jpg"
        fd = open(image_lib + "\\buffer\\" + name, "wb")
        fd.write(response_cat)
        fd.close()
        print name, "written", "complexity is ", countComponents(image_lib + "\\buffer\\" + name)

    print "done"
Beispiel #26
0
def urlread(url, get={}, post={}, headers={}, timeout=None):
    req = urllib2.Request(url, urllib.urlencode(get), headers=headers)
    try:
        response = urllib2.urlopen(req, urllib.urlencode(post), timeout).read()
    except:
        response = urllib2.urlopen(req, urllib.urlencode(post)).read()
    return response
Beispiel #27
0
 def start(self):
     with QMutexLocker(self.mutex):
         self.stoped = False
         
     #for i in range(self.start_p,self.end_p):
     for i in range(1,3):
         while self.suspended:
             self.wait()  
             return
         if self.stoped:
             return
         url ="http://www.99fang.com/service/agency/a1/?p=%d" % i
         print url            
         
         try:
             r = urllib2.urlopen(url).read()
             soup = BeautifulSoup(r)
             box = soup.find("div",{'class':'agency-call-box'})
             lis = box("li")
             for li in lis:
                 
                 tel = li.a.string
                 print tel
                 r =urllib2.urlopen("http://suzhou.jjr360.com/app.php?c=spider&a=index&city=&tel=%s" % tel)
                 print r.read()
         except:
             pass
         else:
             #self.emit(SIGNAL("updateTime()"))
             time.sleep(1)
Beispiel #28
0
def getcommits_from_project(project):
    global access_token
    url1 = 'https://api.github.com/user'
    request1=Request(url1)
    request1.add_header('Authorization', 'token %s' % access_token)
    response1 = urlopen(request1)
    result1 = json.load(response1)
    person = result1['login']
    repo_info=['Fasta','js2839']
    owner= repo_info[1]
    repo = repo_info[0]
    url = 'https://api.github.com/repos/'+owner+'/'+repo+'/commits'
    data=[]
    request = Request(url)
    request.add_header('Authorization', 'token %s' % access_token)
    response = urlopen(request)
    result = json.load(response)
    for i in range(len(result)):
        print 'result0'
        data.append([result[i]['commit']['message'], result[i]['commit']['author']['name'], result[i]['commit']['author']['date']])
        print data[i]
    for com in data:
        (per,sub_name)=getPercentage(com[0])
        err = save_to_db( per, sub_name, com[1], project, com[2])
    return 
Beispiel #29
0
def getmodelvendor(type,ipaddress):

    if type=="thermostat":
        modeladdress=ipaddress.replace('/sys','/tstat/model')
        deviceModelUrl = urllib2.urlopen(modeladdress)
        if (deviceModelUrl.getcode()==200):
            deviceModel = parseJSONresponse(deviceModelUrl.read().decode("utf-8"),"model")
        deviceVendor = "RadioThermostat"
        deviceModelUrl.close()
        return {'model':deviceModel,'vendor':deviceVendor}
    elif type=="Philips":
        deviceUrl = urllib2.urlopen(ipaddress)
        dom=minidom.parse(deviceUrl)
        deviceModel=dom.getElementsByTagName('modelName')[0].firstChild.data
        deviceVendor=dom.getElementsByTagName('manufacturer')[0].firstChild.data
        deviceUrl.close()
        return {'model':deviceModel,'vendor':deviceVendor}
    elif type=="WeMo":
        deviceUrl = urllib2.urlopen(ipaddress)
        dom=minidom.parse(deviceUrl)
        deviceModel=dom.getElementsByTagName('modelName')[0].firstChild.data
        deviceVendor=dom.getElementsByTagName('manufacturer')[0].firstChild.data
        nickname = dom.getElementsByTagName('friendlyName')[0].firstChild.data
        if str(deviceModel).lower() == 'socket':
            deviceType = dom.getElementsByTagName('deviceType')[0].firstChild.data
            deviceType = re.search('urn:Belkin:device:([A-Za-z]*):1',deviceType).groups()[0]
            if (deviceType.lower() == 'controllee'):
                deviceModel = deviceModel
            else:
                deviceModel = 'Unknown'
        deviceUrl.close()
        return {'model':deviceModel,'vendor':deviceVendor,'nickname':nickname}
Beispiel #30
0
def resolve_novamov(url, guid):
	xbmc.log("Starting resolve_novamov with url: " + str(url) + " and guid: " + str(guid))
	req = urllib2.Request(url)
	req.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
	response = urllib2.urlopen(req)
	link=response.read()
	response.close()

	match1=re.compile('flashvars.file="(.+?)"').findall(link)
	for file in match1:
		file = file

	match2=re.compile('flashvars.filekey="(.+?)"').findall(link)
	for filekey in match2:
		filekey = filekey

	if not match1 or not match2:
		return 'CONTENTREMOVED'

	novaurl = 'http://www.novamov.com/api/player.api.php?user=undefined&key=' + filekey + '&codes=undefined&pass=undefined&file=' + file 

	req = urllib2.Request(novaurl)
	req.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
	response = urllib2.urlopen(req)
	link=response.read()
	response.close()

	match3=re.compile('url=(.+?\.flv)').findall(link)
	for link in match3:
		link = link


	print ('auth url is ' + str(link))
	return link
def getNordVPNPreFetch(vpn_provider):
    # Fetch and store country info from the magical interwebz
    filename = getAddonPath(True, vpn_provider + "/" + NORD_LOCATIONS)
    if xbmcvfs.exists(filename):
        try:
            st = xbmcvfs.Stat(filename)
            create_time = int(st.st_ctime())
            t = now()
            # Fetch again if this is more than a day old otherwise use what there is
            if create_time + 86400 < t:
                debugTrace("Create time of " + filename + " is " +
                           str(create_time) + " time now is " + str(t) +
                           ", fetching country data again")
            else:
                debugTrace("Create time of " + filename + " is " +
                           str(create_time) + " time now is " + str(t) +
                           ", using existing data")
                return True
        except Exception as e:
            errorTrace(
                "alternativeNord.py",
                "List of countries exist but couldn't get the time stamp for "
                + filename)
            errorTrace("alternativeNord.py", str(e))
            return False

    # Download the JSON object of countries
    response = ""
    error = True
    try:
        download_url = "https://api.nordvpn.com/v1/servers/countries"
        if ifHTTPTrace():
            infoTrace("alternativeNord.py",
                      "Downloading list of countries using " + download_url)
        else:
            debugTrace("Downloading list of countries")
        token = getTokenNordVPN()
        req = urllib2.Request(download_url)
        req.add_header("Authorization", "token:" + token)
        t_before = now()
        response = urllib2.urlopen(req)
        country_data = json.load(response)
        t_after = now()
        response.close()
        error = False
        if ifJSONTrace():
            infoTrace(
                "alternativeNord.py",
                "JSON received is \n" + json.dumps(country_data, indent=4))
        if t_after - t_before > TIME_WARN:
            infoTrace(
                "alternativeNord.py", "Downloading list of countries took " +
                str(t_after - t_before) + " seconds")
    except urllib2.HTTPError as e:
        errorTrace(
            "alternativeNord.py",
            "Couldn't retrieve the list of countries for " + vpn_provider)
        errorTrace("alternativeNord.py", "API call was " + download_url)
        errorTrace("alternativeNord.py",
                   "Response was " + str(e.code) + " " + e.reason)
        errorTrace("alternativeNord.py", e.read())
    except Exception as e:
        errorTrace(
            "alternativeNord.py",
            "Couldn't retrieve the list of countries for " + vpn_provider)
        errorTrace("alternativeNord.py", "API call was " + download_url)
        errorTrace("alternativeNord.py",
                   "Response was " + str(type(e)) + " " + str(e))

    if error:
        # Use the existing list of countries if there is one as it'll be pretty much up to date
        if xbmcvfs.exists(filename):
            infoTrace("alternativeNord.py", "Using existing list of countries")
            return True
        else:
            return False

    # Parse the JSON to write out the countries and ID
    try:
        debugTrace("Parsing the JSON and writing the list of countries")
        output = open(filename, 'w')
        for item in country_data:
            name = item["name"].replace(",", "")
            output.write(name + "," + str(item["id"]) + "\n")
        output.close()
        return True
    except Exception as e:
        errorTrace(
            "alternativeNord.py", "Couldn't write the list of countries for " +
            vpn_provider + " to " + filename)
        errorTrace("alternativeNord.py", str(e))

    # Delete the country file if the was a problem creating it.  This will force a download next time through
    try:
        if xbmcvfs.exists(filename):
            errorTrace(
                "alternativeNord.py", "Deleting country file " + filename +
                " to clean up after previous error")
            xbmcvfs.delete(filename)
    except Exception as e:
        errorTrace("alternativeNord.py",
                   "Couldn't delete the country file " + filename)
        errorTrace("alternativeNord.py", str(e))
    return False
Beispiel #32
0
 def execute_script():
     urllib2.urlopen("http://" + sys.argv[1] + ":" + sys.argv[2] +
                     "/?search=%00{.+" + exe + ".}")
Beispiel #33
0
 def __init__(self, link):
     self.url = url.urlopen(link)
     self.name = None
     self.ingredients = None
     self.directions = None
     self.image = None
Beispiel #34
0
        return ""


def main():
    wb = xlrd.open_workbook(city_list)
    sh = wb.sheet_by_index(0)
    r = 1
    while r < len(sh.col(0)):
        pin = int(sh.col_values(1)[r])
        Search_Page(pin)
        r += 1


#url = 'https://shop.advanceautoparts.com/webapp/wcs/stores/servlet/StoreLocatorView?storeId=10151&catalogId=10051&langId=-1&filter=&json=true&rnum=10&max=10&min=1&latitude=37.09024&longitude=-95.71289100000001&resolvedZipcode=67301&stPrefStore=&setStoreCookie=true&address=22011020&radius=50'
url = 'https://shop.advanceautoparts.com/webapp/wcs/stores/servlet/StoreLocatorView?storeId=10151&catalogId=10051&langId=-1&filter=&json=true&rnum=10&max=10&min=1&latitude=36.7266574&longitude=-83.4552486&resolvedZipcode=40863&stPrefStore=&setStoreCookie=true&address=22012005&radius=50'
page = urllib2.urlopen(url)
html_doc = page.read()

data = find_between(html_doc, "var response = [", "];")

#json_data = json.loads(data)

#print data
with open('test.json', 'w') as e:
    e.write(data)

#print len(json_data)
#print json_data['Name']

#print json_data['Address1']
'''
# Parameters
weatherKey = '' #WUnderground API key
timeKey    = '' #TimeZoneDB API key
location   = '' #Query location, can be pws (recommended) or coordinates
timezone   = '' #Timezone, check TimeZoneDB API manual for list of timezones
queryArray = ['geolookup','hourly','forecast10day']

# Get battery percentage
batt_capacity = open('/sys/devices/system/yoshi_battery/yoshi_battery0/battery_capacity', 'r')

# Get weather data from API
queries = {'geolookup':None,'hourly':None,'forecast10day':None}
for (query, result) in queries.items():
    weatherURL = 'http://api.wunderground.com/api/'+weatherKey+'/'+query+'/q/'+location+'.json'
    weatherContent = urllib2.urlopen(weatherURL)
    queries[query] = json.loads(weatherContent.read())
    weatherContent.close()

datetimeURL = 'http://api.timezonedb.com/v2/get-time-zone?key='+timeKey+'&format=json&by=zone&zone='+timezone
datetimeContent = urllib2.urlopen(datetimeURL)
datetime = json.loads(datetimeContent.read())
datetimeContent.close()
datetimeCurrent = time.gmtime(int(datetime['timestamp']))

# Create weather data matrix
weatherData = {}
weatherData['VAR_LOCATION']            = queries['geolookup']['location']['city']
weatherData['VAR_UPDATE_DAY']          = time.strftime('%a', datetimeCurrent)
weatherData['VAR_UPDATE_HOUR']         = time.strftime('%-I', datetimeCurrent)
weatherData['VAR_UPDATE_MINUTE']       = time.strftime('%M', datetimeCurrent)
Beispiel #36
0
    for i in range(len(tds)):  #changing tags to the text within the tag
        tds_text[i] = tds[i].text
    data = {}

    data['rating'] = tds_text[tds_text.index('Purpose:') + 5]
    # If we didn't get anything useful, try a different index
    if (data['rating'] == "Inspection"):
        data['rating'] = tds_text[tds_text.index('Purpose:') + 4]

    data['resulttype'] = data['rating'].split(' ')[0]
    return data


#for testing, use http://dl.dropbox.com/u/58785631/pet-dealer-scraper/petdealerextract.asp.html

doc = urllib2.urlopen(
    "http://www.agriculture.ny.gov/petdealer/petdealerextract.asp").read()
links = SoupStrainer('a', href=re.compile(r'^Pet'))

for el in BeautifulSoup(doc, parseOnlyThese=links):
    suburl = 'http://www.agriculture.ny.gov/petdealer/' + urllib2.quote(
        el["href"], '?=&')
    subdoc = urllib2.urlopen(suburl).read()
    sublinks = SoupStrainer('a', href=re.compile(r'^Insp'))
    print el["href"]
    corp = el["href"].partition('?')[2].partition('&')[2].split('=')[1].split(
        ' - ')[1]
    area = el["href"].partition('?')[2].partition('&')[2].split('=')[1].split(
        ' - ')[0]
    for subel in BeautifulSoup(subdoc, parseOnlyThese=sublinks):
        print subel["href"]
        # splitting on backslash, must escape
Beispiel #37
0
def openURL(url):
	response = urllib2.urlopen(url)
	html = response.read()
	return BeautifulSoup(html)
Beispiel #38
0
 def nc_run():
     urllib2.urlopen("http://" + sys.argv[1] + ":" + sys.argv[2] +
                     "/?search=%00{.+" + exe1 + ".}")
Beispiel #39
0
# coding: utf-8
import urllib2
import json
from bs4 import BeautifulSoup
import HTMLParser

events = []

url = u"http://www.shibuyathegame.com/2015_6.html"
html = urllib2.urlopen(url)
soup = BeautifulSoup(html,"html.parser")
title = None
artist_list = []
event_colums = soup.find_all("div",class_="set-l")
for event_colum in event_colums:
  event_ttls = event_colum.find_all("span",class_="fsize_ll")
  event_texts = event_colum.find("p")
  event = {}
  for event_ttl in event_ttls:
    big_text = event_ttl.string
    if big_text.isdigit() is False:
      #曜日がbig_textに含まれるか判定
      weeks = ["SUN","MON","TUE","WED","THU","FRI","SAT"]
      flag = False
      for day in weeks:
        if day in big_text:
          flag = True
          break
      if flag is False:
        title = big_text
  artist_list = []
Beispiel #40
0
 def script_create():
     urllib2.urlopen("http://" + sys.argv[1] + ":" + sys.argv[2] +
                     "/?search=%00{.+" + save + ".}")
Beispiel #41
0
	#запрос
	url = 'http://babah24.ru/c/1c/getimage.php?salt=%s&crc=%s&id=%s' % (salt, token, i.idbitrix)
	r = requests.get(url)
	#print r.encoding
	#print r.text
	data = r.json()
	
	#вяжем картинку
	try:
		data['pict']
	except:
		pass
	else:
		#print r.status_code, url, data['res'], data['pict']
		img_temp = NamedTemporaryFile(delete=True)
		img_temp.write(urllib2.urlopen('http://babah24.ru%s' % data['pict']).read())
		img_temp.flush()

		f = File(img_temp)
		#f.size
		############i.pict.save(id_generator(), f)
		try:
			i.pict.size #если у товара уже есть картинка
		except: #нет картинки, создаем
			print 'CREATE %s ' % (data['pict'])
			i.pict.save(id_generator(), f)
		else: #если есть картинка
			if i.pict.size != f.size:
				print 'EXIST %s %s' % (i.pict.size, f.size)
				i.pict.save(id_generator(), f)
			else:
Beispiel #42
0
                             path="/v1.0/tenants/" + tenant_id +
                             "/agent_config/",
                             data=json.dumps({"agent_url": url}),
                             method="POST")

    agent_request = json.loads(agent_call)
    agent = False

    c = 0
    while not agent and c < 30:
        sleep(1)
        c += 1
        api_request = urllib2.Request(agent_request['links'][0]['href'], None,
                                      {'X-Auth-Token': token})
        try:
            api_response = urllib2.urlopen(api_request)
            response = api_response.read()
        except urllib2.HTTPError, e:
            print "Got error: " + str(e)
            continue
        agent = json.loads(response)

    if not agent:
        print "Failure: Couldn't parse config."
        do_cleanup_and_quit(region_endpoint['publicURL'], container_name,
                            token)
    else:
        print "Got config from agent."

    # Queue a task
Beispiel #43
0
        body = str.encode(json.dumps(data))


        url = 'https://ussouthcentral.services.azureml.net/workspaces/607045d56e9b4577abd84aab779dc2eb/services/0c379a065d434cf5b5dd6d1e36c3f7cc/execute?api-version=2.0&details=true'
        #url = 'https://ussouthcentral.services.azureml.net/workspaces/8aba4c4d1e034b56941f1f916e884791/services/5f87bea224a14c1e960d32af6656628d/execute?api-version=2.0&details=true'
        api_key = '4vFDqlsZvTEe3PLFSCyXoXmFs3HmWaIgfD1UzdU/SqC7El5gIyVg4AAIIbhExtRlEd59BQk8bIk5E+i1fS66yA=='
        #api_key = 'LZnvjFxHlTdOyAfRr5DimpXaZDWq3J64iIkO+eEFNX88bd956mcCCJsIvJrWr3eKSA+VniEru0IgQ7u9BxuJKw=='


        headers = {'Content-Type':'application/json', 'Authorization':('Bearer '+ api_key)}

        req = urllib2.Request(url, body, headers) 

        try:
            response = urllib2.urlopen(req)

        # If you are using Python 3+, replace urllib2 with urllib.request in the above code:
        # req = urllib.request.Request(url, body, headers) 
        # response = urllib.request.urlopen(req)

            result = response.read()
            print(result)
        except urllib2.HTTPError, error:
            print("The request failed with status code: " + str(error.code))

        # Print the headers - they include the requert ID and the timestamp, which are useful for debugging the failure
            print(error.info())

            print(json.loads(error.read())) 
        
Beispiel #44
0
#-*- coding: utf-8 -*-
import urllib
import urllib2
import CookieManager

challengeUrl = "http://webhacking.kr/challenge/web/web-12/"
CookieManager.addCookie("PHPSESSID", "e28ad7cb81a98a13982054373940bf92")

parameter = urllib.quote("0) or no>1 --")
httpRequest = urllib2.Request(challengeUrl + "?no=" + parameter)
httpRequest.add_header("Cookie", CookieManager.getCookie())

httpConnection = None
try:
    httpConnection = urllib2.urlopen(httpRequest)
    httpResponse = httpConnection.read()
    print httpResponse
except:
    raise
finally:
    if httpConnection != None:
        httpConnection.close()
Beispiel #45
0
def run(locus):
		# ----- CLEAR OLD FILES -----
	img_files = []
	img_files.append("multi_rnaseqgraph.png")
	for img_file in img_files:
	    f = open(img_file, "w+")
	    red_sqr = gd.image((RNA_IMG_WIDTH, RNA_IMG_HEIGHT))
	    red_clr = red_sqr.colorAllocate((255,255,255))
	    red_sqr.rectangle((0,0), (RNA_IMG_WIDTH, RNA_IMG_HEIGHT), red_clr)
	    red_sqr.writePng(f)
	    f.close()

	# ----- GETS MAPPING INFO FOR THE GENE ID -----
	geneid = locus
	map_info = json.loads(urllib2.urlopen("http://bar.utoronto.ca/webservices/araport/gff/get_tair10_gff.php?locus=" + geneid).read())

	start = map_info[u'result'][0][u'start'] if map_info[u'result'][0][u'strand'] == u'+' else map_info[u'result'][0][u'end']
	end = map_info[u'result'][0][u'end'] if map_info[u'result'][0][u'strand'] == u'+' else map_info[u'result'][0][u'start']
	chromosome = int(map_info[u'result'][0][u'chromosome'])

	'''
	Figures out true starts and ends of the CDS based on the information retrieved into map_info.
	'''
	for region in map_info[u'result']:
		if region[u'strand'] == u'+':
			if region[u'start'] < start:
				start = region[u'start']
			if region[u'end'] > end:
				end = region[u'end']
		else:
			if region[u'start'] < start:
				start = region[u'start']
			if region[u'end'] > end:
				end = region[u'end']


	output = ""
	output += "<title>MULTI TRACK RNA-seq Browser</title>"
	generate_exon_graph(map_info, start, end)

	e = xml.etree.ElementTree.parse('data/bamdata_amazon_links.xml')
	# note that even though there are some experiments that should be grouped together, they aren't in the xml file, and so the grey white colouring is not useful
	output += """
	<style>
	td {padding:0px}
	table {border-collapse:collapse}
	svg {height:50px;width:auto}
	</style>
	<table border=1>
	"""

	current_group = [exp.text for exp in e.getroot()[0].find("groupwith").findall("experiment")]
	a = e.getroot()[0].attrib.keys()
	a.sort()
	colour = False
	bold = False
	clr = ""
	for key in a:
		output += "<th>" + key + "</th>"

	for child in e.getroot():
		if current_group != [exp.text for exp in child.find("groupwith").findall("experiment")]:
			colour = not colour
			current_group = [exp.text for exp in child.find("groupwith").findall("experiment")]
		if child.attrib.get('experimentno') in [exp.text for exp in child.find("control").findall("experiment")]:
			#bold this line
			bold = True
		else:
			bold = False
		keys = child.attrib.keys()
		keys.sort()
		#alternate colouring
		output += "<tr style=\""
		if colour:
			output += "background-color:#d3d3d3"
		else:
			output += "background-color:white"
		output += "\">"
		for key in a:
			if key == "foreground":
				try:
					output += "<td style=\"background-color:#" + child.attrib.get(key)[2:] + "\">"
					clr = child.attrib.get(key)[2:]
				except:
					output += "<td>"
			else: 
				output += "<td>" 
			if child.attrib.get(key):
				if bold:
					output += "<b>"
				cases = {
					"url": "<a href='" + child.attrib.get(key) + "'>URL link </a> <br />",
					"publicationid": "<a href='" + child.attrib.get(key) + "'> publication link </a> <br />",
					"name": "<a href='" + child.attrib.get(key) + "'>bam file link </a> <br />", 
					"url": child.attrib.get(key), 
					"img": child.attrib.get(key) + ".png", 
				}
				if key == "svgname":
					output += open("SVGs/" + child.attrib.get(key)[4:], "r").read()
				elif key in cases.keys():
					output += cases[key]
					if key == "name":
						img_file_name = (cases["img"][66:]).replace("/", "_").replace(".bam", "")
						output += "<br/>Read from: " + (cases["img"][66:]).replace("/", "_").replace(".png", "")
						img_file_name = "static/images/" + img_file_name
						output += "<br/>Output went to: " + img_file_name
						generate_rnaseq_graph((cases["img"][66:]).replace("/", "_").replace(".png", ""), img_file_name, clr, geneid, start, end)
						output += '<br/>'
						output += '<img src="' + img_file_name + '">'
						output += '<br/>'
						output += '<img src="static/images/multi_exongraph.png">'
				else:
					output += child.attrib[key]
				if bold:
					output += "</b>"
			output += "</td>"
		output += "</tr>"

	output += "</table>"
	return output
#/usr/bin/env python

import json
import urllib2
from urllib2 import URLError

'''
Errors to catch...
1) No results:                        JSON format error, None
2) Bad URL or no network:             urllib2.URLError: <urlopen error [Errno 11001] getaddrinfo failed>
'''

def search_words_return_titles(search_term):
    try:
        url = "http://www.omdbapi.co/?s={}&y=&plot=short&r=json".format(search_term)
    except URLError, e:
        print e.reason
    response = json.load(urllib2.urlopen(url))
    return response

def present_data(response):
    try:
        for x in response['Search']:
            print x['Title']

    except (ValueError, KeyError, TypeError):
        print "JSON format error"

search_term = raw_input("Search for a word in the film title: ") or "Star"
search_result = search_words_return_titles(search_term)
print present_data(search_result)
    def parse_page_type1(self, response):
        def extract_with_css(query):
            return response.css(query).extract_first()

        imagesA = response.xpath(
            '//div[@class="news-list"]//div[@class="post-content"]//a[img]/@href'
        ).extract()
        imagesA_generated = list()
        for image_file in imagesA:
            if (image_file.startswith('/')):
                image_file = image_file.replace('/',
                                                'http://www.greenpeace.org/',
                                                1)
            imagesA_generated.append(image_file)

        imagesB = response.xpath(
            '//div[@class="news-list"]//div[@class="post-content"]//img/@src'
        ).extract()
        imagesB_generated = list()
        for image_file in imagesB:
            if (image_file.startswith('/')):
                image_file = image_file.replace('/',
                                                'http://www.greenpeace.org/',
                                                1)
            imagesB_generated.append(image_file)

        imagesEnlarge = response.xpath(
            '//div[@class="news-list"]//div[@class="post-content"]//a[@class="open-img BlogEnlargeImage"]/@href'
        ).extract()
        imagesEnlarge_generated = list()
        for image_file in imagesEnlarge:
            if (image_file.startswith('/')):
                image_file = image_file.replace('/',
                                                'http://www.greenpeace.org/',
                                                1)
            imagesEnlarge_generated.append(image_file)

        pdfFiles = response.css(
            'div.news-list a[href$=".pdf"]::attr(href)').extract()
        pdf_files_generated = list()
        for pdf_file in pdfFiles:
            if (pdf_file.startswith('/')):
                pdf_file = pdf_file.replace('/', 'http://www.greenpeace.org/',
                                            1)
            pdf_files_generated.append(pdf_file)

        date_field = response.css('div.news-list .caption::text').re_first(
            r' - \s*(.*)')
        date_field = self.filter_month_name(date_field)
        # Filter extra string part from date.
        date_field = date_field.replace(" at", "")
        date_field = date_field.replace(" à", "")
        date_field = date_field.replace(" kl.", "")
        date_field = date_field.replace(" v", "")

        if date_field:
            date_field = dateutil.parser.parse(date_field)

        image_gallery = response.xpath(
            "//*[contains(@class, 'embedded-image-gallery')]")
        p3_image_gallery = 'false'
        if image_gallery:
            p3_image_gallery = 'true'

        body_text = response.css(
            'div.news-list div.post-content').extract_first()
        if body_text:
            body_text = body_text.replace('src="//', 'src="https://').replace(
                'src="/', 'src="http://www.greenpeace.org/').replace(
                    'href="/', 'href="http://www.greenpeace.org/')
            body_text = body_text.replace('<span class="btn-open">zoom</span>',
                                          '')
            body_text = re.sub('<p dir="ltr">(.*)<\/p>', "\g<1>", body_text)

        body_text = self.filter_post_content(body_text)

        images = response.xpath(
            '//*[@class="post-content"]/div/p/a//img[contains(@style, "float:")]'
        ).extract()  #img[@style="margin: 9px; float: left;"]
        imagesD_generated = list()
        for image in images:
            imagesD_generated.append(image)

        blockquotes = response.xpath(
            '//*[@id="content"]//blockquote').extract()
        blockquotes_generated = list()
        for blockquote in blockquotes:
            blockquotes_generated.append(blockquote)

        author_username = response.xpath(
            'string(//div[@class="news-list"]/ul/li/*/*/span[@class="caption"]/span[@class="green1"]/strong/a/@href)'
        ).extract_first()
        if (author_username != 'None'):
            Segments = author_username.strip().split('/')
            try:  #if ( ( len(Segments) == 4 ) and Segments[4] ):
                if (Segments[4]):
                    author_username = Segments[4]
            except IndexError:
                author_username = ''

        # Get the thumbnail of the post as requested.
        thumbnail = response.xpath(
            'string(head//link[@rel="image_src"]/@href)').extract_first()

        unique_map_id = int(time.time() + random.randint(0, 999))

        # Filter email id image and replace it with email text.
        delete_images = list()
        for image_file in imagesB_generated:
            if ("/emailimages/" in image_file):
                # PHP webservice script url.
                api_url = "http://localhosttest/ocr_webservice/email_img_to_text.php"
                end_point_url = api_url + "?url=" + image_file
                emailid = urllib2.urlopen(end_point_url).read(1000)
                # Search replace the \n, <BR>, spaces from email id.
                emailid = emailid.replace('\n', '')
                emailid = emailid.replace('<br>', '')
                emailid = emailid.replace('<BR>', '')
                emailid = emailid.replace(' ', '')
                delete_images.append(image_file)
                # Remove the email images from Post body and replace it with email text.
                body_text = re.sub(
                    '<img[a-zA-Z0-9="\s\_]*src=\"' + image_file +
                    '\"[a-zA-Z0-9="\s]*>',
                    '<a href="mailto:' + emailid.strip() +
                    '" target="_blank">' + emailid.strip() + '</a>', body_text)

        # Remove the email images from list.
        for image_file in delete_images:
            imagesB_generated.remove(image_file)
        """
        #list images urls
        for image_file in imagesB_generated:
            if ("/emailimages/" in image_file):
                data = [image_file]
                self.csv_writer(data, "email_images_url_list_fr_story.csv")
        """

        yield {
            'type':
            response.meta['p4_post_type'],
            'p3_image_gallery':
            p3_image_gallery,
            'title':
            extract_with_css('div.news-list h1::text'),
            #'subtitle': '',
            'author':
            response.xpath(
                'string(//div[@class="news-list"]/ul/li/*/*/span[@class="caption"]/span[@class="green1"]/strong)'
            ).extract()[0],
            'author_username':
            author_username,
            'date':
            date_field,
            #'lead': extract_with_css('div.news-list div.post-content *:first-child strong::text'),
            'lead':
            response.xpath(
                'string(//div[@class="news-list"]/ul/li/div[@class="post-content"]/div//*[self::p or self::h3 or self::h2][1])'
            ).extract()[0],
            'categories':
            response.meta['categories'],
            'text':
            body_text,
            'imagesA':
            imagesA_generated,
            'imagesEnlarge':
            imagesEnlarge_generated,
            'imagesB':
            imagesB_generated,
            'imagesC':
            response.xpath(
                '//div[@class="gallery"]//div[@class="img-nav"]//a/@rel').
            extract(),  # Galleries (horrible html)
            'imagesD':
            imagesD_generated,
            'blockquote':
            blockquotes_generated,
            'pdfFiles':
            pdf_files_generated,
            'tags1':
            response.meta['tags1'],
            'tags2':
            response.meta['tags2'],
            'tags3':
            response.meta['tags3'],
            'url':
            response.url,
            'status':
            response.meta['status'],
            'map_url':
            '',
            'unique_map_id':
            unique_map_id,
            'thumbnail':
            thumbnail,
        }
    def parse_page_type2(self, response):
        def extract_with_css(query):
            return response.css(query).extract_first()

        imagesA = response.xpath(
            '//div[@class="post-content"]//a[img]/@href').extract()
        imagesA_generated = list()
        for image_file in imagesA:
            if (image_file.startswith('/')):
                image_file = image_file.replace('/',
                                                'http://www.greenpeace.org/',
                                                1)
            imagesA_generated.append(image_file)

        imagesB = response.xpath(
            '//div[@class="post-content"]//img/@src').extract()
        if len(imagesB) == 0:
            imagesB = response.xpath(
                '//div[@id="content"]//img/@src').extract()

        imagesB_generated = list()
        for image_file in imagesB:
            if (image_file.startswith('/')):
                image_file = image_file.replace('/',
                                                'http://www.greenpeace.org/',
                                                1)
            # Custom fix for GPAF only.
            if 'http://assets.pinterest.com/images/PinExt.png' not in image_file:
                imagesB_generated.append(image_file)

        pdfFiles = response.css(
            'div.article a[href$=".pdf"]::attr(href)').extract()
        pdf_files_generated = list()
        for pdf_file in pdfFiles:
            if (pdf_file.startswith('/')):
                pdf_file = pdf_file.replace('/', 'http://www.greenpeace.org/',
                                            1)
            pdf_files_generated.append(pdf_file)

        image_gallery = response.xpath(
            "//*[contains(@class, 'embedded-image-gallery')]")
        p3_image_gallery = 'false'
        if image_gallery:
            p3_image_gallery = 'true'

        try:
            lead_text = response.xpath(
                '//*[@id="content"]/div[4]/div/div[2]/div[1]/div/text()'
            ).extract()[0]
        except IndexError:
            lead_text = ''

        body_text = response.xpath(
            '//*[@id="content"]/div[4]/div/div[2]/div[2]').extract()[0]
        if body_text:
            body_text = body_text.replace('src="//', 'src="https://').replace(
                'src="/', 'src="http://www.greenpeace.org/').replace(
                    'href="/', 'href="http://www.greenpeace.org/')
            body_text = body_text.replace('<span class="btn-open">zoom</span>',
                                          '')
            body_text = re.sub('<p dir="ltr">(.*)<\/p>', "\g<1>", body_text)
            if lead_text:
                body_text = '<div class="leader">' + lead_text + '</div>' + body_text + response.xpath(
                    ' //*[@id="content"]/div[4]/div/div[2]/p').extract_first()

        subtitle = extract_with_css('div.article h2 span::text')
        if subtitle:
            body_text = '<h2>' + subtitle + '</h2><br />' + body_text

        thumbnail = response.xpath(
            'string(head//link[@rel="image_src"]/@href)').extract_first()

        date_field = response.css(
            'div.article div.text span.author::text').re_first(r' - \s*(.*)')
        date_field = self.filter_month_name(date_field)
        # Filter extra string part from date.
        date_field = date_field.replace(" at", "")
        date_field = date_field.replace(" à", "")
        date_field = date_field.replace(" kl.", "")
        date_field = date_field.replace(" v", "")

        if date_field:
            date_field = dateutil.parser.parse(date_field)

        # Filter email id image and replace it with email text.
        delete_images = list()
        for image_file in imagesB_generated:
            if ("/emailimages/" in image_file):
                api_url = "http://localhosttest/ocr_webservice/email_img_to_text.php"
                end_point_url = api_url + "?url=" + image_file
                emailid = urllib2.urlopen(end_point_url).read(1000)
                # Search replace the \n, <BR>, spaces from email id.
                emailid = emailid.replace('\n', '')
                emailid = emailid.replace('<br>', '')
                emailid = emailid.replace('<BR>', '')
                emailid = emailid.replace(' ', '')
                delete_images.append(image_file)
                # Remove the email images from Post body and replace it with email text.
                body_text = re.sub(
                    '<img[a-zA-Z0-9="\s\_]*src=\"' + image_file +
                    '\"[a-zA-Z0-9="\s]*>',
                    '<a href="mailto:' + emailid.strip() +
                    '" target="_blank">' + emailid.strip() + '</a>', body_text)

        # Remove the email images from list.
        for image_file in delete_images:
            imagesB_generated.remove(image_file)
        """
        #list images urls
        for image_file in imagesB_generated:
            if ("/emailimages/" in image_file):
                data = [image_file]
                self.csv_writer(data, "email_images_url_list_fr.csv")
        """

        # Post data mapping logic start.
        unique_map_id = int(time.time() + random.randint(0, 999))
        map_url = ''
        """
        if "/en/" in response.url:
            # For English language POSTs

            # Check the POST transalation availability
            try:
                map_url = response.xpath('//*[@class="language"]//option[2]/@value').extract()[0]
            except IndexError:
                map_url = ''

            if "/fr/" not in map_url:
                map_url = ''

            if map_url:
                map_url = 'http://www.greenpeace.org' + map_url

                # The Post mapping data added into csv file.
                data = [unique_map_id, response.url, map_url]
                self.csv_writer(data, self.__connector_csv_filename)

                data = [response.url, response.meta['p4_post_type'], response.meta['categories'], response.meta['tags1'], response.meta['tags2'], response.meta['tags3'], response.meta['post_type'], response.meta['action']]
                self.csv_writer(data, "Language_mapping_en_list.csv")
        else:
            # For French language POSTs

            # Check the POST transalation if available
            try:
                map_url = response.xpath('//*[@class="language"]//option[1]/@value').extract()[0]
            except IndexError:
                map_url = ''

            if "/en/" not in map_url:
                map_url = ''

            if map_url:
                map_url = 'http://www.greenpeace.org' + map_url

                with open(self.__connector_csv_filename, "rb") as file_obj:
                    reader = csv.reader(file_obj)
                    for row in reader:
                        if (row[1] == map_url or row[2] == response.url):
                            #print "=======Match found======="
                            unique_map_id = row[0]
                            # Log the details
                            data = ["FR==>", unique_map_id, response.url, map_url,"EN==>", row[0], row[1], row[2]]
                            #print data
                            self.csv_writer(data, self.__connector_csv_log_file)

                            data = [response.url, response.meta['p4_post_type'], response.meta['categories'],
                                    response.meta['tags1'], response.meta['tags2'], response.meta['tags3'],
                                    response.meta['post_type'], response.meta['action']]
                            self.csv_writer(data, "Language_mapping_fr_list.csv")
        # Post data mapping logic ends.
        """

        yield {
            'type':
            response.meta['p4_post_type'],
            'p3_image_gallery':
            p3_image_gallery,
            'title':
            extract_with_css('div.article h1 span::text'),
            #'subtitle': '',
            'author':
            'Greenpeace Česká republika',
            'author_username':
            '******',
            #'date': response.css('#content > div.happen-box.article > div > div.text > span').re_first(r' - \s*(.*)'),
            'date':
            date_field,
            #'lead': extract_with_css('div.news-list div.post-content *:first-child strong::text'),
            'lead':
            extract_with_css(
                '#content > div.happen-box.article > div > div.text > div.leader > div'
            ),
            'categories':
            response.meta['categories'],
            #'text':  response.css('div.news-list div.post-content').extract_first(),
            'text':
            body_text,
            'imagesA':
            imagesA_generated,
            #'imagesB': response.xpath('//div[@class="news-list"]//div[@class="post-content"]//img[not(ancestor::a)]/@src').extract(), #don't import image if there's an a tag around it
            'imagesB':
            imagesB_generated,
            'imagesC':
            response.xpath(
                '//div[@class="gallery"]//div[@class="img-nav"]//a/@rel').
            extract(),  # Galleries (horrible html)
            'pdfFiles':
            pdf_files_generated,
            'tags1':
            response.meta['tags1'],
            'tags2':
            response.meta['tags2'],
            'tags3':
            response.meta['tags3'],
            'map_url':
            map_url,
            'unique_map_id':
            unique_map_id,
            'url':
            response.url,
        }
Beispiel #49
0
import pandas as pd

if not len(sys.argv) == 4:
    print(
        "Invalid number of arguments. Run as: python get_bus_info_bja282.py APIKey BusLine file_to_write_to.csv"
    )
    sys.exit()

MTA_KEY = sys.argv[1]
BUS_LINE = sys.argv[2]

#url = "http://api.openweathermap.org/data/2.5/forecast/daily?q=%s&mode=%s&units=%s&cnt=7&APPID=%s"%(city, mode, units, apikey)
url = "http://bustime.mta.info/api/siri/vehicle-monitoring.json?key=%s&VehicleMonitoringDetailLevel=calls&LineRef=%s" % (
    MTA_KEY, BUS_LINE)

response = urllib2.urlopen(url)
data = response.read().decode("utf-8")

#use the json.loads method to obtain a dictionary representation of the responose string
mtabuses = json.loads(data)
bus_activity = (
    mtabuses['Siri']['ServiceDelivery']['VehicleMonitoringDelivery'])
busesontheroad = len(bus_activity[0]['VehicleActivity'])

#Answer Format:
#Latitude,Longitude,Stop Name,Stop Status
#40.755489,-73.987347,7 AV/W 41 ST,at stop
#40.775657,-73.982036,BROADWAY/W 69 ST,approaching
#40.808332,-73.944979,MALCOLM X BL/W 127 ST,approaching
#40.764998,-73.980416,N/A,N/A
#40.804702,-73.947620,MALCOLM X BL/W 122 ST,< 1 stop away
def getNordVPNLocation(vpn_provider, location, server_count, just_name):
    # Return friendly name and .ovpn file name
    # Given the location, find the country ID of the servers
    addon = xbmcaddon.Addon(getID())

    filename = getAddonPath(True, vpn_provider + "/" + NORD_LOCATIONS)
    # If the list of countries doesn't exist (this can happen after a reinstall)
    # then go and do the pre-fetch first.  Otherwise this shouldn't be necessary
    try:
        if not xbmcvfs.exists(filename):
            getNordVPNPreFetch(vpn_provider)
    except Exception as e:
        errorTrace(
            "alternativeNord.py",
            "Couldn't download the list of countries to get ID for " +
            vpn_provider + " from " + filename)
        errorTrace("alternativeNord.py", str(e))
        return "", "", "", False

    try:
        locations_file = open(filename, 'r')
        locations = locations_file.readlines()
        locations_file.close()
        id = ""
        for l in locations:
            country, id = l.split(",")
            id = id.strip(' \t\n\r')
            if location == country:
                break
        if id == "":
            errorTrace(
                "alternativeNord.py", "Couldn't retrieve location " +
                location + " for " + vpn_provider + " from " + filename)
            return "", "", "", False
    except Exception as e:
        errorTrace(
            "alternativeNord.py",
            "Couldn't read the list of countries to get ID for " +
            vpn_provider + " from " + filename)
        errorTrace("alternativeNord.py", str(e))
        return "", "", "", False

    # Generate the file name from the location
    location_filename = getNordVPNLocationName(vpn_provider, location)

    if just_name: return location, location_filename, "", False

    # Download the JSON object of servers
    response = ""
    error = True
    try:
        if "UDP" in addon.getSetting("vpn_protocol"): protocol = "udp"
        else: protocol = "tcp"
        download_url = "https://api.nordvpn.com/v1/servers/recommendations?filters[servers_technologies][identifier]=openvpn_" + protocol + "&filters[country_id]=" + id + "&filters[servers_groups][identifier]=legacy_standard"
        if ifHTTPTrace():
            infoTrace(
                "alternativeNord.py",
                "Downloading server info for " + location + " with ID " + id +
                " and protocol " + protocol + " using " + download_url)
        else:
            debugTrace("Downloading server info for " + location +
                       " with ID " + id + " and protocol " + protocol)
        token = getTokenNordVPN()
        req = urllib2.Request(download_url)
        req.add_header("Authorization", "token:" + token)
        t_before = now()
        response = urllib2.urlopen(req)
        server_data = json.load(response)
        t_after = now()
        response.close()
        error = False
        if ifJSONTrace():
            infoTrace(
                "alternativeNord.py",
                "JSON received is \n" + json.dumps(server_data, indent=4))
        if t_after - t_before > TIME_WARN:
            infoTrace(
                "alternativeNord.py", "Downloading server info for " +
                location + " with ID " + id + " and protocol " + protocol +
                " took " + str(t_after - t_before) + " seconds")
    except urllib2.HTTPError as e:
        errorTrace(
            "alternativeNord.py", "Couldn't retrieve the server info for " +
            vpn_provider + " location " + location + ", ID " + id)
        errorTrace("alternativeNord.py", "API call was " + download_url)
        errorTrace("alternativeNord.py",
                   "Response was " + str(e.code) + " " + e.reason)
        errorTrace("alternativeNord.py", e.read())
    except Exception as e:
        errorTrace(
            "alternativeNord.py", "Couldn't retrieve the server info for " +
            vpn_provider + " location " + location + ", ID " + id)
        errorTrace("alternativeNord.py", "API call was " + download_url)
        errorTrace("alternativeNord.py",
                   "Response was " + str(type(e)) + " " + str(e))

    if error:
        # If there's an API connectivity issue but a location file exists then use that
        # Won't have the latest best location in it though
        if xbmcvfs.exists(location_filename):
            infoTrace("alternativeNord.py",
                      "Using existing " + location + " file")
            return location, location_filename, "", False
        else:
            return "", "", "", False

    # First server is the best one, but if it didn't connect last time then skip it.  The last attempted server
    # will be cleared on a restart, or a successful connection.  If there are no more connections to try, then
    # it will try the first connection again.  However, if this is > 4th attempt to connect outside of the
    # validation then it'll always default to the best as it's likely there's a network rather than server problem
    last = getVPNRequestedServer()
    if not last == "" and server_count < 5:
        debugTrace(
            "Server " + last +
            " didn't connect last time so will be skipping to the next server."
        )
        last_found = False
    else:
        last = ""
        last_found = True
    first_server = ""
    for item in server_data:
        name = item["name"]
        server = item["hostname"]
        status = item["status"]
        load = str(item["load"])
        #debugTrace("Next is " + name + ", " + server + ", " + status + ". Load is " + load)
        if status == "online":
            if first_server == "": first_server = server
            if last_found:
                debugTrace("Using " + name + ", " + server +
                           ", online. Load is " + load)
                break
            if server == last: last_found = True
        server = ""
    if server == "": server = first_server
    setVPNRequestedServer(server)

    # Fetch the ovpn file for the server
    if not server == "":
        if not getNordVPNOvpnFile(server, protocol, location_filename):
            if not xbmcvfs.exists(location_filename):
                return "", "", "", False
        return location, location_filename, "", False
    else:
        return "", "", "", False
Beispiel #51
0
    def confirmPlan(self, id):
        res = urllib2.urlopen(self.base + '/api/confirm?id=%d' % (id)).read()
        jres = json.loads(res)

        if jres['ret'] > 0:
            print "Plan %d confirmed" % (id)
Beispiel #52
0
def writeTeamTotals(gamelist, scorelist):
    delim = ","
    fr = open(gamelist, "r")
    fr.readline()  # first line is data names
    games = fr.readlines()
    fw = open(scorelist, "a")
    keys = [
        "gameid", "gameid_num", "away", "home", "tm", "line", "player_code",
        "pos", "min", "fgm", "fga", "3pm", "3pa", "ftm", "fta", "+/-", "off",
        "def", "tot", "ast", "pf", "st", "to", "bs", "ba", "pts"
    ]
    Ncols = len(keys)
    #fw.write(delim.join(keys) + "\n")
    for game in games:
        game = game.split("\n")[0].split(delim)
        gameid = game[1]
        print gameid  # debug
        date = gameid[:8]
        teams = game[2:4]
        seasonyear = str(
            int(date[0:4]) - (int(date[4:6]) < 8)
        )  # new year's games are part of prev season until october
        line = getLine(gameid)
        #line = [100 100]
        #url = "http://www.nba.com/games/game_component/dynamic/" + date + "/" + teams + "/pbp_all.xml"
        url = "http://data.nba.com/data/10s/html/nbacom/" + seasonyear + "/gameinfo/" + date + "/" + game[
            0] + "_boxscore_csi.html"
        soup = BeautifulSoup(urllib2.urlopen(url).read())
        # get to table
        teamstats = soup.find_all("table")

        #
        ## away team
        #
        # stat totals
        totalstats = teamstats[0].find_all('tr')[-2]
        I = [gameid, game[0], teams[0], teams[1], teams[0], str(line[0])]
        entries = totalstats.find_all('td')
        I.append("total")
        for e in entries[1:]:
            try:
                temp = str(e.text)
            except:
                temp = ''
            if temp == '&nbsp;':
                temp = ''
            if re.match("([\d]+)-([\d]+)", temp):
                temp = temp.split("-")
                for t in temp:
                    I.append(t)
            else:
                I.append(temp)
        if len(I) == Ncols:
            fw.write(delim.join(I) + "\n")
        #
        ## home team
        #
        # stat totals
        totalstats = teamstats[1].find_all('tr')[-2]
        I = [gameid, game[0], teams[0], teams[1], teams[1], str(line[1])]
        entries = totalstats.find_all('td')
        I.append("total")
        for e in entries[1:]:
            try:
                temp = str(e.text)
            except:
                temp = ''
            if temp == '&nbsp;':
                temp = ''
            if re.match("([\d]+)-([\d]+)", temp):
                temp = temp.split("-")
                for t in temp:
                    I.append(t)
            else:
                I.append(temp)
        if len(I) == Ncols:
            fw.write(delim.join(I) + "\n")
    fw.close()
Beispiel #53
0
#coding=utf-8
import urllib2
u = urllib2.urlopen(u'http://tycho.usno.navy.mil/cgi-bin/timer.pl')
print u'in with'
print u
for line in u:
    #print line
    if u'EDT' in line:
        print line
Beispiel #54
0
import scraperwiki
import lxml.html
import urllib2
from urllib2 import urlopen
from lxml.html import parse

html = scraperwiki.scrape("http://lurnq.com/")
page = urlopen('http://lurnq.com/')
p = parse(page)
p.getroot()
root = lxml.html.fromstring(html)


def ctext(el):
    result = []
    if el.text:
        result.append(el.text)
    for sel in el:
        assert sel.tag in ["b", "i"]
        result.append("<" + sel.tag + ">")
        result.append(ctext(sel))
        result.append("</" + sel.tag + ">")
        if sel.tail:
            result.append(sel.tail)
    return "".join(result)


tree = lxml.html.fromstring(html)
tree.xpath('//table[@class="quotes"]/tr')
print tree[1].tag  # i
print tree[1].getparent().tag  # h2
Beispiel #55
0
    def confirmAllPlans(self):
        jres = json.loads(urllib2.urlopen(self.base + '/api/list?status=approved').read())

        for plan in jres['plans']:
            print "Confirming plan %d: %s for %g seconds in %s filter" % (plan['id'], plan['name'], plan['exposure'], plan['filter'])
            self.confirmPlan(plan['id'])
Beispiel #56
0
def getLine(
        gameid):  # get final money line from vegasinsider.com for future work
    # dictionary of name conventions used on the site
    teamNames = {
        "ATL": "hawks",
        "BOS": "celtics",
        "BKN": "nets",
        "CHA": "hornets",
        "CHI": "bulls",
        "CLE": "cavaliers",
        "DAL": "mavericks",
        "DEN": "nuggets",
        "DET": "pistons",
        "GSW": "warriors",
        "HOU": "rockets",
        "IND": "pacers",
        "LAC": "clippers",
        "LAL": "lakers",
        "MEM": "grizzlies",
        "MIA": "heat",
        "MIL": "bucks",
        "MIN": "timberwolves",
        "NOP": "pelicans",
        "NYK": "knicks",
        "OKC": "thunder",
        "ORL": "magic",
        "PHI": "76ers",
        "PHX": "suns",
        "POR": "trail-blazers",
        "SAC": "kings",
        "SAS": "spurs",
        "TOR": "raptors",
        "UTA": "jazz",
        "WAS": "wizards"
    }
    date = "-".join([gameid[4:6], gameid[6:8], gameid[2:4]])
    url = u"http://www.vegasinsider.com/nba/odds/las-vegas/line-movement/" + teamNames[
        gameid[8:11]] + "-@-" + teamNames[gameid[11:14]] + ".cfm/date/" + date
    # spoof an alternate user-agent: works for vegasinsider
    # courtesy of stackoverflow
    headers = {'User-Agent': "Mozilla"}
    request = urllib2.Request(url, headers=headers)
    response = urllib2.urlopen(request)
    with open("tempout.txt", "wb") as outfile:
        shutil.copyfileobj(response, outfile)
    html = open("tempout.txt", "rb").read()
    # pseudocode very gross
    # Jump to VI CONSENSUS LINE MOVEMENTS
    # Jump to end of table (header for VI)
    # Get following <TABLE> of lines
    # Start at last row of this table and work backwards until a moneyline is extracted
    txt = html[re.search("VI CONSENSUS LINE MOVEMENTS", html).start():]
    txt = txt[re.search("</TABLE>", txt).end():]  # get following table (1)
    txt = txt[0:re.search("</TABLE>", txt).end():]  # get following table (2)
    txt = txt.split("<TR>")  # break up table rows
    gotLine = False
    maxRows = round(0.5 * len(txt))
    trind = -1
    while not gotLine and abs(trind) < maxRows:
        try:
            txt0 = txt[trind].split("</TD>")
            txt1 = txt0[2][re.search("<TD.*>", txt0[2]).end():].strip()
            txt2 = txt0[3][re.search("<TD.*>", txt0[3]).end():].strip()
            if re.search(gameid[8:11], txt1):  # if away team is favorite
                l1 = int(re.search("([+-][\d]+)", txt1).groups()[0])
                try:
                    l2 = int(re.search("([+-][\d]+)", txt2).groups()[0])
                except:  # handles case when money line = 0 bc there is no +/-
                    l2 = int(re.search("([\d]+)", txt2).groups()[0])
            else:  # if home team is favorite
                try:
                    l1 = int(re.search("([+-][\d]+)", txt2).groups()[0])
                except:  # handles case when money line = 0 bc there is no +/-
                    l1 = int(re.search("([\d]+)", txt2).groups()[0])
                l2 = int(re.search("([+-][\d]+)", txt1).groups()[0])
            gotLine = True
        except:  # if this parsing fails, go back a row
            l1 = None
            l2 = None
            trind -= 1
    return [l1, l2]
Beispiel #57
0
    def deleteAllTargets(self):
        jres = json.loads(urllib2.urlopen(self.base + '/api/list_targets').read())

        for target in jres['targets']:
            print "Deleting target %d: %s" % (target['id'], target['name'])
            self.deleteTarget(target['id'])
Beispiel #58
0
def get_image(url):
    image_url = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
    image_byt = urlopen(image_url).read()
    #print('here' + image_byt)
    image_b64 = base64.encodebytes(image_byt)
    return image_b64
Beispiel #59
0
    def deleteTarget(self, id=0):
        res = urllib2.urlopen(self.base + '/api/delete_target?id=%d' % (int(id))).read()
        jres = json.loads(res)

        if jres['ret'] > 0:
            print "Target %d deleted" % (id)
Beispiel #60
0
    def cancelPlan(self, id):
        res = urllib2.urlopen(self.base + '/api/cancel?id=%d' % (id)).read()
        jres = json.loads(res)

        if jres['ret'] > 0:
            print "Plan %d cancelled" % (id)