Example #1
1
def reportStateMessage(vmHostname, message, serverHostname=None):
    """Sends a state message to the Server
      Keyword arguments:
      vmHostname -- Hostname of the virtual machine
      message -- message to be sent
  """
    if serverHostname == None:
        serverHostname = vnConfigNode.getServerHostname()
    message = message.replace(" ", "%20")
    logging.info("sending a state message " + message + " to " + serverHostname)
    logging.info(
        proto
        + serverHostname
        + path
        + "?vmAction=setstatemessage&vmHostname="
        + vmHostname
        + "&stateMessage="
        + message
    )
    urllib2.urlopen(
        proto
        + serverHostname
        + path
        + "?vmAction=setstatemessage&vmHostname="
        + vmHostname
        + "&stateMessage="
        + message
    ).close()
Example #2
1
def on_drag_data_received(widget, context, x, y, selection, target_type, timestamp):
    if target_type == TARGET_TYPE_URI_LIST:
        uri = selection.get_data().strip("\r\n\x00")
        print "uri", uri
        uri_splitted = uri.split()  # we may have more than one file dropped
        for uri in uri_splitted:
            path = get_file_path_from_dnd_dropped_uri(uri)
            print "path to open", path
            if os.path.isfile(path):  # is it file?
                data = file(path).read()
                # print data
            thread.start_new_thread(serve, (path, 1))
            filename = os.path.splitext(os.path.split(path)[1])[0]
            extension = os.path.splitext(path)[1]
            encoded_string = urllib.quote_plus("http://" + localip + ":8080/" + base64.b32encode(filename) + extension)
            full_url = "http://" + w.entry.get_text() + ":2020/stream?url="
            if os.path.isfile(os.path.splitext(path)[0] + ".srt"):
                urllib2.urlopen(
                    full_url
                    + encoded_string
                    + "&subtitles="
                    + urllib.quote_plus(
                        "http://" + localip + ":8080/" + base64.b32encode(os.path.splitext(os.path.split(path)[1])[0])
                    )
                    + ".srt"
                ).read()
            else:
                urllib2.urlopen(full_url + encoded_string).read()
Example #3
1
def get_feed(max_feed=5):
    """
		This function is responsible to get RSS feeds.
	"""
    try:
        f = urllib2.urlopen("http://feeds.feedburner.com/XSSPosed")
    except urllib2.URLError:
        raise Exception("Internet connection problems")

    tree = xml_parser.fromstring(f.read())
    channel = tree.find("channel")
    items = channel.findall("item")
    for item in reversed(list(items)[0:max_feed]):
        link = list(item.iter("link"))[0].text
        if link not in CONTAINER:
            CONTAINER.add(link)
            site_response = urllib2.urlopen(link).read()
            details = get_details(site_response)
            yield {
                "title": list(item.iter("title"))[0].text,
                "description": details["description"],
                "link": link,
                "status": str(details["status"]),
                "exploit": get_exploit(site_response),
                "post_data": details["post_data"],
            }
def renren_set(email, password, text):
    import urllib2, cookielib, urllib
    from BeautifulSoup import BeautifulSoup

    cj = cookielib.CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
    urllib2.install_opener(opener)
    postdata = urllib.urlencode(
        {
            "email": email,
            "password": password,
            "captcha_type": "web_login",
            "icode": "",
            "origURL": "http://www.renren.com/indexcon",
            "domain": "renren.com",
            "key_id": "1",
        }
    )
    req = urllib2.Request(url="http://www.renren.com/PLogin.do", data=postdata)
    response = urllib2.urlopen(req)
    content = response.read()
    j = content[content.find("script") :]
    rtk = j[j.find("get_check_x:") + 13 : j.find("get_check_x:") + 21]  # 寻找rtk,rtk是人人的一个校验码,每次登陆不同。
    newdata = urllib.urlencode({"_rtk": rtk, "channel": "renren", "content": text, "hostid": "729595072"})
    newreq = urllib2.Request(url="http://shell.renren.com/729595072/status", data=newdata)
    newcontent = urllib2.urlopen(newreq)
Example #5
0
def getTop100Info(resultsPageUrl, pageIndex, destinationFolder):
    try:
        response = urllib2.urlopen(resultsPageUrl + str(pageIndex))
        html = response.read()

        fromRank = (pageIndex - 1) * 20 + 1
        toRank = fromRank + 19
        localFile = open(os.path.join(destinationFolder, str(fromRank) + "-" + str(toRank) + ".html"), "w")
        localFile.write(html)
        localFile.close()

        ebooksDestinationFolder = os.path.join(destinationFolder, "ebooks")
        if not os.path.exists(ebooksDestinationFolder):
            os.makedirs(ebooksDestinationFolder)

        # Must use the lxml HTML r even though it has external dependecies, because the default python one is not good enough to  the Kijiji pages
        # lxml Windows distribution was downloaded from http://www.lfd.uci.edu/~gohlke/pythonlibs/#lxml
        soup = BeautifulSoup(html, "lxml")

        # Extract details of each book in the top 100
        for bookDiv in soup.body.find_all("div", {"class", "zg_itemImmersion"}):
            itemWrapper = bookDiv.find("div", {"class", "zg_itemWrapper"})
            ebookUrl = itemWrapper.find("div", {"class", "zg_itemImageImmersion"}).a["href"].strip()
            asin = ebookUrl[-10:]
            response = urllib2.urlopen(ebookUrl)
            ebookHtml = response.read()
            ebookFileName = os.path.join(ebooksDestinationFolder, asin + ".html")
            ebookFile = open(ebookFileName, "w")
            ebookFile.write(ebookHtml)
            ebookFile.close()
            print(asin)

    except urllib2.URLError as e:
        print("Failed to download from " + resultsPageUrl + ":" + e.reason)
Example #6
0
def dif_html(a, b, lower_criteria=0.5, upper_criteria=0.7, link_criteria=0.4, img_criteria=0.2, script_criteria=0.2):
    from urllib2 import urlopen
    from difflib import SequenceMatcher
    from my_stats import jaccard as jac

    # 詳細にページをみる
    try:
        html_a, html_b = urlopen(a).read(), urlopen(b).read()
        matcher = SequenceMatcher()
        matcher.set_seq1(html_a)
        matcher.set_seq2(html_b)
        if matcher.ratio() >= lower_criteria:
            # lower_criteria以上だけどupper_criteria以下の場合にはリンク構造など詳しく調査する
            if matcher.ratio() >= upper_criteria:
                print "white", matcher.ratio(), upper_criteria, lower_criteria, a, b
                return True
            else:
                # ここにnon-negative-matrix-factorizationを入れてより精緻に分析する予定だったけど、まだあまりうまく行かないのでペンディング(T.B.D.)
                print "grey", matcher.ratio(), upper_criteria, lower_criteria, a, b
                links_a, titles_a, imgs_a, scripts_a = analyze_html(html_a)
                links_b, titles_b, imgs_b, scripts_b = analyze_html(html_b)
                j_links, j_imgs, j_scripts = jac(links_a, links_b), jac(imgs_a, imgs_b), jac(imgs_a, imgs_b)
                if j_links >= link_criteria and j_imgs >= img_criteria and j_scripts >= script_criteria:
                    return True
                else:
                    return False
        # lower_criteria以上に似てない部分がある場合には似てないと判断する
        else:
            print "black", matcher.ratio(), upper_criteria, lower_criteria, a, b
            return False
    except:
        # クローリングできない場合には異なるページと見なす
        return False
def webservice(url, retries=RETRIES, postdata=False):
    """
    A wrapper around urllib.urlopen to allow for retries
    :param url: The url to fetch content from.
    :param retries: The number of times to retry the urlopen method.
    :return: A dictionary with a boolean at success, and
    """
    attempt = 0
    data = [False, ""]
    while attempt < retries:
        try:
            if postdata is False:
                logging.info("urlopen: GET %s" % url)
                data[1] = urllib2.urlopen(url, timeout=TIMEOUT).read().strip()
                data[0] = True
            else:
                logging.info("urlopen: POST %s" % url)
                data[1] = urllib2.urlopen(url, postdata, timeout=TIMEOUT).read().strip()
                data[0] = True
            logging.info("urlopen: RESP received %s" % data[1].replace("\n", r"\n"))
            break
        except urllib2.HTTPError, e:
            # Handle specific error types
            code = e.getcode()
            logging.error("Received HTTP response code: %d - %s" % (code, data[1]))
        except BaseException as e:
            logging.error(e.message)
Example #8
0
 def publish(self, url):
     """Try to publish this descriptor to a given directory URL.  Returns
        'accept' if the publication was successful, 'reject' if the
        server refused to accept the descriptor, and 'error' if
        publication failed for some other reason."""
     fname = self.getDescriptorFileName()
     descriptor = readFile(fname)
     fields = urllib.urlencode({"desc": descriptor})
     f = None
     try:
         try:
             #############################################
             # some python versions verify certificates
             # anemone.mooo.com uses a self-signed cert
             # this workaround is not a problem because
             # the directory information is already signed
             # (although as Zax says, it is certainly a
             # kludge ;)
             if sys.version_info >= (2, 7, 9):
                 ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
                 ctx.check_hostname = False
                 ctx.verify_mode = ssl.CERT_NONE
                 f = urllib2.urlopen(url, fields, context=ctx)
             else:
                 f = urllib2.urlopen(url, fields)
             #############################################
             # f = urllib2.urlopen(url, fields)
             info = f.info()
             reply = f.read()
         except IOError, e:
             LOG.error("Error while publishing server descriptor: %s", e)
             return "error"
         except:
             LOG.error_exc(sys.exc_info(), "Error publishing server descriptor")
             return "error"
Example #9
0
def getXMLPageCore(headers={}, params={}, config={}):
    """  """
    # returns a XML containing params['limit'] revisions (or current only), ending in </mediawiki>
    # if retrieving params['limit'] revisions fails, returns a current only version
    # if all fail, returns the empty string
    xml = ""
    c = 0
    maxseconds = 100  # max seconds to wait in a single sleeping
    maxretries = 5  # x retries and skip
    increment = 20  # increment every retry
    while not re.search(r"</mediawiki>", xml):
        if c > 0 and c < maxretries:
            wait = increment * c < maxseconds and increment * c or maxseconds  # incremental until maxseconds
            print '    XML for "%s" is wrong. Waiting %d seconds and reloading...' % (params["pages"], wait)
            time.sleep(wait)
            if (
                params["limit"] > 1
            ):  # reducing server load requesting smallest chunks (if curonly then limit = 1 from mother function)
                params["limit"] = params["limit"] / 2  # half
        if c >= maxretries:
            print "    We have retried %d times" % (c)
            print '    MediaWiki error for "%s", network error or whatever...' % (params["pages"])
            # If it's not already what we tried: our last chance, preserve only the last revision...
            # config['curonly'] means that the whole dump is configured to save nonly the last
            # params['curonly'] means that we've already tried this fallback, because it's set by the following if and passed to getXMLPageCore
            if not config["curonly"] and not params["curonly"]:
                print "    Trying to save only the last revision for this page..."
                params["curonly"] = 1
                logerror(
                    config=config,
                    text='Error while retrieving the full history of "%s". Trying to save only the last revision for this page'
                    % (params["pages"]),
                )
                return getXMLPageCore(headers=headers, params=params, config=config)
            else:
                print "    Saving in the errors log, and skipping..."
                logerror(
                    config=config,
                    text='Error while retrieving the last revision of "%s". Skipping.' % (params["pages"]),
                )
                return ""  # empty xml

        data = urllib.urlencode(params)
        req = urllib2.Request(url=config["index"], data=data, headers=headers)
        try:
            f = urllib2.urlopen(req)
        except:
            try:
                print "Server is slow... Waiting some seconds and retrying..."
                time.sleep(15)
                f = urllib2.urlopen(req)
            except:
                print 'An error have occurred while retrieving "%s"' % (params["pages"])
                print "Please, resume the dump, --resume"
                sys.exit()
                # The error is usually temporary, but we exit the dump altogether.
        xml = f.read()
        c += 1

    return xml
def post_to_facebook(app_access_token, user_social_auth, message, link):
    url = "https://graph.facebook.com/%s/feed" % user_social_auth.uid

    params = {"access_token": app_access_token, "message": message, "link": link}

    req = urllib2.Request(url, urllib.urlencode(params))
    urllib2.urlopen(req)
Example #11
0
    def _check_prerequisites(self):
        """Sanity checks before attempting to mount SOFS."""

        # config is mandatory
        config = CONF.scality_sofs_config
        if not config:
            msg = _("Value required for 'scality_sofs_config'")
            LOG.warn(msg)
            raise exception.NovaException(msg)

        # config can be a file path or a URL, check it
        if urlparse.urlparse(config).scheme == "":
            # turn local path into URL
            config = "file://%s" % config
        try:
            urllib2.urlopen(config, timeout=5).close()
        except urllib2.URLError as e:
            msg = _("Cannot access 'scality_sofs_config': %s") % e
            LOG.warn(msg)
            raise exception.NovaException(msg)

        # mount.sofs must be installed
        if not os.access("/sbin/mount.sofs", os.X_OK):
            msg = _("Cannot execute /sbin/mount.sofs")
            LOG.warn(msg)
            raise exception.NovaException(msg)
Example #12
0
def download_all_logs(dirname=".", filename=None):
    """
  Download H2O Log Files to Disk
  :param dirname: (Optional) A character string indicating the directory that the log file should be saved in.
  :param filename: (Optional) A string indicating the name that the CSV file should be
  :return: path of logs written (as a string)
  """
    url = "http://" + H2OConnection.ip() + ":" + str(H2OConnection.port()) + "/Logs/download"
    response = urllib2.urlopen(url)

    if not os.path.exists(dirname):
        os.mkdir(dirname)
    if filename == None:
        for h in response.headers.headers:
            if "filename=" in h:
                filename = h.split("filename=")[1].strip()
                break
    path = os.path.join(dirname, filename)

    with open(path, "w") as f:
        response = urllib2.urlopen(url)
        f.write(response.read())
        f.close()

    print "Writing H2O logs to " + path
    return path
Example #13
0
    def load(self, filename):
        """
        _load_

        Unpickle data from file

        """

        # TODO: currently support both loading from file path or url
        # if there are more things to filter may be separate the load function

        # urllib2 needs a scheme - assume local file if none given
        if not urlparse(filename)[0]:
            filename = "file:" + filename
            handle = urlopen(Request(filename, headers={"Accept": "*/*"}))
            self.data = pickle.load(handle)
            handle.close()
        elif filename.startswith("file:"):
            handle = urlopen(Request(filename, headers={"Accept": "*/*"}))
            self.data = pickle.load(handle)
            handle.close()
        else:
            # use own request class so we get authentication if needed
            from WMCore.Services.Requests import Requests

            request = Requests(filename)
            data = request.makeRequest("", incoming_headers={"Accept": "*/*"})
            self.data = pickle.loads(data[0])

        # TODO: use different encoding scheme for different extension
        # extension = filename.split(".")[-1].lower()

        return
    def test_webhook_no_ssl(self):
        print("Testing Webhook without SSL")
        self._setup_updater("", messages=0)
        d = self.updater.dispatcher
        d.addTelegramMessageHandler(self.telegramHandlerTest)

        # Select random port for travis
        port = randrange(1024, 49152)
        self.updater.start_webhook("127.0.0.1", port)
        sleep(0.5)

        # Now, we send an update to the server via urlopen
        message = Message(1, User(1, "Tester 2"), datetime.now(), Chat(1, "group", title="Test Group 2"))

        message.text = "Webhook Test 2"
        update = Update(1)
        update.message = message

        try:
            payload = bytes(update.to_json(), encoding="utf-8")
        except TypeError:
            payload = bytes(update.to_json())

        header = {"content-type": "application/json", "content-length": str(len(payload))}

        r = Request("http://127.0.0.1:%d/" % port, data=payload, headers=header)

        urlopen(r)
        sleep(1)
        self.assertEqual(self.received_message, "Webhook Test 2")
Example #15
0
File: solr.py Project: rubic/salt
def _http_request(url, request_timeout=None):
    """
    PRIVATE METHOD
    Uses json.load to fetch the json results from the solr api.

    url : str
        a complete url that can be passed to urllib.open
    request_timeout : int (None)
        The number of seconds before the timeout should fail. Leave blank/None to
        use the default. __opts__['solr.request_timeout']

    Return: dict<str,obj>::

         {'success':boolean, 'data':dict, 'errors':list, 'warnings':list}
    """
    try:

        request_timeout = __opts__["solr.request_timeout"]
        if request_timeout is None:
            data = json.load(urllib2.urlopen(url))
        else:
            data = json.load(urllib2.urlopen(url, timeout=request_timeout))
        return _get_return_dict(True, data, [])
    except Exception as e:
        return _get_return_dict(False, {}, ["{0} : {1}".format(url, e)])
Example #16
0
def INDEX(url, name):
    name = "" + name
    req = urllib2.Request(url)
    req.add_header("User-Agent", user_agent)
    response = urllib2.urlopen(req)
    link = response.read()
    response.close()
    match = re.compile('class="widget widget_text"><h3>' + name + "</h3>(.+?)</div></div>", re.DOTALL).findall(link)
    for items in match:
        match1 = re.compile(
            '<span style="font-size: small;">(.+?)<a href="(.+?)">(.+?)</a>(.+?)</span>', re.DOTALL
        ).findall(items)
        for jnk1, channelurl, name, spc in match1:
            try:
                req = urllib2.Request(channelurl)
                response = urllib2.urlopen(req)
                link = response.read()
                response.close()

                match = re.compile('<div class="entry">(.+?)</div></div>', re.DOTALL).findall(link)

                for rawhtml in match:
                    playertype = identifyLink(rawhtml, name)

                streamLink = getStreamLink(playertype, rawhtml, name)
                addLink(name.title() + "-" + playertype, streamLink, "")
            except:
                pass
Example #17
0
def ping_newrelic(ctx):
    if settings.NEW_RELIC_API_KEY and settings.NEW_RELIC_APP_ID:
        with ctx.lcd(settings.SRC_DIR):
            log_cmd = "git log --oneline {0}..{1}".format(OLDREV, NEWREV)
            changelog = ctx.local(log_cmd).out.strip()

        print "Post deployment to New Relic"
        desc = generate_desc(OLDREV, NEWREV, changelog)
        if changelog:
            github_url = GITHUB_URL.format(oldrev=OLDREV, newrev=NEWREV)
            changelog = "{0}\n\n{1}".format(changelog, github_url)
        data = urllib.urlencode(
            {
                "deployment[description]": desc,
                "deployment[revision]": NEWREV,
                "deployment[app_id]": settings.NEW_RELIC_APP_ID,
                "deployment[changelog]": changelog,
            }
        )
        headers = {"x-api-key": settings.NEW_RELIC_API_KEY}
        try:
            request = urllib2.Request(NEW_RELIC_URL, data, headers)
            urllib2.urlopen(request)
        except urllib.URLError as exp:
            print "Error notifying New Relic: {0}".format(exp)
Example #18
0
def test_once_mode_three_times(tmpdir):
    testfile = str(tmpdir.join("recordmode.yml"))
    with vcr.use_cassette(testfile, record_mode="once"):
        # get three of the same file
        response1 = urllib2.urlopen("http://httpbin.org/").read()
        response2 = urllib2.urlopen("http://httpbin.org/").read()
        response2 = urllib2.urlopen("http://httpbin.org/").read()
    def load(self):
        # load docs from a list
        info("loading sample data from http://openlibrary.org/people/anand/lists/OL1815L")
        system(INTERP + " ./scripts/copydocs.py --list /people/anand/lists/OL1815L")

        # Hack to load borrowable ebooks to store so that they appear in the return cart
        urllib2.urlopen("http://0.0.0.0:8080/_dev/process_ebooks").read()
def enterprise_main(dest=None):
    """
    Build out the JSON file for each organization
    """
    output_dir = dest if dest else default_dest

    # Make sure the destination folder exists
    if not dest and not os.path.exists(default_dest):
        os.makedirs(default_dest)

    # Get the contents of the big data.json file
    response = None
    try:
        response = urllib2.urlopen(enterprise_datajson_source)
    except urllib2.URLError:
        # Fall back on default
        response = urllib2.urlopen("http://localhost:5000/enterprisedata.json")

    datasets = json.load(response)

    # group datasets by organization
    datasets_grouped_by_org = {}
    for dataset in datasets:
        org = dataset["organization"]
        if org in datasets_grouped_by_org:
            datasets_grouped_by_org[org].append(dataset)
        else:
            datasets_grouped_by_org[org] = [dataset]

    # write the organization.json files
    for org_name, org_datasets in datasets_grouped_by_org.iteritems():
        with open(os.path.join(output_dir, org_name + "_enterprise.json"), "w") as outfile:
            json.dump(org_datasets, outfile)
Example #21
0
def json_processing_callback(entry):
    """
    Send an HTTP post to the registered callback url, if any.
    """
    if not entry.processing_metadata:
        _log.debug("No processing callback for {0}".format(entry))
        return

    url = entry.processing_metadata[0].callback_url

    _log.debug("Sending processing callback for {0} ({1})".format(entry, url))

    headers = {"Content-Type": "application/json"}

    data = {"id": entry.id, "state": entry.state}

    # Trigger testing mode, no callback will be sent
    if url.endswith("secrettestmediagoblinparam"):
        TESTS_CALLBACKS.update({url: data})
        return True

    request = create_post_request(url, data, headers=headers, data_parser=json.dumps)

    try:
        urlopen(request)
        _log.debug("Processing callback for {0} sent".format(entry))

        return True
    except HTTPError:
        _log.error("Failed to send callback: {0}".format(traceback.format_exc()))

        return False
Example #22
0
def continuity(url):
    import md5

    format = "%25s: %s"

    # first fetch the file with the normal http handler
    opener = urllib2.build_opener()
    urllib2.install_opener(opener)
    fo = urllib2.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5.new(foo)
    print format % ("normal urllib", m.hexdigest())

    # now install the keepalive handler and try again
    opener = urllib2.build_opener(HTTPHandler())
    urllib2.install_opener(opener)

    fo = urllib2.urlopen(url)
    foo = fo.read()
    fo.close()
    m = md5.new(foo)
    print format % ("keepalive read", m.hexdigest())

    fo = urllib2.urlopen(url)
    foo = ""
    while 1:
        f = fo.readline()
        if f:
            foo = foo + f
        else:
            break
    fo.close()
    m = md5.new(foo)
    print format % ("keepalive readline", m.hexdigest())
Example #23
0
def UploadCodeTallyJson(server, json_file):
    """Reads in the json_file and sends it to the given server."""
    with open(json_file, "r") as f:
        json_data = f.read()

    data = urllib.urlencode({"data": json_data})
    req = urllib2.Request(server + UPLOAD_PATH, data)

    # If the upload encounters an exception that doesn't seem to be this scripts
    # fault, retry a few times.
    for attempt in range(5):
        try:
            urllib2.urlopen(req)
            print "Sucessfully uploaded data to server"
            return 0
        except urllib2.HTTPError as e:
            if e.code < 500:
                print "Encounter an exception, not retrying.\n%s" % str(e)
                return 1
            print "Encountered a server error, retrying\n%s" % str(e)
        except (httplib.HTTPException, urllib2.URLError) as e:
            print "Encountered an exception, retrying.\n%s" % str(e)

        # Add an exponential backoff.
        duration = random.random() * 3 + math.pow(1.5, (attempt + 1))
        duration = min(10, max(0.1, duration))
        time.sleep(duration)

    print "Unable to upload to the server after 5 attempts, aborting."
    return 1
Example #24
0
    def post_http_request(self, url, data=None, testcase=None, max_attempts=1, json=False):
        """
        INPUT
            url: the HTTP url to use.
            data: a dictionary of data values to encode into the url when posting.
            testcase: a testcase object supplied when executing function as part of a testcase step.
            max attempts: the maximum number of attempts to perform the operation.
            json: whether to post the request using JSON format or not.
        OUPUT
            successful: whether the function executed successfully or not.
            response: the response to the request.
        """

        self.log.debug("Posting HTTP request %s %s ..." % (url, str(data)))
        result = {"successful": False, "response": None}

        attempt = 1
        while not result["successful"] and attempt <= max_attempts:

            try:
                if data is not None:
                    if json:
                        # transform the data object into a form data encoded string (using JSON)
                        s_data = dumps(data) if data is not None else ""
                    else:
                        # transform the data object into a form data encoded string
                        s_data = urlencode(data)
                        s_data = s_data.replace("+", "%20").replace("%28", "(").replace("%29", ")")
                else:
                    s_data = ""

                # post the request
                if json:
                    request = Request(url, s_data, {"Content-Type": "application/json"})
                    result["response"] = urlopen(request).read().strip()
                    self.log.trace("Posted JSON request.")
                else:
                    result["response"] = urlopen(url, s_data).read().strip()
                    self.log.trace("Posted HTTP request.")

                self.log.trace("Response: %s" % result["response"])

                if result["response"] is not None and result["response"] != "":
                    result["successful"] = True
                    break
                else:
                    self.log.trace(
                        "Failed to post HTTP request %s %s (attempt %s). "
                        "No response received from server."
                        "Re-attempting in 5 seconds ..." % (url, str(s_data), attempt)
                    )
            except HTTPError, e:
                self.log.trace(str(e))
                self.log.trace(
                    "Failed to post HTTP request %s %s (attempt %s). "
                    "Re-attempting in 5 seconds ..." % (url, str(s_data), attempt)
                )
            except BaseException, e:
                self.handle_exception(e, operation="post HTTP request %s %s (attempt %s)" % (url, str(s_data), attempt))
                self.log.trace("Re-attempting in 5 seconds ...")
def pullPhotos(query):

    print "looking for", query
    url1 = "https://www.google.com/search?biw=1309&bih=704&sei=bsHjUbvaEILqrQeA-YCYDw&tbs=itp:lineart&tbm=isch&"
    query2 = urllib.urlencode({"q": query})

    req = urllib2.Request(url1 + query2, headers={"User-Agent": "Chrome"})
    response = urllib2.urlopen(req).read()
    parser = MyHTMLParser()
    parser.feed(response)

    print image_lib + "\\buffer\\" + query

    if not os.path.exists(image_lib + "\\buffer"):
        os.mkdir(image_lib + "\\buffer")  # make directory to put them in

    if not os.path.exists(image_lib + "\\buffer\\" + query):
        os.mkdir(image_lib + "\\buffer\\" + query)  # make directory to put them in

    for i in xrange(5):
        req_cat = urllib2.Request(cat_urls[i], headers={"User-Agent": "Chrome"})
        response_cat = urllib2.urlopen(req_cat).read()
        name = query + os.sep + query + str(i) + ".jpg"
        fd = open(image_lib + "\\buffer\\" + name, "wb")
        fd.write(response_cat)
        fd.close()
        print name, "written", "complexity is ", countComponents(image_lib + "\\buffer\\" + name)

    print "done"
Example #26
0
 def _canUpdate(self):
     addon_list_page = urllib2.urlopen(self.url[self.addon_type]["file_list"] % self.addon_id)
     # <a href="/addons/recount/files/1478-v4-2-0e-release/">v4.2.0e release</a>
     m = re.search(r'href="(/addons/' + self.addon_id + '/files/([a-zA-Z0-9-]+?)/)">(.+?)<', addon_list_page.read())
     if m:
         self.last_version = m.group(3)
         if self.last_version != self.current_version:
             download_page = urllib2.urlopen(self.url[self.addon_type]["base"] + m.group(1))
             # <a href="http://static.wowace.com/content/files/529/668/Recount-v4.2.0e_release.zip">
             # <a href="http://www.curseforge.com/media/files/527/306/TradeSkillMaster-v0.2.4Beta.zip">
             # <a href="http://www.curseforge.com/media/files/530/116/TradeSkillMaster_Crafting-r438.zip">
             m = re.search(
                 r'href="(' + self.url[self.addon_type]["static"] + '/media/files/(.+?).zip)"', download_page.read()
             )
             if m:
                 self.file = m.group(1)
                 return True
             else:
                 if self.logging:
                     self.logging.error(
                         "CurseAddon: Regex didn't match, probably download page has been changed. (%s)",
                         self.addon_name,
                     )
                 return False
         else:
             return False
     else:
         if self.logging:
             self.logging.error(
                 "CurseAddon: Regex didn't match, probably file list page has been changed. (%s)", self.addon_name
             )
         return False
Example #27
0
def Downloadcode(cookie, savefile="code.gif", url=URL):
    """
    download code image from url
    """
    import urllib2

    # url = url + '?ticks=' + str(int(time.time()*1000))
    # URL = url
    # print url
    global ASPXAUTH
    global ASPNET_SESSIONID
    # print ASPXAUTH,'\n', ASPNET_SESSIONID
    # print cookie
    cookie += "; %s=%s" % (".ASPXAUTH", ASPXAUTH)
    cookie += "; %s=%s" % ("ASP.NET_SessionId", ASPNET_SESSIONID)
    # print cookie
    req = urllib2.Request(url)
    req.add_header("Cookie", cookie)
    req.add_header(
        "User-Agent",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; Zune 4.7; MS-RTC LM 8; InfoPath.3; BRI/2)",
    )
    try:
        res_img = urllib2.urlopen(req)
        f = open(savefile, "wb")
        f.write(res_img.read())
        f.close()
    except:
        res_img = urllib2.urlopen(req)
        f = open(savefile, "wb")
        f.write(res_img.read())
        f.close()
Example #28
0
def _send_notify(msg, api_key=None, id=None):
    """
    Sends a Telegram notification
    :param msg: The message string to send
    :param id: The Telegram user/group id to send the message to
    :param api_key: Your Telegram bot API token

    :returns: True if the message succeeded, False otherwise
    """
    id = autosub.TELEGRAMID if id is None else id
    api_key = autosub.TELEGRAMAPI if api_key is None else api_key

    log.debug("telegram: Notification send to ID: %s" % id)

    message = "{0} : {1}".format("Auto-Sub", msg)
    payload = urllib.urlencode({"chat_id": id, "text": message})
    telegram_api = "https://api.telegram.org/bot%s/%s"

    req = urllib2.Request(telegram_api % (api_key, "sendMessage"), payload)

    try:
        urllib2.urlopen(req)
        log.debug("telegram : message sent successfully.")
    except Exception as error:
        log.error("telegram: %s" % error)
        return False
    return True
Example #29
0
def scrap_items():
    for itemlist in ITEMLIST:
        soup = BS(urllib2.urlopen("".join([LOLWIKI, itemlist])).read())
        item_table = soup.find("table", class_="stdt sortable")

        for tr in item_table.find_all("tr"):
            tds = tr.find_all("td")
            if len(tds) < 1:
                continue
            if tr.find("p") == None:
                continue

            item_name = tr.find("p").text.strip()
            item_url = tr.find("img")["src"]

            if item_url.split(":")[0] == "data":
                item_url = tr.find("img")["data-src"]

            if not HOOKED:
                continue

                # store item in database
            d_item = Item()
            d_item.name = item_name

            t_img = NamedTemporaryFile(delete=True)
            t_img.write(urllib2.urlopen(item_url).read())
            t_img.flush()
            t_img.name = ".".join([item_name, "jpg"])

            d_item.picture = File(t_img)
            d_item.save()
    def get_dbusage_info(self, rest_api_port):
        """Collects database usage information from all db nodes
        Returns:
        A dictionary with db node name as key and db usage in % as value
        """

        to_return = {}
        try:
            uve_url = (
                "http://127.0.0.1:" + str(rest_api_port) + "/analytics/uves/database-nodes?cfilt=DatabaseUsageInfo"
            )
            node_dburls = json.loads(urllib2.urlopen(uve_url).read())

            for node_dburl in node_dburls:
                # calculate disk usage percentage for analytics in each cassandra node
                db_uve_state = json.loads(urllib2.urlopen(node_dburl["href"]).read())
                db_usage_in_perc = (
                    100
                    * float(db_uve_state["DatabaseUsageInfo"]["database_usage"][0]["analytics_db_size_1k"])
                    / float(
                        db_uve_state["DatabaseUsageInfo"]["database_usage"][0]["disk_space_available_1k"]
                        + db_uve_state["DatabaseUsageInfo"]["database_usage"][0]["disk_space_used_1k"]
                    )
                )
                to_return[node_dburl["name"]] = db_usage_in_perc
        except Exception as inst:
            self._logger.error(type(inst))  # the exception instance
            self._logger.error(inst.args)  # arguments stored in .args
            self._logger.error(inst)  # __str__ allows args to be printed directly
            self._logger.error("Could not retrieve db usage information")

        self._logger.info("db usage:" + str(to_return))
        return to_return