Exemplo n.º 1
0
Arquivo: mixins.py Projeto: digicyc/q
class GoodReadsBookMixin(object):
    def __init__(self):
        self.http = HTTPConnection("www.goodreads.com")
        self.stats = None
        if self.id is not None and self._goodreads_id == 0:
            self._get_goodreads_stats()

    def _get_goodreads_stats(self):
        """
        {
            u'work_ratings_count': 4129,
            u'isbn': u'0767905385',
            u'text_reviews_count': 98,
            u'isbn13': u'9780767905381',
            u'reviews_count': 932,
            u'work_reviews_count': 6445,
            u'average_rating': u'3.83',
            u'work_text_reviews_count': 537,
            u'ratings_count': 529,
            u'id': 375789
        }
        """
        url = "/book/review_counts.json?isbns=%s&key=%s" % (self.isbn13, settings.GOODREADS_KEY)
        self.http.request('GET', url)
        json = self.http.getresponse().read()
        try:
            data = simplejson.loads(json)
        except ValueError, e:
            self._goodreads_id = -1
            self.save()
            return
        self.stats = data["books"][0]
Exemplo n.º 2
0
def YahooGeocoder(address, locality, region, postalcode, appId):
  yahooUrl = "/geocode?"
  queryParams = {
      'q' : address + ' ' + locality +' ' + region + ' ' + str(postalcode),
      'appid' : appId,
      'flags' : 'J'
  }
  requestUrl = yahooUrl + urlencode(queryParams)
  conn = HTTPConnection('where.yahooapis.com')
  geocodeRequest = conn.request('GET', requestUrl)
  response = conn.getresponse()
  if response.status != 200:
    raise LocationNotFoundError('Status code was not 200')

  responseString = response.read()
  resultSet = json.loads(responseString)['ResultSet']
  if resultSet['Error'] != 0:
    raise LocationNotFoundError('Results contained error')

  if resultSet['Found'] <= 0:
    raise LocationNotFoundError('Location not found')


  results = resultSet['Results']
  return (float(results[0]['latitude']), float(results[0]['longitude']))
Exemplo n.º 3
0
    def _httplib_request(self, method, url, fmt, params):
        '''Make a HTTP request via httplib.

        Make a HTTP request to `url` using `method`. `fmt` denotes a desired
        response format (sent as ``Accept:`` header), `params` are sent as
        the query string or the request body, depending on the method.
        Return a `httplib.HTTPResponse` instance.
        '''
        methods = ('DELETE', 'GET', 'POST', 'PUT')
        method = method.upper()
        if not method in methods:
            raise MyGengoException('Invalid or unsupported HTTP method %s' % method)
        query_string = urlencode(params)
        url_parts = urlparse(url)
        path = url_parts.path
        if method in ('POST', 'PUT'):
            body = query_string
        else:
            path += '?' + query_string
            body = None
        if self.config.debug:
            print url_parts.hostname + path
        headers = self._make_header(fmt)
        if method == 'POST':
            headers['Content-Type'] = 'application/x-www-form-urlencoded'
        try:
            conn = HTTPConnection(url_parts.hostname, url_parts.port)
            headers['User-Agent'] = 'pythonMyGengo 1.0'
            conn.request(method, path, body, headers=headers)
            return conn.getresponse()
        except Exception, e:
            raise MyGengoException(e.args)
Exemplo n.º 4
0
def report_sauce_status(job_id, test_status):
    """Report test status to Sauce service

    :param string job_id: [required] saucelabs job id
    :param string test_status: [required] status of test
    :returns: request status code
    :rtype: int or string

    Example::

        Report sauce status  ${SESSION_ID}  ${TEST_STATUS}

    """
    username = environ.get('SAUCE_USERNAME')
    access_key = environ.get('SAUCE_ACCESS_KEY')

    if not job_id:
        return u"No Sauce job id found. Skipping..."
    elif not username or not access_key:
        return u"No Sauce environment variables found. Skipping..."

    token = encodestring('%s:%s' % (username, access_key))[:-1]
    body = dumps({'passed': test_status == 'PASS'})

    connection = HTTPConnection('saucelabs.com')
    connection.request('PUT', '/rest/v1/%s/jobs/%s' % (
        username, job_id), body,
        headers={'Authorization': 'Basic %s' % token}
    )

    return connection.getresponse().status
Exemplo n.º 5
0
def run(query='insurance', query_template_file='query.txt', host='localhost',
        port='8953', interval='1.0', timeout='10'):
    with open(join(dirname(__file__), query_template_file)) as fh:
        url = string.Template(fh.read()).substitute(
            host=host,
            port=port,
            query=quote_plus(query))

    print 'Starting load'

    con = HTTPConnection(host, port, timeout=float(timeout))

    while True:
        try:
            con.request('GET', url)
            sys.stdout.write('.')
            sys.stdout.flush()
            response = con.getresponse()

            content = response.read()
            assert 'HTTP ERROR: 500' not in content, 'Server error %s' % content

            time.sleep(float(interval))
        except KeyboardInterrupt:
            break

    print '\nEnd'
Exemplo n.º 6
0
def send(sms_to, sms_body, **kwargs):
    """
    Site: http://smsaero.ru/
    API: http://smsaero.ru/api/
    """
    headers = {
        "User-Agent": "DBMail/%s" % get_version(),
    }

    kwargs.update({
        'user': settings.SMSAERO_LOGIN,
        'password': settings.SMSAERO_MD5_PASSWORD,
        'from': kwargs.pop('sms_from', settings.SMSAERO_FROM),
        'to': sms_to.replace('+', ''),
        'text': from_unicode(sms_body)
    })

    http = HTTPConnection(kwargs.pop("api_url", "gate.smsaero.ru"))
    http.request("GET", "/send/?" + urlencode(kwargs), headers=headers)
    response = http.getresponse()

    if response.status != 200:
        raise AeroSmsError(response.reason)

    body = response.read().strip()
    if '=accepted' not in body:
        raise AeroSmsError(body)

    return int(body.split('=')[0])
Exemplo n.º 7
0
 def __init__(self, *args, **kwargs):
     '''I overrode the init and copied a lot of the code from the parent
     class because HTTPConnection when this happens has been replaced by
     VCRHTTPConnection,  but doing it here lets us use the original one.'''
     HTTPConnection.__init__(self, *args, **kwargs)
     self.key_file = kwargs.pop('key_file', None)
     self.cert_file = kwargs.pop('cert_file', None)
Exemplo n.º 8
0
def get_embed(video_url):
	if video_url.startswith('http://www.youtube.com/watch?v='):
		yid = video_url.partition('=')[2]
		return 'http://www.youtube.com/embed/%(yid)s?wmode=opaque&fs=1&theme=dark&autoplay=0&' % {'yid':yid}
	conn = HTTPConnection("video.yandex.net")
	get_embed_url = "/embed.xml?params=%26embed%3Dy&width=612.8&height=344.7&link=" + quote(video_url)
	logger.info("Getting embed: " + get_embed_url)
	conn.request("GET",get_embed_url)
	res = conn.getresponse()
	result = res.getheader("Location")

	if result is not None:
		result = result.replace("?autoplay=1","?autoplay=0")
		result = result.replace("&autoplay=1","&autoplay=0")
		result = result.replace("?autoStart=true","?autoStart=false")
		result = result.replace("&autoStart=true","&autoStart=false")
		result = result.replace("?auto=1","?auto=0")
		result = result.replace("&auto=1","&auto=0")
		result = result.replace("?ap=1","?ap=0")
		result = result.replace("&ap=1","&ap=0")

		if 'video.yandex.ru' in result:
			result = result.replace("&autoplay=0","")
			result = result.replace("?autoplay=0&","?")
			result = result.replace("?autoplay=0","")

	return result
Exemplo n.º 9
0
def GetServerList():
    """
    @note 通过http接口获取主机信息字符串,并整理为字典列表结构
    """
    try:
        httpClient = HTTPConnection('server.yw.rrgdev.com', 80)
        httpClient.request('GET', '/seasia.py')
        response = httpClient.getresponse()
        if (404 == response.status):
            return None

        s = response.read()
        s = s.lstrip("{")
        s = s.rstrip("\n")
        s = s.rstrip("}")
        s = s.strip("\n")

        lstStr = s.split("\n")
        lstServer = []
        for strTmp in lstStr:
            if (0 == len(strTmp)):
                logger.error("invalid server information string")
                continue;

            strTmp = strTmp.rstrip(",")
            dictTmp = eval(strTmp)
            if (len(dictTmp) < 14):
                logger.error("invalid para num in server information string")
                continue
            lstServer.append(dictTmp)
    except Exception, e:
        logger.error(e)
        return None
Exemplo n.º 10
0
    def __init__(self,hostName, portNumber):

        HTTPConnection.__init__(self, hostName, portNumber)
        
        self._hostName   = hostName
        self._portNumber = portNumber
        self._commandList = []
Exemplo n.º 11
0
    def getSearchXML(self, query):
        #this function will return the search XML for this server.
        passthruURL = 'http://atv.plexconnect/passthru?URL=http://' + self.address + ':' + self.port
        conn = HTTPConnection(self.address + ":" + self.port)
        path = '/search?type=4&query=' + query
        headers = {}

        if self.token!=None:
            headers = {'X-Plex-Token' : self.token}

        try:
            conn.request('GET', path, None, headers)
            responseXML = str(conn.getresponse().read())
        except:
            dprint(__name__, 0, 'No Response from myPlex Server')
            return False

        #get the XML
        root = ET.fromstring(responseXML)

        #and parse it
        XML = ET.ElementTree(root)
        for vid in XML.findall('Video'):
            vid.set('key', passthruURL + vid.get('key'))
            #Will need to fix art/thumb here...

        for part in XML.iter('Part'):
            part.set('key', passthruURL + part.get('key'))

        return ET.tostring(root)
Exemplo n.º 12
0
    def discoverSections(self, force=False):
        if self.lastSection!=None and (datetime.now() - self.lastSection).total_seconds()<3600 and force==False:
            dprint(__name__, 0, "Nothing doing. Cache time less than 1h threshold")
            return
        #we're here, so let's get the list of sections.
        self.sections = []
        #start by grabbing a reference to this server.
        dprint(__name__, 2, "Add: {0}, port: {1}", self.address, self.port)
        conn = HTTPConnection(self.address + ":" + self.port)
        path = '/library/sections'
        headers = {}

        if self.token!=None:
            headers = {'X-Plex-Token' : self.token}

        try:
            conn.request('GET', path, None, headers)
            responseXML = str(conn.getresponse().read())
        except:
            dprint(__name__, 0, 'No Response from myPlex Server')
            return False

        #get the XML
        root = ET.fromstring(responseXML)

        #and parse it
        XML = ET.ElementTree(root)

        for dir in XML.findall("Directory"):
            #and add the sections
            self.sections.append(CSection(dir.get('title'), dir.get('type'), dir.get('key')))
            dprint(__name__, 2, "{0} : Found section: {1}", self.name, dir.get('title'))
Exemplo n.º 13
0
    def _send_request(self, method, path, body='', headers={}):
        if TIMEOUTS_AVAILABLE:
            url = self.url.replace(self.path, '')            
            http = Http(timeout=self.timeout)
            headers, response = http.request(url + path, method=method, body=body, headers=headers)
            
            if int(headers['status']) != 200:
                raise SolrError(self._extract_error(headers, response))
            
            return response
        else:
            if headers is None:
                headers = {}
            
            conn = HTTPConnection(self.host, self.port)            
            conn.request(method, path, body, headers)
            

            
            response = conn.getresponse()
            
            if response.status != 200:
                raise SolrError(self._extract_error(dict(response.getheaders()), response.read()))
            
            return response.read()
Exemplo n.º 14
0
def proxy(request):
    if 'url' not in request.GET:
        return HttpResponse(
                "The proxy service requires a URL-encoded URL as a parameter.",
                status=400,
                content_type="text/plain"
                )

    url = urlsplit(request.GET['url'])
    locator = url.path
    if url.query != "":
        locator += '?' + url.query
    if url.fragment != "":
        locator += '#' + url.fragment

    headers = {}
    if settings.SESSION_COOKIE_NAME in request.COOKIES:
        headers["Cookie"] = request.META["HTTP_COOKIE"]

    if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META:
        headers["Content-Type"] = request.META["CONTENT_TYPE"]

    conn = HTTPConnection(url.hostname, url.port)
    conn.request(request.method, locator, request.raw_post_data, headers)
    result = conn.getresponse()
    response = HttpResponse(
            result.read(),
            status=result.status,
            content_type=result.getheader("Content-Type", "text/plain")
            )
    return response
Exemplo n.º 15
0
    def _run_server(self, app):
        """Run a wsgi server in a separate thread"""
        ip, port = get_free_port()
        self.app = app = WSGIApplication(app, (ip, port))

        def run():
            logger = logging.getLogger("SeleniumWebDriverApp")

            def log_message(self, format, *args):
                logger.info("%s - - [%s] %s\n" %
                            (self.address_string(),
                             self.log_date_time_string(),
                             format % args))

            # monkey patch to redirect request handler logs
            WSGIRequestHandler.log_message = log_message

            httpd = simple_server.make_server(ip, port, app,
                                              server_class=WSGIServer,
                                              handler_class=WSGIRequestHandler)

            httpd.serve_forever()

        app.thread = Process(target=run)
        app.thread.start()
        conn = HTTPConnection(ip, port)
        time.sleep(.5)
        for i in range(100):
            try:
                conn.request('GET', '/__application__')
                conn.getresponse()
            except (socket.error, CannotSendRequest):
                time.sleep(.3)
            else:
                break
Exemplo n.º 16
0
def send(sms_to, sms_body, **kwargs):
    """
    Site: http://iqsms.ru/
    API: http://iqsms.ru/api/
    """
    headers = {
        "User-Agent": "DBMail/%s" % get_version(),
        'Authorization': 'Basic %s' % b64encode(
            "%s:%s" % (
                settings.IQSMS_API_LOGIN, settings.IQSMS_API_PASSWORD
            )).decode("ascii")
    }

    kwargs.update({
        'phone': sms_to,
        'text': from_unicode(sms_body),
        'sender': kwargs.pop('sms_from', settings.IQSMS_FROM)
    })

    http = HTTPConnection(kwargs.pop("api_url", "gate.iqsms.ru"))
    http.request("GET", "/send/?" + urlencode(kwargs), headers=headers)
    response = http.getresponse()

    if response.status != 200:
        raise IQSMSError(response.reason)

    body = response.read().strip()
    if '=accepted' not in body:
        raise IQSMSError(body)

    return int(body.split('=')[0])
Exemplo n.º 17
0
    def connect(self):
        """
        Establishes a connection to the REST API server as defined by the host,
        port and ssl instance variables
        """
        if self._token:
            self._debug("Forcing logout from old session.\n")

            orig_value = self.poll_incomplete
            self.poll_incomplete = False
            self.execute('/REST/Session', 'DELETE')
            self.poll_incomplete = orig_value

            self._token = None


        self._conn = None

        if self.ssl:
            msg = "Establishing SSL connection to %s:%s\n" % (
                self.host, self.port
            )
            self._debug(msg)
            self._conn = HTTPSConnection(self.host, self.port)

        else:
            msg = "Establishing unencrypted connection to %s:%s\n" % (
                self.host, self.port
            )
            self._debug(msg)
            self._conn = HTTPConnection(self.host, self.port)
Exemplo n.º 18
0
def check_server(port, port2server, pids, timeout=CHECK_SERVER_TIMEOUT):
    server = port2server[port]
    if server[:-1] in ('account', 'container', 'object'):
        if int(server[-1]) > 4:
            return None
        path = '/connect/1/2'
        if server[:-1] == 'container':
            path += '/3'
        elif server[:-1] == 'object':
            path += '/3/4'
        try_until = time() + timeout
        while True:
            try:
                conn = HTTPConnection('127.0.0.1', port)
                conn.request('GET', path)
                resp = conn.getresponse()
                # 404 because it's a nonsense path (and mount_check is false)
                # 507 in case the test target is a VM using mount_check
                if resp.status not in (404, 507):
                    raise Exception(
                        'Unexpected status %s' % resp.status)
                break
            except Exception, err:
                if time() > try_until:
                    print err
                    print 'Giving up on %s:%s after %s seconds.' % (
                        server, port, timeout)
                    raise err
                sleep(0.1)
Exemplo n.º 19
0
 def connect(self):
     """
     Override the connect() function to intercept calls to certain
     host/ports.
     
     If no app at host/port has been registered for interception then 
     a normal HTTPConnection is made.
     """
     if debuglevel:
         sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,))
                          
     try:
         (app, script_name) = self.get_app(self.host, self.port)
         if app:
             if debuglevel:
                 sys.stderr.write('INTERCEPTING call to %s:%s\n' % \
                                  (self.host, self.port,))
             self.sock = wsgi_fake_socket(app, self.host, self.port,
                                          script_name)
         else:
             HTTPConnection.connect(self)
             
     except Exception, e:
         if debuglevel:              # intercept & print out tracebacks
             traceback.print_exc()
         raise
Exemplo n.º 20
0
def test_ranges(webapp):
    connection = HTTPConnection(webapp.server.host, webapp.server.port)

    connection.request("GET", "%s/static/largefile.txt" %
                       webapp.server.http.base, headers={"Range": "bytes=0-50,51-100"})
    response = connection.getresponse()
    assert response.status == 206
Exemplo n.º 21
0
 def testGet(self):
     self.assertTrue(self.pasteThread.isAlive())
     from httplib import HTTPConnection
     http_connection = HTTPConnection("localhost", port=self.port)
     http_connection.request("GET", "/api/Crawl")
     response = http_connection.getresponse()
     self.assertTrue(response.status == 404 or response.status == 200 or response.status == 500)
Exemplo n.º 22
0
 def _select(self, params):
     # encode the query as utf-8 so urlencode can handle it
     params['q'] = unicode_safe(params['q'])
     path = '%s/select/?%s' % (self.path, urlencode(params))
     conn = HTTPConnection(self.host, self.port)
     conn.request('GET', path)
     return conn.getresponse()
Exemplo n.º 23
0
def _make_request(method, host, path, body='', headers={}):
  """
  Makes an HTTP request.

  Args:
    method |str| = Type of request. GET, POST, PUT or DELETE.
    host |str| = The host and port of the server. E.g. vmpooler.myhost.com:8080
    path |str| = The path of the url. E.g. /vm/vm_name
    body |str| = The body data to send with the request.
    headers |{str:str}| = Optional headers for the request.

  Returns:
    |HTTPResponse| = Response from the request.

  Raises:
    |RuntimeError| = If the vmpooler URL can't be reached
  """

  try:
    conn = HTTPConnection(host)
    conn.request(method, path, body, headers)
    resp = conn.getresponse()

    return resp
  except gaierror:
    error = "Couldn't connect to address '{}'. Ensure this is the correct URL for " \
            "the vmpooler".format(host)
    raise RuntimeError(error)
  except:
    print("Unkown error occured while trying to connect to {}".format(host))
    raise
Exemplo n.º 24
0
class RESTClient(object):
    """
    HTTP client
    HTTPS client based on the provided URL (http:// or https://)
    
    """

    def __init__(self, url, cert=None, key=None):
        logging.info("RESTClient URL: %s" % url)
        if url.startswith("https://"):
            logging.info("Using HTTPS protocol, getting user identity files ...")
            proxy_file = "/tmp/x509up_u%s" % os.getuid()
            if not os.path.exists(proxy_file):
                proxy_file = "UNDEFINED"
            cert_file = cert or os.getenv("X509_USER_CERT", os.getenv("X509_USER_PROXY", proxy_file))
            key_file = key or os.getenv("X509_USER_KEY", os.getenv("X509_USER_PROXY", proxy_file))
            logging.info("Identity files:\n\tcert file: '%s'\n\tkey file:  '%s' " % (cert_file, key_file))
            url = url.replace("https://", "")
            logging.info("Creating connection HTTPS ...")
            self.conn = HTTPSConnection(url, key_file=key_file, cert_file=cert_file)
        if url.startswith("http://"):
            logging.info("Using HTTP protocol, creating HTTP connection ...")
            url = url.replace("http://", "")
            self.conn = HTTPConnection(url)

    def http_request(self, verb, uri, data=None, header=None):
        logging.debug("Request: %s %s %s ..." % (verb, uri, data))
        self.conn.request(verb, uri, body=data, headers=self.headers)
        resp = self.conn.getresponse()
        data = resp.read()
        logging.debug("Status: %s" % resp.status)
        logging.debug("Reason: %s" % resp.reason)
        return resp.status, data
Exemplo n.º 25
0
def proxy(request):
    if 'url' not in request.GET:
        return HttpResponse(
                "The proxy service requires a URL-encoded URL as a parameter.",
                status=400,
                content_type="text/plain"
                )

    url = urlsplit(request.GET['url'])

    # Don't allow localhost connections unless in DEBUG mode
    if not settings.DEBUG and re.search('localhost|127.0.0.1', url.hostname):
        return HttpResponse(status=403)

    locator = url.path
    if url.query != "":
        locator += '?' + url.query
    if url.fragment != "":
        locator += '#' + url.fragment

    # Strip all headers and cookie info
    headers = {}


    conn = HTTPConnection(url.hostname, url.port) if url.scheme == "http" else HTTPSConnection(url.hostname, url.port)
    conn.request(request.method, locator, request.raw_post_data, headers)
    result = conn.getresponse()

    response = HttpResponse(
            valid_response(result.read()),
            status=result.status,
            content_type=result.getheader("Content-Type", "text/plain")
            )
    return response
Exemplo n.º 26
0
    def request(self, method, endpoint, qargs={}, data=None):
        path = self.format_path(endpoint, qargs)
        conn = HTTPConnection('%s:80' % self.account)
        auth = b64encode("%s:%s" % (self.api_key, self.password))
        headers = {
            'Authorization': 'Basic %s' % auth,
            'Content-Type': 'application/xml'
        }

        done = False
        while not done:
            try:
                conn.request(method, path, headers=headers, body=data)
            except socket.gaierror:
                if self.retry_on_socket_error:
                    time.sleep(self.retry_timeout)
                    continue
                else:
                    raise

            resp = conn.getresponse()
            body = resp.read()

            if resp.status == 503 and self.retry_on_503:
                time.sleep(self.retry_timeout)
            else:
                done = True

        if 200 <= resp.status < 300:
            return body
        else:
            raise ApiError("%s request to %s returned: %s\n%s" % 
                           (method, path, resp.status, body), resp.status)
Exemplo n.º 27
0
class HTTPSpeakerClient:
    """Emacspeak HTTP speech client, for HTTPSpeaker instances."""
    def __init__(self, host="127.0.0.1", port=8000):
        "Initialize client to connect to server at given host and port."
        self._connection=HTTPConnection(host, port)

    def postCommand(self, command, arg=""):
        """Post command, with argument arg (default, empty), to the speech server.
        Returns the body of the server's HTTP response, if any. On error, HTTPSpeakerError is raised."""
        body = command
        if arg:
            body += ": " + arg
        self._connection.request("POST", "/", body, {"Content-type": "text/plain"})
        response=self._connection.getresponse()
        if response.status != 200:
            raise HTTPSpeakerError(response.status, response.reason)
        return response.read()

    def speak(self, text):
        "Speak the supplied string."
        self.postCommand("speak", text)

    def stop(self):
        "Stop speaking."
        self.postCommand("stop")

    def isSpeaking(self):
        "Return '0' when not speaking."
        return self.postCommand("isSpeaking")

    def close(self):
        "Close the connection to the speech server."
        self._connection.close()
Exemplo n.º 28
0
def commit_koalemos(r):
    conn = HTTPConnection(OPTIONS['host'], OPTIONS['port'])
    conn.request('POST', '/commit',
                 body=urlencode({'r': r}),
                 headers={'Content-Type': 'application/x-www-form-urlencoded'})
    res = conn.getresponse()
    return (None, False) if res.status == 204 else (res.read(), True)
Exemplo n.º 29
0
Arquivo: xlb.py Projeto: win0x86/XLB
class HTTPHelper:
    def __init__(self, host, url, headers, method = "GET", body = None):
        self._url = url
        self._host = host
        self._headers = headers
        self._method = method
        self._body = body


    def _connect(self):
        self.conn = HTTPConnection(self._host)


    def _request(self):
        self.conn.request(self._method, self._url, body = self._body, headers = self._headers)


    def _response(self):
        self._response = self.conn.getresponse()
        return self._response.read()


    def run(self):
        self._connect()
        self._request()
        r = self._response()

        _encode = self._response.getheader("Content-Encoding")
        if _encode and "gzip" in _encode:
            r = gzip_decompress(r)
            
        return r
Exemplo n.º 30
0
 def __init__(self, host, port=None, key_file=None, cert_file=None,
              strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
              source_address=None):
     HTTPConnection.__init__(self, host, port, strict, timeout,
                             source_address)
     self.key_file = key_file
     self.cert_file = cert_file
Exemplo n.º 31
0
def proxy(request):
    PROXY_ALLOWED_HOSTS = getattr(settings, 'PROXY_ALLOWED_HOSTS', ())

    host = None

    if 'geonode.geoserver' in settings.INSTALLED_APPS:
        from geonode.geoserver.helpers import ogc_server_settings
        hostname = (ogc_server_settings.hostname,) if ogc_server_settings else ()
        PROXY_ALLOWED_HOSTS += hostname
        host = ogc_server_settings.netloc

    if 'url' not in request.GET:
        return HttpResponse("The proxy service requires a URL-encoded URL as a parameter.",
                            status=400,
                            content_type="text/plain"
                            )

    raw_url = request.GET['url']
    url = urlsplit(raw_url)
    locator = str(url.path)
    if url.query != "":
        locator += '?' + url.query
    if url.fragment != "":
        locator += '#' + url.fragment

    if not settings.DEBUG:
        if not validate_host(url.hostname, PROXY_ALLOWED_HOSTS):
            return HttpResponse("DEBUG is set to False but the host of the path provided to the proxy service"
                                " is not in the PROXY_ALLOWED_HOSTS setting.",
                                status=403,
                                content_type="text/plain"
                                )
    headers = {}

    if settings.SESSION_COOKIE_NAME in request.COOKIES and is_safe_url(url=raw_url, host=host):
        headers["Cookie"] = request.META["HTTP_COOKIE"]

    if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META:
        headers["Content-Type"] = request.META["CONTENT_TYPE"]

    if url.scheme == 'https':
        conn = HTTPSConnection(url.hostname, url.port)
    else:
        conn = HTTPConnection(url.hostname, url.port)
    conn.request(request.method, locator, request.body, headers)

    result = conn.getresponse()

    # If we get a redirect, let's add a useful message.
    if result.status in (301, 302, 303, 307):
        response = HttpResponse(('This proxy does not support redirects. The server in "%s" '
                                 'asked for a redirect to "%s"' % (url, result.getheader('Location'))),
                                status=result.status,
                                content_type=result.getheader("Content-Type", "text/plain")
                                )

        response['Location'] = result.getheader('Location')
    else:
        response = HttpResponse(
            result.read(),
            status=result.status,
            content_type=result.getheader("Content-Type", "text/plain"))

    return response
Exemplo n.º 32
0
    def fproxyGet(self, path):
        """
        Fetches from fproxy, returns (status, mimetype, data)
        """
        server = self.server
        headers = self.headers

        print "--------------------------------------------"
        print "** path=%s" % path

        # first scenario - user is pointing their browser directly at
        # fproxyfproxy, barf!
        if not path.startswith("http://"):
            self.send_response(400)
            self.send_header("Content-type", "text/html")
            data = "\n".join([
                "<html><head>",
                "<title>Access Denied</title>",
                "</head><body>",
                "<h1>Access Denied</h1>",
                "Sorry, but FProxyProxy is an http proxy server.<br>",
                "Please don't try to access it like a web server.",
                "</body></html>",
                "",
            ])
            self.send_header("Content-Length", str(len(data)))
            self.send_header("Location", location)
            self.end_headers()
            self.wfile.write(data)
            self.wfile.flush()
            return

        # convert path to relative
        path = "/" + path[7:].split("/", 1)[-1]
        #print "** path=%s" % repr(path)

        try:
            # check host header
            hostname = headers.get("Host", 'fproxy')
            pathbits = path.split("/")

            print "** hostname = %s" % hostname

            # second scenario, user has just given a domain name without trailing /
            if len(pathbits) == 1:
                # redirect to force trailing slash
                location = path + "/"
                print "** redirecting to: %s" % location

                self.send_response(301)
                self.send_header("Content-type", "text/html")
                data = "\n".join([
                    "<html><head>",
                    "<title>Permanent redirect: new URI</title>",
                    "</head><body>",
                    "<h1>Permanent redirect: new URI</h1>",
                    "<a href=\"%s\">Click here</a>",
                    "</body></html>",
                    "",
                ]) % location
                self.send_header("Content-Length", str(len(data)))
                self.send_header("Location", location)
                self.end_headers()
                self.wfile.write(data)
                self.wfile.flush()
                return

            tail = "/".join(pathbits[1:])

            # third scenario - request into fproxy
            if hostname == 'fproxy':

                # tis an fproxy request, go straight through
                conn = HTTPConnection(server.fproxyHost, server.fproxyPort)
                conn.request("GET", path)
                resp = conn.getresponse()
                self.send_response(resp.status)
                self.send_header("Content-type",
                                 resp.getheader("Content-Type", "text/plain"))
                data = resp.read()
                self.send_header("Content-Length", str(len(data)))
                self.end_headers()
                self.wfile.write(data)
                self.wfile.flush()
                conn.close()
                return

            else:
                # final scenario - some other domain, try lookup
                uri = server.node.namesiteLookup(hostname)

                if not uri:
                    # lookup failed, do the usual 404 thang
                    print "** lookup of domain %s failed" % hostname
                    self.send_response(404)
                    self.send_header("Content-type", "text/html")
                    data = "\n".join([
                        "<html><head>",
                        "<title>404 - Freenet name not found</title>",
                        "</head><body",
                        "<h1>404 - Freenet name not found</title>",
                        "The pyFreenet name service was unable to resolve ",
                        "the name %s" % hostname,
                        "<br><br>",
                        "You might like to find its freenet uri and try that ",
                        "within <a href=\"/fproxy/\">FProxy</a>",
                        "</body></html>",
                        "",
                    ])
                    self.send_header("Content-Length", str(len(data)))
                    self.end_headers()
                    self.wfile.write(data)
                    self.wfile.flush()

                # lookup succeeded - ok to go now via fproxy
                conn = HTTPConnection(server.fproxyHost, server.fproxyPort)
                newpath = "/" + uri
                if tail:
                    if not newpath.endswith("/"):
                        newpath += "/"
                    newpath += tail
                print "** newpath=%s" % newpath
                conn.request("GET", newpath)
                resp = conn.getresponse()
                print "** status=%s" % resp.status
                self.send_response(resp.status)
                self.send_header("Content-type",
                                 resp.getheader("Content-Type", "text/plain"))

                # has fproxy sent us a redirect?
                if resp.status == 301:
                    # yuck, fproxy is telling us to redirect, which
                    # sadly means we have to lose the domain name
                    # from our browser address bar
                    location = resp.getheader("location")
                    newLocation = "http://fproxy" + location
                    print "*** redirected!!!"
                    print "*** old location = %s" % location
                    print "***  --> %s" % newLocation
                    self.send_header("Location", newLocation)

                # get the data from fproxy and send it up to the client
                data = resp.read()
                self.send_header("Content-Length", str(len(data)))
                self.end_headers()
                self.wfile.write(data)
                self.wfile.flush()
                conn.close()
                return

            return

        except socket.error:
            raise
Exemplo n.º 33
0
 def __init__(self, host, *fakedata):
     HTTPConnection.__init__(self, host)
     self.fakedata = list(fakedata)
Exemplo n.º 34
0
 def __init__(self, host, port=None, timeout=20):
     HTTPConnection.__init__(self, host, port)
     self.timeout = timeout
Exemplo n.º 35
0
 def request(self, method, url, body=None, headers={}):
     self.request_length = 0
     HTTPConnection.request(self, method, url, body, headers)
Exemplo n.º 36
0
class SingleDownload(SingleDownloadHelperInterface):

    def __init__(self, downloader, url):
        SingleDownloadHelperInterface.__init__(self)
        self.downloader = downloader
        self.baseurl = url
        try:
            self.scheme, self.netloc, path, pars, query, fragment = urlparse(url)
        except:
            self.downloader.errorfunc('cannot parse http seed address: ' + url)
            return

        if self.scheme != 'http':
            self.downloader.errorfunc('http seed url not http: ' + url)
            return
        self.proxyhost = find_proxy(url)
        try:
            if self.proxyhost is None:
                self.connection = HTTPConnection(self.netloc)
            else:
                self.connection = HTTPConnection(self.proxyhost)
        except:
            self.downloader.errorfunc('cannot connect to http seed: ' + url)
            return

        self.seedurl = path
        if pars:
            self.seedurl += ';' + pars
        self.seedurl += '?'
        if query:
            self.seedurl += query + '&'
        self.seedurl += 'info_hash=' + urllib.quote(self.downloader.infohash)
        self.measure = Measure(downloader.max_rate_period)
        self.index = None
        self.url = ''
        self.requests = []
        self.request_size = 0
        self.endflag = False
        self.error = None
        self.retry_period = 30
        self._retry_period = None
        self.errorcount = 0
        self.goodseed = False
        self.active = False
        self.cancelled = False
        self.resched(randint(2, 10))

    def resched(self, len = None):
        if len is None:
            len = self.retry_period
        if self.errorcount > 3:
            len = len * (self.errorcount - 2)
        self.downloader.rawserver.add_task(self.download, len)

    def _want(self, index):
        if self.endflag:
            return self.downloader.storage.do_I_have_requests(index)
        else:
            return self.downloader.storage.is_unstarted(index)

    def download(self):
        if DEBUG:
            print 'http-sdownload: download()'
        if self.is_frozen_by_helper():
            if DEBUG:
                print 'http-sdownload: blocked, rescheduling'
            self.resched(1)
            return
        self.cancelled = False
        if self.downloader.picker.am_I_complete():
            self.downloader.downloads.remove(self)
            return
        self.index = self.downloader.picker.next(haveall, self._want, self)
        if self.index is None and self.frozen_by_helper:
            self.resched(0.01)
            return
        if self.index is None and not self.endflag and not self.downloader.peerdownloader.has_downloaders():
            self.endflag = True
            self.index = self.downloader.picker.next(haveall, self._want, self)
        if self.index is None:
            self.endflag = True
            self.resched()
        else:
            self.url = self.seedurl + '&piece=' + str(self.index)
            self._get_requests()
            if self.request_size < self.downloader.storage._piecelen(self.index):
                self.url += '&ranges=' + self._request_ranges()
            rq = Thread(target=self._request)
            rq.setName('HoffmanHTTPDownloader' + rq.getName())
            rq.setDaemon(True)
            rq.start()
            self.active = True

    def _request(self):
        import encodings.ascii
        import encodings.punycode
        import encodings.idna
        self.error = None
        self.received_data = None
        try:
            self.connection.request('GET', self.url, None, {'User-Agent': VERSION})
            r = self.connection.getresponse()
            self.connection_status = r.status
            self.received_data = r.read()
        except Exception as e:
            log_exc()
            self.error = 'error accessing http seed: ' + str(e)
            try:
                self.connection.close()
            except:
                pass

            try:
                self.connection = HTTPConnection(self.netloc)
            except:
                self.connection = None

        self.downloader.rawserver.add_task(self.request_finished)

    def request_finished(self):
        self.active = False
        if self.error is not None:
            if self.goodseed:
                self.downloader.errorfunc(self.error)
            self.errorcount += 1
        if self.received_data:
            self.errorcount = 0
            if not self._got_data():
                self.received_data = None
        if not self.received_data:
            self._release_requests()
            self.downloader.peerdownloader.piece_flunked(self.index)
        if self._retry_period:
            self.resched(self._retry_period)
            self._retry_period = None
            return
        self.resched()

    def _got_data(self):
        if self.connection_status == 503:
            try:
                self.retry_period = max(int(self.received_data), 5)
            except:
                pass

            return False
        if self.connection_status != 200:
            self.errorcount += 1
            return False
        self._retry_period = 1
        if len(self.received_data) != self.request_size:
            if self.goodseed:
                self.downloader.errorfunc('corrupt data from http seed - redownloading')
            return False
        self.measure.update_rate(len(self.received_data))
        self.downloader.measurefunc(len(self.received_data))
        if self.cancelled:
            return False
        if not self._fulfill_requests():
            return False
        if not self.goodseed:
            self.goodseed = True
            self.downloader.seedsfound += 1
        if self.downloader.storage.do_I_have(self.index):
            self.downloader.picker.complete(self.index)
            self.downloader.peerdownloader.check_complete(self.index)
            self.downloader.gotpiecefunc(self.index)
        return True

    def _get_requests(self):
        self.requests = []
        self.request_size = 0L
        while self.downloader.storage.do_I_have_requests(self.index):
            r = self.downloader.storage.new_request(self.index)
            self.requests.append(r)
            self.request_size += r[1]

        self.requests.sort()

    def _fulfill_requests(self):
        start = 0L
        success = True
        while self.requests:
            begin, length = self.requests.pop(0)
            if not self.downloader.storage.piece_came_in(self.index, begin, [], self.received_data[start:start + length], length):
                success = False
                break
            start += length

        return success

    def _release_requests(self):
        for begin, length in self.requests:
            self.downloader.storage.request_lost(self.index, begin, length)

        self.requests = []

    def _request_ranges(self):
        s = ''
        begin, length = self.requests[0]
        for begin1, length1 in self.requests[1:]:
            if begin + length == begin1:
                length += length1
                continue
            else:
                if s:
                    s += ','
                s += str(begin) + '-' + str(begin + length - 1)
                begin, length = begin1, length1

        if s:
            s += ','
        s += str(begin) + '-' + str(begin + length - 1)
        return s

    def helper_forces_unchoke(self):
        pass

    def helper_set_freezing(self, val):
        self.frozen_by_helper = val
Exemplo n.º 37
0
class TestRegression(TestCase):

    def setUp(self):
        self.ip = "localhost"
        self.port = "8081"
        self.headers = {'Content-Type': 'application/json',
                        'Accept': 'application/json'}
        self.timeout = 10
        # connect
        self.conn = HTTPConnection(self.ip, self.port, timeout=self.timeout)

    def tearDown(self):
        self.conn.close()

    def test_1_basic_request(self):
        """
        """

        # request an pa computation
        path = "/5gt/so/v1/PAComp"
        data = {"ReqId": "string",
                "nfvi": {
                    "NFVIPoPs": [
                        {
                            "Id": "DC1",
                            "gw_ip_address": "172.16.102.116",
                            "capabilities": {
                                "cpu": 3,
                                "ram": 2400,
                                "storage": 135000
                            },
                            "availableCapabilities": {
                                "cpu": 78,
                                "ram": 2400,
                                "storage": 135000
                            },
                            "internal_latency": 1.04
                        },
                        {
                            "Id": "DC2",
                            "gw_ip_address": "172.16.102.117",
                            "capabilities": {
                                "cpu": 3,
                                "ram": 160,
                                "storage": 7000
                            },
                            "availableCapabilities": {
                                "cpu": 85,
                                "ram": 160,
                                "storage": 7000
                            },
                            "internal_latency": 4.16
                        },
                        {
                            "Id": "DC3",
                            "gw_ip_address": "172.16.102.118",
                            "capabilities": {
                                "cpu": 2,
                                "ram": 160,
                                "storage": 7000
                            },
                            "availableCapabilities": {
                                "cpu": 40,
                                "ram": 160,
                                "storage": 7000
                            },
                            "internal_latency": 4.16
                        }
                    ],
                    "LLs": [
                        
                        {
                            "LLid": "LL1",
                            "capacity": {
                                "total": 125000000000,
                                "available": 1250000000
                            },
                            "delay": 3.3,
                            "length": 250,
                            "source": {
                                "Id": "DC1",
                                "GwIpAddress": "172.16.102.116"
                            },
                            "destination": {
                                "Id": "DC2",
                                "GwIpAddress": "172.16.102.117"
                            }
                        },
                        {
                            "LLid": "LL2",
                            "capacity": {
                                "total": 1250000000000,
                                "available": 1250000000
                            },
                            "delay": 1.3,
                            "length": 150,
                            "source": {
                                "Id": "DC1",
                                "GwIpAddress": "172.16.102.116"
                            },
                            "destination": {
                                "Id": "DC3",
                                "GwIpAddress": "172.16.102.118"
                            }
                        }
                        
                    ]
                },
                "nsd": {
                    "Id": "Req1",
                    "name": "req1",
                    "VNFs": [
                        {
                            "VNFid": "VNFa",
                            "instances": 1,
                            "requirements": {
                                "cpu": 3,
                                "ram": 8,
                                "storage": 1
                            }
                        },
                        {
                            "VNFid": "VNFb",
                            "instances": 2,
                            "requirements": {
                                "cpu": 4,
                                "ram": 5,
                                "storage": 2
                            }
                        }
                    ],
                    "VNFLinks": [
                        {
                            "source": "172.16.102.116",
                            "required_capacity": 1250000
                        }
                    ],
                    "max_latency": 9.6
                },
                "callback": "string"
                }
        self.conn.request("POST", path, dumps(data), self.headers)
        self.conn.sock.settimeout(self.timeout)
        rsp = self.conn.getresponse()
        data = rsp.read()
        self.assertEqual(rsp.status, 201)
Exemplo n.º 38
0
    def run(self):

        if self.file:
            dataFile = open(self.file, "r")
            cdata = dataFile.read()

            conn = HTTPConnection(self.proxy)
            conn.request("GET", self.url)
            resp = conn.getresponse()
            rdata = resp.read()

            if rdata == cdata:
                self.result = True
            self.data = rdata
            conn.close()
            dataFile.close()
        else:
            conn = HTTPConnection(self.proxy)
            conn.request("GET", self.url)
            resp = conn.getresponse()
            rdata = resp.read()

            if resp.status == httplib.OK:
                self.result = True
            conn.close()
Exemplo n.º 39
0
 def __call__ (self, host, req):
    conn = HTTPConnection (host)
    conn.request ("GET", req)
    r = conn.getresponse()
    return r.read()
        except KeyboardInterrupt:
            self.server.socket.close()


try:
    conf = open("./portconf", "r")
    pport = conf.readline().rstrip().split(':')[1]
    sport1 = conf.readline().rstrip().split(':')[1]
    sport2 = conf.readline().rstrip().split(':')[1]
    server1 = ServerThread(int(sport1))
    server1.start()
    dataFile = open('basic', "r")
    cdata = dataFile.read()
    r = False
    proxy = '127.0.0.1:' + pport
    conn = HTTPConnection(proxy)
    conn.request("GET", "http://127.0.0.1:" + sport1 + "/basic")
    resp = conn.getresponse()
    data = resp.read()
    conn.close()

    time.sleep(3)
    conn2 = HTTPConnection(proxy)
    conn2.request("GET", "http://127.0.0.1:" + sport1 + "/basic")
    resp2 = conn2.getresponse()
    data2 = resp2.read()
    conn2.close()

    time.sleep(3)
    conn3 = HTTPConnection(proxy)
    conn3.request("GET", "http://127.0.0.1:" + sport1 + "/basic")
Exemplo n.º 41
0
def file_to_url(method,
                url,
                file,
                content_type='application/octet-stream',
                user_token=None,
                product_token=None):
    hash = md5.new()
    length = 0

    while True:
        block = file.read(4096)
        if not block:
            break
        length += len(block)
        hash.update(block)

    headers = devpay_headers(user_token, product_token)
    headers = rest_headers(method, url, hash, content_type, headers)

    file.seek(0)

    #print 'Content-Length:', str(length)
    headers['Content-Length'] = str(length)

    c = HTTPConnection(REST_HOST)
    #c.set_debuglevel(9)
    c.connect()
    c.putrequest(method, url)
    for key, value in headers.items():
        c.putheader(key, value)
    c.endheaders()

    while length > 4096:
        block = file.read(4096)
        if not block:
            raise "Unexpected EOF"

        c.send(block)
        sys.stdout.write('.')
        sys.stdout.flush()
        length -= len(block)

    while length > 0:
        block = file.read(length)
        if not block:
            raise "Unexpected EOF"
        c.send(block)
        length -= len(block)

    return c.getresponse()
Exemplo n.º 42
0
 def CreateLabels(self):
     print "[+] Creating three labels..."
     for i in range (0,3):
             conn = HTTPConnection(target,port)
             conn.request("POST", path + "index.php?action=pm;sa=manlabels;sesc="+self.sesc, urlencode({"label" : i, "add" : "Add+New+Label"}), {"Accept": "text/plain","Content-type": "application/x-www-form-urlencoded","Referer": "http://" + target + path + "/index.php?action=pm;sa=manlabels", "Cookie": sn + "=" + sv + ";"})
             sleep(4)
Exemplo n.º 43
0
 def Inject(self):
     print "[+] Sql code is going to be injected."
     conn = HTTPConnection(target,port)
     conn.request("POST", path + "index.php?debug;action=pm;sa=manlabels;sesc="+self.sesc, urlencode({"label_name[0]" : "o rly" + unquote("%a3%27"),"label_name[1]" : "ID_GROUP=1 WHERE/*", "label_name[2]" : "*/ID_MEMBER=" + uid + "/*", "save" : "Save", "sc" : self.sesc, "db_character_set": "big5"}), {"Accept": "text/plain","Content-type": "application/x-www-form-urlencoded","Referer": "http://" + target + path + "/index.php?action=pm;sa=manlabels", "Cookie": sn + "=" + sv + "; 1102461922=1; -1283274824=1;"})
Exemplo n.º 44
0
video_queue = connection.channel()
video_queue.queue_declare(queue=settings.QUEUE_NAME, durable=True)

use_s3 = settings.USE_S3
if use_s3:
    s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID,
                         settings.AWS_SECRET_ACCESS_KEY)
    bucket = s3.get_bucket(settings.AWS_VIDEO_BUCKET_NAME)

ec2_name = settings.INSTANCE_NAME

tmp_folder = os.path.dirname(os.path.abspath(__file__))
if not use_s3:
    tmp_folder = os.path.join(tmp_folder, os.path.pardir, os.path.pardir,
                              'media', 'tmp')
conn = HTTPConnection(settings.DOMAIN_NAME)
conn.request("GET", "/amazon/ec2-ready/%s/" % settings.INSTANCE_ID)
conn.getresponse()


def encode(ch, method, properties, body):
    """Fetches the temporary video file from s3 and encodes both web and mobile
    versions.
    
    Communicates with the screenbird server with the following
    - sets the `ec2_node` field of the associated `VideoStatus` for the video 
      for identification of where to get encoding progress output of ffmpeg
    - sets the `web_available` field of the associated `VideoStatus` for the
      video after successfully encoding the web version
    - sets the `mobile_available` field of the associated `VideoStatus` for the
      video after successfully encoding the mobile version
Exemplo n.º 45
0
 def send(self, astr):
     HTTPConnection.send(self, astr)
     self.request_length += len(astr)
Exemplo n.º 46
0
def encode(ch, method, properties, body):
    """Fetches the temporary video file from s3 and encodes both web and mobile
    versions.
    
    Communicates with the screenbird server with the following
    - sets the `ec2_node` field of the associated `VideoStatus` for the video 
      for identification of where to get encoding progress output of ffmpeg
    - sets the `web_available` field of the associated `VideoStatus` for the
      video after successfully encoding the web version
    - sets the `mobile_available` field of the associated `VideoStatus` for the
      video after successfully encoding the mobile version
    - sets the `video_duration` field of the associated `Video` object 
    
    """
    param = eval(body)
    slug_tmp = param[0]
    host = param[1]
    mp4 = os.path.join(tmp_folder, "%s.mp4" % slug_tmp)
    slug = slug_tmp.replace("_tmp", "")
    text_file = os.path.join(tmp_folder, "%s.txt" % slug)
    output_mp4 = os.path.join(tmp_folder, "%s.mp4" % slug)
    output_mobile_mp4 = os.path.join(tmp_folder, "%s__mobile.mp4" % slug)
    if not use_s3:
        shutil.copyfile(output_mp4, mp4)
    if use_s3:
        key = bucket.get_key(slug_tmp)
    else:
        key = slug
    logger.info("Started encoding for %s" % slug_tmp)

    # Define these as variables to make it easier to change later on if need be
    set_encoding_url = "/set-encoding-ec2/%s/%s/"
    set_available_url = "/set-available/%s/%s/"
    time_diff = None

    if key:
        if use_s3:
            # Download source video to be encoded
            try:
                key.get_contents_to_filename(mp4)
            except:
                ch.basic_ack(delivery_tag=method.delivery_tag)

        # Notify requesting server that video is being processed
        try:
            conn = HTTPConnection(host)
            conn.request("GET", set_encoding_url % (slug, ec2_name))
            response = conn.getresponse()
        except Exception, e:
            pass
        logger.info("Encoding for %s - web version" % slug_tmp)
        available = False

        # Check from requesting server if video is already encoded
        try:
            conn = HTTPConnection(host)
            conn.request("GET", "/is-available/%s/%s/" % (slug, 'web'))
            response = conn.getresponse()
            available = eval(response.read())
        except:
            pass
        try:
            # Start encoding for video if not yet encoded
            if not available:
                start_time = datetime.datetime.now()
                encode_video({
                    'mp4': str(mp4),
                    'output_mp4': str(output_mp4),
                    'text_file': str(text_file)
                })
                end_time = datetime.datetime.now()
                time_diff = end_time - start_time
                logger.info("Web encoding time: %s" % time_diff)
            else:
                logger.info(
                    "Encoded version for %s already available - web version" %
                    slug_tmp)
        except Exception, e:
            logger.info("Encoding for %s failed - web version: %s" %
                        (slug_tmp, e))
            ch.basic_ack(delivery_tag=method.delivery_tag)
            ch.basic_publish(exchange='',
                             routing_key=settings.QUEUE_NAME,
                             body=body)
Exemplo n.º 47
0
 def __init__(self, host, port=None, strict=None):
     HTTPConnection.__init__(self, host, port, strict)
     self.request_length = 0
Exemplo n.º 48
0
def geoserver_rest_proxy(request, proxy_path, downstream_path, workspace=None):

    if not request.user.is_authenticated():
        return HttpResponse(
            "You must be logged in to access GeoServer",
            content_type="text/plain",
            status=401)

    def strip_prefix(path, prefix):
        assert path.startswith(prefix)
        return path[len(prefix):]

    path = strip_prefix(request.get_full_path(), proxy_path)

    access_token = None
    if 'access_token' in request.session:
        access_token = request.session['access_token']

    headers = {}
    affected_layers = None
    cookies = None
    for cook in request.COOKIES:
        name = str(cook)
        value = request.COOKIES.get(name)
        if name == 'csrftoken':
            headers['X-CSRFToken'] = value

        cook = "%s=%s" % (name, value)
        if not cookies:
            cookies = cook
        else:
            cookies = cookies + '; ' + cook

    # if cookies:
    #     if 'JSESSIONID' in request.session and request.session['JSESSIONID']:
    #         cookies = cookies + '; JSESSIONID=' + \
    #             request.session['JSESSIONID']
    #     headers['Cookie'] = cookies

    if 'HTTP_AUTHORIZATION' in request.META:
        auth = request.META.get('HTTP_AUTHORIZATION', request.META.get('HTTP_AUTHORIZATION2'))
        if auth:
            headers['Authorization'] = auth
    elif access_token:
        # TODO: Bearer is currently cutted of by Djano / GeoServer
        if request.method in ("POST", "PUT"):
            if access_token:
                headers['Authorization'] = 'Bearer {}'.format(access_token)
        if access_token and 'access_token' not in path:
            query_separator = '&' if '?' in path else '?'
            path = ('%s%saccess_token=%s' % (path, query_separator, access_token))

    raw_url = str("".join([ogc_server_settings.LOCATION, downstream_path, path]))

    if settings.DEFAULT_WORKSPACE or workspace:
        ws = (workspace or settings.DEFAULT_WORKSPACE)
        if ws and ws in path:
            # Strip out WS from PATH
            try:
                path = "/%s" % strip_prefix(path, "/%s:" % (ws))
            except:
                pass

        if downstream_path in ('rest/styles') and len(request.body) > 0:
            # Lets try http://localhost:8080/geoserver/rest/workspaces/<ws>/styles/<style>.xml
            _url = str("".join([ogc_server_settings.LOCATION,
                                'rest/workspaces/', ws, '/styles',
                                path]))
            raw_url = _url

    url = urlsplit(raw_url)
    if url.scheme == 'https':
        conn = HTTPSConnection(url.hostname, url.port)
    else:
        conn = HTTPConnection(url.hostname, url.port)

    if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META:
        headers["Content-Type"] = request.META["CONTENT_TYPE"]
        # if user is not authorized, we must stop him
        # we need to sync django here and check if some object (styles) can
        # be edited by the user
        # we should remove this geonode dependency calling layers.views straight
        # from GXP, bypassing the proxy
        if downstream_path in ('rest/styles', 'rest/layers', 'rest/workspaces') and len(request.body) > 0:
            if not style_change_check(request, downstream_path):
                return HttpResponse(
                    _("You don't have permissions to change style for this layer"),
                    content_type="text/plain",
                    status=401)
            elif downstream_path == 'rest/styles':
                logger.info("[geoserver_rest_proxy] Updating Style to ---> url %s" % url.path)
                affected_layers = style_update(request, raw_url)

    conn.request(request.method, raw_url, request.body, headers=headers)
    response = conn.getresponse()
    content = response.read()
    status = response.status
    content_type = response.getheader("Content-Type", "text/plain")

    # update thumbnails
    if affected_layers:
        for layer in affected_layers:
            logger.debug('Updating thumbnail for layer with uuid %s' % layer.uuid)
            create_gs_thumbnail(layer, True)

    return HttpResponse(
        content=content,
        status=status,
        content_type=content_type)
Exemplo n.º 49
0
class DynectRest(object):
    """
    A class for interacting with the Dynect Managed DNS REST API.

    @ivar host: The host to connect to (defaults to api.dynect.net)
    @type host: C{str}

    @ivar port: The port to connect to (defaults to 443)
    @type port: C{int}

    @ivar ssl: A boolean indicating whether or not to use SSL encryption
    (defaults to True)
    @type ssl: C{bool}

    @ivar poll_incomplete: A boolean indicating whether we should continue to
    poll for a result if a job comes back as incomplete (defaults to True)
    @type poll_incomplete: C{bool}

    @ivar api_version: The version of the API to request (defaults to
    "current")
    @type api_version: C{str}
    """
    def __init__(self,
                 host='api.dynect.net',
                 port=443,
                 ssl=True,
                 api_version="current"):
        """
        Basic initializer method

        @param host: The host to connect to
        @type host: C{str}
        @param port: The port to connect to
        @type port: C{int}
        @param ssl: A boolean indicating whether or not to use SSL encryption
        @type ssl: C{bool}
        """
        self.host = host
        self.port = port
        self.ssl = ssl

        # Continue polling for response if a job comes back as incomplete?
        self.poll_incomplete = True

        self.verbose = False
        self.api_version = api_version
        self.content_type = "application/json"

        self._token = None
        self._conn = None
        self._last_response = None

        self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT'))

    def _debug(self, msg):
        """
        Debug output.
        """
        if self.verbose:
            sys.stderr.write(msg)

    def connect(self):
        """
        Establishes a connection to the REST API server as defined by the host,
        port and ssl instance variables
        """
        if self._token:
            self._debug("Forcing logout from old session.\n")

            orig_value = self.poll_incomplete
            self.poll_incomplete = False
            self.execute('/REST/Session', 'DELETE')
            self.poll_incomplete = orig_value

            self._token = None

        self._conn = None

        if self.ssl:
            msg = "Establishing SSL connection to %s:%s\n" % (self.host,
                                                              self.port)
            self._debug(msg)
            self._conn = HTTPSConnection(self.host, self.port)

        else:
            msg = "Establishing unencrypted connection to %s:%s\n" % (
                self.host, self.port)
            self._debug(msg)
            self._conn = HTTPConnection(self.host, self.port)

    def execute(self, uri, method, args=None):
        """
        Execute a commands against the rest server

        @param uri: The uri of the resource to access.  /REST/ will be prepended
        if it is not at the beginning of the uri.
        @type uri: C{str}

        @param method: One of 'DELETE', 'GET', 'POST', or 'PUT'
        @type method: C{str}

        @param args: Any arguments to be sent as a part of the request
        @type args: C{dict}
        """
        if self._conn == None:
            self._debug("No established connection\n")
            self.connect()

        # Make sure the command is prefixed by '/REST/'
        if not uri.startswith('/'):
            uri = '/' + uri

        if not uri.startswith('/REST'):
            uri = '/REST' + uri

        # Make sure the method is valid
        if method.upper() not in self._valid_methods:
            msg = "%s is not a valid HTTP method.  Please use one of %s" % (
                method, ", ".join(self._valid_methods))
            raise ValueError(msg)

        # Prepare arguments
        if args is None:
            args = {}

        args = self.format_arguments(args)

        self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args))

        # Send the command and deal with results
        self.send_command(uri, method, args)

        # Deal with the results
        response = self._conn.getresponse()
        body = response.read()
        self._last_response = response

        if self.poll_incomplete:
            response, body = self.poll_response(response, body)
            self._last_response = response

        if sys.version_info[0] == 2:
            ret_val = json.loads(body)
        elif sys.version_info[0] == 3:
            ret_val = json.loads(body.decode('UTF-8'))

        self._meta_update(uri, method, ret_val)

        return ret_val

    def _meta_update(self, uri, method, results):
        """
        Private method, not intended for use outside the class
        """
        # If we had a successful log in, update the token
        if uri.startswith('/REST/Session') and method == 'POST':
            if results['status'] == 'success':
                self._token = results['data']['token']

        # Otherwise, if it's a successful logout, blank the token
        if uri.startswith('/REST/Session') and method == 'DELETE':
            if results['status'] == 'success':
                self._token = None

    def poll_response(self, response, body):
        """
        Looks at a response from a REST command, and while indicates that the
        job is incomplete, poll for response
        """

        while response.status == 307:
            time.sleep(1)
            uri = response.getheader('Location')
            self._debug("Polling %s\n" % uri)

            self.send_command(uri, "GET", '')
            response = self._conn.getresponse()
            body = response.read()

        return response, body

    def send_command(self, uri, method, args):
        """
        Responsible for packaging up the API request and sending it to the 
        server over the established connection

        @param uri: The uri of the resource to interact with
        @type uri: C{str}

        @param method: The HTTP method to use
        @type method: C{str}

        @param args: Encoded arguments to send to the server
        @type args: C{str}
        """
        if '%' not in uri:
            uri = pathname2url(uri)

        self._conn.putrequest(method, uri)

        # Build headers
        headers = {
            'Content-Type': self.content_type,
            'API-Version': self.api_version,
        }

        if self._token is not None:
            headers['Auth-Token'] = self._token

        for key, val in headers.items():
            self._conn.putheader(key, val)

        # Now the arguments
        self._conn.putheader('Content-length', '%d' % len(args))
        self._conn.endheaders()

        if sys.version_info[0] == 2:
            self._conn.send(args)

        elif sys.version_info[0] == 3:
            self._conn.send(bytes(args, 'UTF-8'))

    def format_arguments(self, args):
        """
        Converts the argument dictionary to the format needed to transmit the 
        REST request.

        @param args: Arguments to be passed to the REST call
        @type args: C{dict}

        @return: The encoded string to send in the REST request body
        @rtype: C{str}
        """
        args = json.dumps(args)

        return args
Exemplo n.º 50
0
def validateURL(url, firstOccurrenceOnly=1, wantRawData=0, groupEvents=0):
    """validate RSS from URL, returns events list, or (events, rawdata) tuple"""
    loggedEvents = []
    request = urllib2.Request(url)
    request.add_header("Accept-encoding", "gzip, deflate")
    request.add_header("User-Agent", "FeedValidator/1.3")
    usock = None
    ctx2 = ssl.create_default_context()
    ctx1 = ssl.create_default_context()
    ctx2.set_ciphers('ALL:@SECLEVEL=2')
    ctx1.set_ciphers('ALL:@SECLEVEL=1')
    try:
        try:
            try:
                usock = urllib2.urlopen(request, context=ctx2)
            except urllib2.URLError as x:
                import re
                if True or re.search("WRONG_SIGNATURE_TYPE", x.reason):
                    loggedEvents.append(
                        HttpsProtocolWarning(
                            {'message':
                             "Weak signature used by HTTPS server"}))
                    usock = urllib2.urlopen(request, context=ctx1)
                else:
                    raise x
            rawdata = usock.read(MAXDATALENGTH)
            if usock.read(1):
                raise ValidationFailure(
                    logging.ValidatorLimit({
                        'limit':
                        'feed length > ' + str(MAXDATALENGTH) + ' bytes'
                    }))

            # check for temporary redirects
            if usock.geturl() != request.get_full_url():
                from urlparse import urlsplit
                (scheme, netloc, path, query, fragment) = urlsplit(url)
                if scheme == 'http':
                    from httplib import HTTPConnection
                    requestUri = (path or '/') + (query and '?' + query)

                    conn = HTTPConnection(netloc)
                    conn.request("GET", requestUri)
                    resp = conn.getresponse()
                    if resp.status != 301:
                        loggedEvents.append(TempRedirect({}))

        except BadStatusLine as status:
            raise ValidationFailure(
                logging.HttpError({'status': status.__class__}))

        except urllib2.HTTPError as status:
            rawdata = status.read()
            if len(rawdata) < 512 or 'content-encoding' in status.headers:
                loggedEvents.append(logging.HttpError({'status': status}))
                usock = status
            else:
                rawdata = re.sub('<!--.*?-->', '', rawdata)
                lastline = rawdata.strip().split('\n')[-1].strip()
                if sniffPossibleFeed(rawdata):
                    loggedEvents.append(logging.HttpError({'status': status}))
                    loggedEvents.append(logging.HttpErrorWithPossibleFeed({}))
                    usock = status
                else:
                    raise ValidationFailure(
                        logging.HttpError({'status': status}))
        except urllib2.URLError as x:
            raise ValidationFailure(logging.HttpError({'status': x.reason}))
        except Timeout as x:
            raise ValidationFailure(
                logging.IOError({
                    "message": 'Server timed out',
                    "exception": x
                }))
        except Exception as x:
            raise ValidationFailure(
                logging.IOError({
                    "message": x.__class__.__name__,
                    "exception": x
                }))

        if usock.headers.get('content-encoding', None) == None:
            loggedEvents.append(Uncompressed({}))

        if usock.headers.get('content-encoding', None) == 'gzip':
            import gzip, StringIO
            try:
                rawdata = gzip.GzipFile(
                    fileobj=StringIO.StringIO(rawdata)).read()
            except:
                import sys
                exctype, value = sys.exc_info()[:2]
                event = logging.IOError({
                    "message":
                    'Server response declares Content-Encoding: gzip',
                    "exception": value
                })
                raise ValidationFailure(event)

        if usock.headers.get('content-encoding', None) == 'deflate':
            import zlib
            try:
                rawdata = zlib.decompress(rawdata, -zlib.MAX_WBITS)
            except:
                import sys
                exctype, value = sys.exc_info()[:2]
                event = logging.IOError({
                    "message":
                    'Server response declares Content-Encoding: deflate',
                    "exception": value
                })
                raise ValidationFailure(event)

        if usock.headers.get('content-type',
                             None) == 'application/vnd.google-earth.kmz':
            import tempfile, zipfile, os
            try:
                (fd, tempname) = tempfile.mkstemp()
                os.write(fd, rawdata)
                os.close(fd)
                zfd = zipfile.ZipFile(tempname)
                namelist = zfd.namelist()
                for name in namelist:
                    if name.endswith('.kml'):
                        rawdata = zfd.read(name)
                zfd.close()
                os.unlink(tempname)
            except:
                import sys
                value = sys.exc_info()[:1]
                event = logging.IOError({
                    "message": 'Problem decoding KMZ',
                    "exception": value
                })
                raise ValidationFailure(event)

        mediaType = None
        charset = None

        # Is the Content-Type correct?
        contentType = usock.headers.get('content-type', None)
        if contentType:
            (mediaType,
             charset) = mediaTypes.checkValid(contentType, loggedEvents)

        # Check for malformed HTTP headers
        for (h, v) in usock.headers.items():
            if (h.find(' ') >= 0):
                loggedEvents.append(HttpProtocolError({'header': h}))

        selfURIs = [request.get_full_url()]
        baseURI = usock.geturl()
        if not baseURI in selfURIs: selfURIs.append(baseURI)

        # Get baseURI from content-location and/or redirect information
        if usock.headers.get('content-location', None):
            from urlparse import urljoin
            baseURI = urljoin(baseURI,
                              usock.headers.get('content-location', ""))
        elif usock.headers.get('location', None):
            from urlparse import urljoin
            baseURI = urljoin(baseURI, usock.headers.get('location', ""))

        if not baseURI in selfURIs: selfURIs.append(baseURI)
        usock.close()
        usock = None

        mediaTypes.contentSniffing(mediaType, rawdata, loggedEvents)

        encoding, rawdata = xmlEncoding.decode(mediaType,
                                               charset,
                                               rawdata,
                                               loggedEvents,
                                               fallback='utf-8')

        if rawdata is None:
            return {'loggedEvents': loggedEvents}

        rawdata = rawdata.replace('\r\n', '\n').replace('\r',
                                                        '\n')  # normalize EOL
        validator = _validate(rawdata,
                              firstOccurrenceOnly,
                              loggedEvents,
                              baseURI,
                              encoding,
                              selfURIs,
                              mediaType=mediaType,
                              groupEvents=groupEvents)

        # Warn about mismatches between media type and feed version
        if mediaType and validator.feedType:
            mediaTypes.checkAgainstFeedType(mediaType, validator.feedType,
                                            validator.loggedEvents)

        params = {
            "feedType": validator.feedType,
            "loggedEvents": validator.loggedEvents
        }
        if wantRawData:
            params['rawdata'] = rawdata
        return params

    finally:
        try:
            if usock: usock.close()
        except:
            pass
Exemplo n.º 51
0
 def putrequest(self, *args, **kw):
     self.url = args[1]
     response = self.fakedata.pop(0)     # get first response
     self.sock = FakeSocket(response)    # and set up a fake socket
     output.new()                        # as well as an output buffer
     HTTPConnection.putrequest(self, *args, **kw)
Exemplo n.º 52
0
 def __init__(self, host, port=None, strict=None, timeout=None):
     HTTPConnection.__init__(self, host, port, strict)
     self._timeout = timeout
Exemplo n.º 53
0
 def make_connection(self, host):
     h = HTTPConnection(host, timeout=self.timeout)
     return h
Exemplo n.º 54
0
 def connect(self):
     HTTPConnection.connect(self)
     if self.sock and self._timeout:
         self.sock.settimeout(self._timeout) 
Exemplo n.º 55
0
 def connect(self):
     HTTPConnection.connect(self)
     self.sock.settimeout(self.timeout)
Exemplo n.º 56
0
def makeRequest(url,
                values=None,
                verb='GET',
                accept="text/plain",
                contentType=None,
                secure=False,
                secureParam={}):
    headers = {}
    contentType = contentType or "application/x-www-form-urlencoded"
    headers = {
        "content-type": contentType,
        "Accept": accept,
        "cms-auth-status": "NONE"
    }
    if secure:
        headers.update({
            "cms-auth-status":
            "OK",
            "cms-authn-dn":
            "/DC=ch/OU=Organic Units/OU=Users/CN=Fake User",
            "cms-authn-name":
            "Fake User",
            "cms-authz-%s" % secureParam['role']:
            "group:%s site:%s" % (secureParam['group'], secureParam['site'])
        })
        headers["cms-authn-hmac"] = _generateHash(secureParam["key"], headers)

    data = None
    if verb == 'GET' and values:
        data = urllib.urlencode(values, doseq=True)
    elif verb != 'GET' and values:
        # needs to test other encoding type
        if contentType == "application/x-www-form-urlencoded":
            data = urllib.urlencode(values)
        else:
            # for other encoding scheme values assumed to be encoded already
            data = values
    parser = urlparse(url)
    uri = parser.path
    if parser.query:
        uri += "?" + parser.query

    if verb == 'GET' and data != None:
        uri = '%s?%s' % (uri, data)

    # need to specify Content-length for POST method
    # TODO: this function needs refactoring - too verb-related branching
    if verb != 'GET':
        if data:
            headers.update({"content-length": len(data)})
        else:
            headers.update({"content-length": 0})

    conn = HTTPConnection(parser.netloc)
    conn.connect()
    conn.request(verb, uri, data, headers)
    response = conn.getresponse()

    data = response.read()
    conn.close()
    cType = response.getheader('content-type').split(';')[0]
    return data, response.status, cType, response
Exemplo n.º 57
0
    def retry_using_http_NTLM_auth(self, req, auth_header_field, realm,
                                   headers):
        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
        if pw is not None:
            user_parts = user.split('\\', 1)
            if len(user_parts) == 1:
                UserName = user_parts[0]
                DomainName = ''
                type1_flags = ntlm.NTLM_TYPE1_FLAGS & ~ntlm.NTLM_NegotiateOemDomainSupplied
            else:
                DomainName = user_parts[0].upper()
                UserName = user_parts[1]
                type1_flags = ntlm.NTLM_TYPE1_FLAGS
                # ntlm secures a socket, so we must use the same socket for the complete handshake
            headers = dict(req.headers)
            headers.update(req.unredirected_hdrs)
            auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(
                user, type1_flags)
            if req.headers.get(self.auth_header, None) == auth:
                return None
            headers[self.auth_header] = auth

            host = req.host

            if not host:
                raise urllib2.URLError('no host given')

            h = None

            if req.get_full_url().startswith('https://'):
                h = HTTPSConnection(host)  # will parse host:port
            else:
                h = HTTPConnection(host)  # will parse host:port

            h.set_debuglevel(self._debuglevel)

            # we must keep the connection because NTLM authenticates the connection, not single requests
            headers["Connection"] = "Keep-Alive"
            headers = dict(
                (name.title(), val) for name, val in headers.items())

            # For some reason, six doesn't do this translation correctly
            # TODO rsanders low - find bug in six & fix it
            try:
                selector = req.selector
            except AttributeError:
                selector = req.get_selector()

            h.request(req.get_method(), selector, req.data, headers)

            r = h.getresponse()

            r.begin()

            r._safe_read(int(r.getheader('content-length')))
            if r.getheader('set-cookie'):
                # this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out)
                headers['Cookie'] = r.getheader('set-cookie')
            r.fp = None  # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open)
            auth_header_value = r.getheader(auth_header_field, None)

            # some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge
            # and another with the 'Negotiate' keyword - make sure we operate on the right one
            m = re.match('(NTLM [A-Za-z0-9+\-/=]+)', auth_header_value)
            if m:
                auth_header_value, = m.groups()

            (ServerChallenge,
             NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE(
                 auth_header_value[5:])
            auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(
                ServerChallenge, UserName, DomainName, pw, NegotiateFlags)
            headers[self.auth_header] = auth
            headers["Connection"] = "Close"
            headers = dict(
                (name.title(), val) for name, val in headers.items())
            try:
                h.request(req.get_method(), selector, req.data, headers)
                # none of the configured handlers are triggered, for example redirect-responses are not handled!
                response = h.getresponse()

                def notimplemented():
                    raise NotImplementedError

                response.readline = notimplemented
                infourl = urllib.addinfourl(response, response.msg,
                                            req.get_full_url())
                infourl.code = response.status
                infourl.msg = response.reason
                return infourl
            except socket.error as err:
                raise urllib2.URLError(err)
        else:
            return None
Exemplo n.º 58
0
 def __init__(self, host, port, timeout):
     HTTPConnection.__init__(self, host, port)
     self.timeout = timeout
Exemplo n.º 59
0
def proxy(request,
          url=None,
          response_callback=None,
          sec_chk_hosts=True,
          sec_chk_rules=True,
          **kwargs):
    # Security rules and settings
    PROXY_ALLOWED_HOSTS = getattr(settings, 'PROXY_ALLOWED_HOSTS', ())

    # Sanity url checks
    if 'url' not in request.GET and not url:
        return HttpResponse(
            "The proxy service requires a URL-encoded URL as a parameter.",
            status=400,
            content_type="text/plain")

    raw_url = url or request.GET['url']
    raw_url = urljoin(settings.SITEURL,
                      raw_url) if raw_url.startswith("/") else raw_url
    url = urlsplit(raw_url)
    locator = str(url.path)
    if url.query != "":
        locator += '?' + url.query
    if url.fragment != "":
        locator += '#' + url.fragment

    access_token = None
    if request and 'access_token' in request.session:
        access_token = request.session['access_token']

    # White-Black Listing Hosts
    if sec_chk_hosts and not settings.DEBUG:
        site_url = urlsplit(settings.SITEURL)
        if site_url.hostname not in PROXY_ALLOWED_HOSTS:
            PROXY_ALLOWED_HOSTS += (site_url.hostname, )

        if check_ogc_backend(geoserver.BACKEND_PACKAGE):
            from geonode.geoserver.helpers import ogc_server_settings
            hostname = (
                ogc_server_settings.hostname, ) if ogc_server_settings else ()
            if hostname not in PROXY_ALLOWED_HOSTS:
                PROXY_ALLOWED_HOSTS += hostname

        if url.query and ows_regexp.match(url.query):
            ows_tokens = ows_regexp.match(url.query).groups()
            if len(
                    ows_tokens
            ) == 4 and 'version' == ows_tokens[0] and StrictVersion(
                    ows_tokens[1]) >= StrictVersion("1.0.0") and StrictVersion(
                        ows_tokens[1]
                    ) <= StrictVersion("3.0.0") and ows_tokens[2].lower() in (
                        'getcapabilities') and ows_tokens[3].upper() in (
                            'OWS', 'WCS', 'WFS', 'WMS', 'WPS', 'CSW'):
                if url.hostname not in PROXY_ALLOWED_HOSTS:
                    PROXY_ALLOWED_HOSTS += (url.hostname, )

        if not validate_host(url.hostname, PROXY_ALLOWED_HOSTS):
            return HttpResponse(
                "DEBUG is set to False but the host of the path provided to the proxy service"
                " is not in the PROXY_ALLOWED_HOSTS setting.",
                status=403,
                content_type="text/plain")

    # Security checks based on rules; allow only specific requests
    if sec_chk_rules:
        # TODO: Not yet implemented
        pass

    # Collecting headers and cookies
    headers = {}
    cookies = None
    csrftoken = None

    if settings.SESSION_COOKIE_NAME in request.COOKIES and is_safe_url(
            url=raw_url, host=url.hostname):
        cookies = request.META["HTTP_COOKIE"]

    for cook in request.COOKIES:
        name = str(cook)
        value = request.COOKIES.get(name)
        if name == 'csrftoken':
            csrftoken = value
        cook = "%s=%s" % (name, value)
        cookies = cook if not cookies else (cookies + '; ' + cook)

    csrftoken = get_token(request) if not csrftoken else csrftoken

    if csrftoken:
        headers['X-Requested-With'] = "XMLHttpRequest"
        headers['X-CSRFToken'] = csrftoken
        cook = "%s=%s" % ('csrftoken', csrftoken)
        cookies = cook if not cookies else (cookies + '; ' + cook)

    if cookies:
        if 'JSESSIONID' in request.session and request.session['JSESSIONID']:
            cookies = cookies + '; JSESSIONID=' + \
                request.session['JSESSIONID']
        headers['Cookie'] = cookies

    if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META:
        headers["Content-Type"] = request.META["CONTENT_TYPE"]

    access_token = None
    if request and 'access_token' in request.session:
        access_token = request.session['access_token']

    if access_token:
        # TODO: Bearer is currently cutted of by Djano / GeoServer
        if request.method in ("POST", "PUT"):
            headers['Authorization'] = 'Bearer %s' % access_token
        if access_token and 'access_token' not in locator:
            query_separator = '&' if '?' in locator else '?'
            locator = ('%s%saccess_token=%s' %
                       (locator, query_separator, access_token))
    elif 'HTTP_AUTHORIZATION' in request.META:
        auth = request.META.get('HTTP_AUTHORIZATION',
                                request.META.get('HTTP_AUTHORIZATION2'))
        if auth:
            headers['Authorization'] = auth

    site_url = urlsplit(settings.SITEURL)

    pragma = "no-cache"
    referer = request.META[
        "HTTP_REFERER"] if "HTTP_REFERER" in request.META else \
        "{scheme}://{netloc}/".format(scheme=site_url.scheme,
                                      netloc=site_url.netloc)
    encoding = request.META[
        "HTTP_ACCEPT_ENCODING"] if "HTTP_ACCEPT_ENCODING" in request.META else "gzip"

    headers.update({
        "Pragma": pragma,
        "Referer": referer,
        "Accept-encoding": encoding,
    })

    if url.scheme == 'https':
        conn = HTTPSConnection(url.hostname, url.port)
    else:
        conn = HTTPConnection(url.hostname, url.port)
    conn.request(request.method, locator.encode('utf8'), request.body, headers)
    response = conn.getresponse()
    content = response.read()
    status = response.status
    content_type = response.getheader("Content-Type", "text/plain")

    # decompress GZipped responses if not enabled
    if content and response.getheader('Content-Encoding') == 'gzip':
        from StringIO import StringIO
        import gzip
        buf = StringIO(content)
        f = gzip.GzipFile(fileobj=buf)
        content = f.read()

    if response_callback:
        kwargs = {} if not kwargs else kwargs
        kwargs.update({
            'response': response,
            'content': content,
            'status': status,
            'content_type': content_type
        })
        return response_callback(**kwargs)
    else:
        # If we get a redirect, let's add a useful message.
        if status in (301, 302, 303, 307):
            _response = HttpResponse(
                ('This proxy does not support redirects. The server in "%s" '
                 'asked for a redirect to "%s"' %
                 (url, response.getheader('Location'))),
                status=status,
                content_type=content_type)
            _response['Location'] = response.getheader('Location')
            return _response
        else:
            return HttpResponse(content=content,
                                status=status,
                                content_type=content_type)
Exemplo n.º 60
0
    def _enrich_layer_metadata(self, geonode_layer):
        url = urlsplit(self.url)

        if url.scheme == 'https':
            conn = HTTPSConnection(url.hostname, url.port)
        else:
            conn = HTTPConnection(url.hostname, url.port)

        workspace, layername = geonode_layer.name.split(
            ":") if ":" in geonode_layer.name else (None, geonode_layer.name)
        conn.request('GET', '/api/layers/?name=%s' % layername, '', {})
        response = conn.getresponse()
        content = response.read()
        status = response.status
        content_type = response.getheader("Content-Type", "text/plain")

        if status == 200 and 'application/json' == content_type:
            try:
                _json_obj = json.loads(content)
                if _json_obj['meta']['total_count'] == 1:
                    _layer = _json_obj['objects'][0]
                    if _layer:
                        r_fields = {}

                        # Update plain fields
                        for field in GeoNodeServiceHandler.LAYER_FIELDS:
                            if field in _layer and _layer[field]:
                                r_fields[field] = _layer[field]
                        if r_fields:
                            Layer.objects.filter(id=geonode_layer.id).update(
                                **r_fields)
                            geonode_layer.refresh_from_db()

                        # Update Thumbnail
                        if "thumbnail_url" in _layer and _layer[
                                "thumbnail_url"]:
                            thumbnail_remote_url = _layer["thumbnail_url"]
                            _url = urlsplit(thumbnail_remote_url)
                            if not _url.scheme:
                                thumbnail_remote_url = "{}{}".format(
                                    geonode_layer.remote_service.service_url,
                                    _url.path)
                            resp, image = http_client.request(
                                thumbnail_remote_url)
                            if 'ServiceException' in image or \
                               resp.status < 200 or resp.status > 299:
                                msg = 'Unable to obtain thumbnail: %s' % image
                                logger.debug(msg)

                                # Replace error message with None.
                                image = None

                            if image is not None:
                                thumbnail_name = 'layer-%s-thumb.png' % geonode_layer.uuid
                                geonode_layer.save_thumbnail(thumbnail_name,
                                                             image=image)
                            else:
                                self._create_layer_thumbnail(geonode_layer)

                        # Add Keywords
                        if "keywords" in _layer and _layer["keywords"]:
                            keywords = _layer["keywords"]
                            if keywords:
                                geonode_layer.keywords.clear()
                                geonode_layer.keywords.add(*keywords)

                        # Add Regions
                        if "regions" in _layer and _layer["regions"]:
                            (regions_resolved,
                             regions_unresolved) = resolve_regions(
                                 _layer["regions"])
                            if regions_resolved:
                                geonode_layer.regions.clear()
                                geonode_layer.regions.add(*regions_resolved)

                        # Add Topic Category
                        if "category__gn_description" in _layer and _layer[
                                "category__gn_description"]:
                            try:
                                categories = TopicCategory.objects.filter(
                                    Q(gn_description__iexact=_layer[
                                        "category__gn_description"]))
                                if categories:
                                    geonode_layer.category = categories[0]
                            except BaseException:
                                traceback.print_exc()
            except BaseException:
                traceback.print_exc()
            finally:
                geonode_layer.save()