示例#1
0
    def __init__(self, core):
        """Constructor"""
        self.core = core
        self.log = core.log

        self.threads = []  # thread list
        self.localThreads = []  #addon+decrypter threads

        self.pause = True

        self.reconnecting = Event()
        self.reconnecting.clear()
        self.downloaded = 0 #number of files downloaded since last cleanup

        self.lock = Lock()

        # some operations require to fetch url info from hoster, so we caching them so it wont be done twice
        # contains a timestamp and will be purged after timeout
        self.infoCache = {}

        # pool of ids for online check
        self.resultIDs = 0

        # threads which are fetching hoster results
        self.infoResults = {}
        # timeout for cache purge
        self.timestamp = 0

        pycurl.global_init(pycurl.GLOBAL_DEFAULT)

        for i in range(self.core.config.get("download", "max_downloads")):
            self.createThread()
示例#2
0
def daemonize():
    """Properly deamonizes the process and closes file desriptors."""
    sys.stderr.flush()
    sys.stdout.flush()

    pid = os.fork()
    if pid != 0:
        # Nothing more to do for the parent
        sys.exit(0)

    os.setsid()
    pid = os.fork()
    if pid != 0:
        # Nothing more to do for the parent
        sys.exit(0)

    os.umask(0)
    os.chdir('/')

    dev_null = open('/dev/null', 'w')
    os.dup2(dev_null.fileno(), sys.stdin.fileno())
    os.dup2(dev_null.fileno(), sys.stdout.fileno())
    os.dup2(dev_null.fileno(), sys.stderr.fileno())

    # Re-initialize cURL. This is necessary to re-initialze the PKCS #11
    # security tokens in NSS. Otherwise any use of SDK after the fork() would
    # lead to the error:
    #
    #    A PKCS #11 module returned CKR_DEVICE_ERROR, indicating that a
    #    problem has occurred with the token or slot.
    #
    pycurl.global_cleanup()
    pycurl.global_init(pycurl.GLOBAL_ALL)
def main():
    global saw_error

    pycurl.global_init(pycurl.GLOBAL_DEFAULT)

    outf = file("/dev/null", "rb+")
    cm = pycurl.CurlMulti()

    # Set multi handle's options
    cm.setopt(pycurl.M_PIPELINING, 1)

    eh = pycurl.Curl()

    for x in range(1, 20):

        eh.setopt(pycurl.WRITEDATA, outf)
        eh.setopt(pycurl.URL, sys.argv[1])
        cm.add_handle(eh)

        while 1:
            ret, active_handles = cm.perform()
            if ret != pycurl.E_CALL_MULTI_PERFORM:
                break

        while active_handles:
            ret = cm.select(1.0)
            if ret == -1:
                continue
            while 1:
                ret, active_handles = cm.perform()
                if ret != pycurl.E_CALL_MULTI_PERFORM:
                    break

        count, good, bad = cm.info_read()

        for h, en, em in bad:
            print "Transfer to %s failed with %d, %s\n" % \
                (h.getinfo(pycurl.EFFECTIVE_URL), en, em)
            raise RuntimeError

        for h in good:
            httpcode = h.getinfo(pycurl.RESPONSE_CODE)
            if httpcode != 200:
                print "Transfer to %s failed with code %d\n" %\
                    (h.getinfo(pycurl.EFFECTIVE_URL), httpcode)
                raise RuntimeError

            else:
                print "Recd %d bytes from %s" % \
                    (h.getinfo(pycurl.SIZE_DOWNLOAD),
                    h.getinfo(pycurl.EFFECTIVE_URL))

        cm.remove_handle(eh)
        eh.reset()

    eh.close()
    cm.close()
    outf.close()

    pycurl.global_cleanup()
示例#4
0
 def test_global_init_ack_eintr(self):
     # the GLOBAL_ACK_EINTR flag was introduced in libcurl-7.30, but can also
     # be backported for older versions of libcurl at the distribution level
     if not util.pycurl_version_less_than(7, 30) or hasattr(pycurl, 'GLOBAL_ACK_EINTR'):
         # initialize libcurl with the GLOBAL_ACK_EINTR flag
         pycurl.global_init(pycurl.GLOBAL_ACK_EINTR)
         pycurl.global_cleanup()
示例#5
0
    def __init__(self, core):
        """Constructor"""
        self.core = core
        self.log = core.log

        self.threads = []  # thread list
        self.localThreads = []  #addon+decrypter threads

        self.pause = True

        self.reconnecting = Event()
        self.reconnecting.clear()
        self.downloaded = 0  #number of files downloaded since last cleanup

        self.lock = Lock()

        # some operations require to fetch url info from hoster, so we caching them so it wont be done twice
        # contains a timestamp and will be purged after timeout
        self.infoCache = {}

        # pool of ids for online check
        self.resultIDs = 0

        # threads which are fetching hoster results
        self.infoResults = {}
        # timeout for cache purge
        self.timestamp = 0

        pycurl.global_init(pycurl.GLOBAL_DEFAULT)

        for i in range(self.core.config.get("download", "max_downloads")):
            self.createThread()
def main():
        global saw_error

        pycurl.global_init(pycurl.GLOBAL_DEFAULT)

        outf = file("/dev/null", "rb+")
        cm = pycurl.CurlMulti()

        # Set multi handle's options
        cm.setopt(pycurl.M_PIPELINING, 1)

        eh = pycurl.Curl()

        for x in range(1, 20):

                eh.setopt(pycurl.WRITEDATA, outf)
                eh.setopt(pycurl.URL, sys.argv[1])
                cm.add_handle(eh)

                while 1:
                        ret, active_handles = cm.perform()
                        if ret != pycurl.E_CALL_MULTI_PERFORM:
                                break

                while active_handles:
                        ret = cm.select(1.0)
                        if ret == -1:
                                continue
                        while 1:
                                ret, active_handles = cm.perform()
                                if ret != pycurl.E_CALL_MULTI_PERFORM:
                                        break

                count, good, bad = cm.info_read()

                for h, en, em in bad:
                        print "Transfer to %s failed with %d, %s\n" % \
                            (h.getinfo(pycurl.EFFECTIVE_URL), en, em)
                        raise RuntimeError

                for h in good:
                        httpcode = h.getinfo(pycurl.RESPONSE_CODE)
                        if httpcode != 200:
                                print "Transfer to %s failed with code %d\n" %\
                                    (h.getinfo(pycurl.EFFECTIVE_URL), httpcode)
                                raise RuntimeError

                        else:
                                print "Recd %d bytes from %s" % \
                                    (h.getinfo(pycurl.SIZE_DOWNLOAD),
                                    h.getinfo(pycurl.EFFECTIVE_URL))

                cm.remove_handle(eh)
                eh.reset()

        eh.close()
        cm.close()
        outf.close()

        pycurl.global_cleanup()
 def assembled(cls):
   cls.needs_registration = True
   cls.needs_perform = False
   pycurl.global_init(pycurl.GLOBAL_ALL)
   cls.multi = pycurl.CurlMulti()
   cls.multi.setopt(pycurl.M_TIMERFUNCTION, cls.curl_timeout)
   cls.multi.setopt(pycurl.M_SOCKETFUNCTION, cls.on_socket)
   cls.instances = {}
示例#8
0
 def cleanPycurl(self):
     """ make a global curl cleanup (currently unused) """
     if self.processingIds():
         return False
     pycurl.global_cleanup()
     pycurl.global_init(pycurl.GLOBAL_DEFAULT)
     self.downloaded = 0
     self.log.debug("Cleaned up pycurl")
     return True
示例#9
0
 def cleanPycurl(self):
     """ make a global curl cleanup (currently unused) """
     if self.processingIds():
         return False
     pycurl.global_cleanup()
     pycurl.global_init(pycurl.GLOBAL_DEFAULT)
     self.downloaded = 0
     self.log.debug("Cleaned up pycurl")
     return True
示例#10
0
 def test_global_init_ack_eintr(self):
     # the GLOBAL_ACK_EINTR flag was introduced in libcurl-7.30, but can also
     # be backported for older versions of libcurl at the distribution level
     if util.pycurl_version_less_than(7, 30) and not hasattr(pycurl, 'GLOBAL_ACK_EINTR'):
         raise nose.plugins.skip.SkipTest('libcurl < 7.30.0 or no GLOBAL_ACK_EINTR')
     
     # initialize libcurl with the GLOBAL_ACK_EINTR flag
     pycurl.global_init(pycurl.GLOBAL_ACK_EINTR)
     pycurl.global_cleanup()
示例#11
0
    def test_global_init_ack_eintr(self):
        # the GLOBAL_ACK_EINTR flag was introduced in libcurl-7.30, but can also
        # be backported for older versions of libcurl at the distribution level
        if util.pycurl_version_less_than(
                7, 30) and not hasattr(pycurl, 'GLOBAL_ACK_EINTR'):
            raise unittest.SkipTest('libcurl < 7.30.0 or no GLOBAL_ACK_EINTR')

        # initialize libcurl with the GLOBAL_ACK_EINTR flag
        pycurl.global_init(pycurl.GLOBAL_ACK_EINTR)
        pycurl.global_cleanup()
    def __init__(self, options):

        webdav_options = get_options(type=WebDAVSettings, from_options=options)
        proxy_options = get_options(type=ProxySettings, from_options=options)

        self.webdav = WebDAVSettings(webdav_options)
        self.proxy = ProxySettings(proxy_options)

        pycurl.global_init(pycurl.GLOBAL_DEFAULT)

        self.default_options = {}
    def __init__(self, options):

        webdav_options = get_options(type=WebDAVSettings, from_options=options)
        proxy_options = get_options(type=ProxySettings, from_options=options)

        self.webdav = WebDAVSettings(webdav_options)
        self.proxy = ProxySettings(proxy_options)

        pycurl.global_init(pycurl.GLOBAL_DEFAULT)

        self.default_options = {}
示例#14
0
文件: client.py 项目: ekohl/ganeti
  def wrapper(*args, **kwargs):
    # curl_global_init(3) and curl_global_cleanup(3) must be called with only
    # one thread running. This check is just a safety measure -- it doesn't
    # cover all cases.
    assert threading.activeCount() == 1, \
           "Found active threads when initializing pycURL"

    pycurl.global_init(pycurl.GLOBAL_ALL)
    try:
      return fn(*args, **kwargs)
    finally:
      pycurl.global_cleanup()
示例#15
0
 def __init__(self, url):
     # Remove trailing slashes from url
     url = url.rstrip('/')
     url = url + '/json/'
     self.url = url
     self.is_error = 0
     self.last_error = ''
     self.curl_response = ''
     self.curl_content = ''
     self.curl_content_type = ''
     self.curl_headers = ''
     self.cookie = ''
     pycurl.global_init(pycurl.GLOBAL_ALL)
示例#16
0
def dj_pycurl_download(url):
    pycurl.global_init(pycurl.GLOBAL_ALL)
    c = pycurl.Curl()

    c.setopt(pycurl.URL, url)
    c.setopt(pycurl.WRITEDATA, fp)
    c.setopt(pycurl.WRITEFUNCTION, dj_pycurl_writeFile)
    c.setopt(pycurl.NOPROGRESS, 0)
    c.setopt(pycurl.CONNECTTIMEOUT, DJ_PYCURL_CONNECTTIMEOUT)
    c.setopt(pycurl.TIMEOUT, DJ_PYCURL_TIMEOUT)
    c.setopt(pycurl.VERBOSE, 1)

    c.perform()
    c.close()
    fp.close()
示例#17
0
def dj_pycurl_download(url):
    pycurl.global_init(pycurl.GLOBAL_ALL)
    c = pycurl.Curl()

    c.setopt(pycurl.URL, url)
    c.setopt(pycurl.WRITEDATA,fp)
    c.setopt(pycurl.WRITEFUNCTION,dj_pycurl_writeFile)
    c.setopt(pycurl.NOPROGRESS,0)
    c.setopt(pycurl.CONNECTTIMEOUT,DJ_PYCURL_CONNECTTIMEOUT)
    c.setopt(pycurl.TIMEOUT,DJ_PYCURL_TIMEOUT)
    c.setopt(pycurl.VERBOSE,1)

    c.perform()
    c.close()
    fp.close()
示例#18
0
def Init():
    """Initializes the module-global HTTP client manager.

  Must be called before using any RPC function and while exactly one thread is
  running.

  """
    # curl_global_init(3) and curl_global_cleanup(3) must be called with only
    # one thread running. This check is just a safety measure -- it doesn't
    # cover all cases.
    assert threading.activeCount() == 1, \
           "Found more than one active thread when initializing pycURL"

    logging.info("Using PycURL %s", pycurl.version)

    pycurl.global_init(pycurl.GLOBAL_ALL)
示例#19
0
文件: rpc.py 项目: ekohl/ganeti
def Init():
  """Initializes the module-global HTTP client manager.

  Must be called before using any RPC function and while exactly one thread is
  running.

  """
  # curl_global_init(3) and curl_global_cleanup(3) must be called with only
  # one thread running. This check is just a safety measure -- it doesn't
  # cover all cases.
  assert threading.activeCount() == 1, \
         "Found more than one active thread when initializing pycURL"

  logging.info("Using PycURL %s", pycurl.version)

  pycurl.global_init(pycurl.GLOBAL_ALL)
示例#20
0
    def perform(self):
        print self.version()
        filesize = self.get_filesize()
        pycurl.global_init(
            pycurl.GLOBAL_ALL)  # GLOBAL_ALL must be set in normal

        if filesize == -1:  # length not known, use single connection instead
            c = self.gen_curl()
            outfile = self.try_soutfile(self.filename)
            c.setopt(pycurl.WRITEFUNCTION, outfile.write)
            c.perform()
            outfile.close()
        else:
            curlpool = []
            blocksize = filesize / self.num_blocks + 1
            print filesize
            for p_start, p_end in [(x, x + blocksize)
                                   for x in xrange(0, filesize, blocksize)]:
                curlpool.append(self.gen_curl(p_start, p_end, filesize))
            m = pycurl.CurlMulti()
            m.handles = []
            for c in curlpool:
                m.add_handle(c)
                m.handles.append(c)
            try:
                while True:
                    ret, num_handles = m.perform()
                    if ret != pycurl.E_CALL_MULTI_PERFORM:
                        break
                while num_handles:
                    ret = m.select(1.0)
                    if ret == -1:
                        continue

                    while True:
                        ret, num_handles = m.perform()
                        if ret != pycurl.E_CALL_MULTI_PERFORM:
                            break
                self.end_perform(normal=True)
                self.event.set()
            except KeyboardInterrupt:
                self.end_perform(normal=False)
            except SystemExit:
                self.end_perform(normal=False)

        pycurl.global_cleanup()
示例#21
0
 def perform(self):
     print self.version()   
     filesize = self.get_filesize()
     pycurl.global_init(pycurl.GLOBAL_ALL) # GLOBAL_ALL must be set in normal
     
     if filesize == -1: # length not known, use single connection instead
         c = self.gen_curl()
         outfile = self.try_soutfile(self.filename)
         c.setopt(pycurl.WRITEFUNCTION, outfile.write)
         c.perform()
         outfile.close()
     else:
         curlpool = []
         blocksize = filesize / self.num_blocks + 1
         print filesize
         for p_start, p_end in [(x, x + blocksize) for x in xrange(0, filesize, blocksize)]:
             curlpool.append(self.gen_curl(p_start, p_end, filesize))
         m = pycurl.CurlMulti()
         m.handles = []
         for c in curlpool:
             m.add_handle(c)
             m.handles.append(c)
         try:
             while True:
                 ret, num_handles = m.perform()
                 if ret != pycurl.E_CALL_MULTI_PERFORM:
                     break
             while num_handles:
                 ret = m.select(1.0)
                 if ret == -1:
                     continue
                 
                 while True:
                     ret, num_handles = m.perform()
                     if ret != pycurl.E_CALL_MULTI_PERFORM:
                         break
             self.end_perform(normal = True)
             self.event.set()
         except KeyboardInterrupt:
             self.end_perform(normal = False)
         except SystemExit:
             self.end_perform(normal = False)
             
     pycurl.global_cleanup()
示例#22
0
    def __init__(self, options):

        self.options = options
        self.server_hostname = options.get("webdav_hostname", '')
        self.server_login = options.get("webdav_login", '')
        self.server_password = options.get("webdav_password", '')
        self.proxy_hostname = options.get("proxy_hostname", '')
        self.proxy_login = options.get("proxy_login", '')
        self.proxy_password = options.get("proxy_password", '')
        self.cert_path = options.get("cert_path", '')
        self.key_path = options.get("key_path", '')

        webdav_root = options.get("webdav_root", '')
        self.webdav_root = Urn(webdav_root).quote() if webdav_root else ''
        self.webdav_root = self.webdav_root.rstrip(Urn.separate)

        pycurl.global_init(pycurl.GLOBAL_DEFAULT)

        self.default_options = {}
示例#23
0
    def __init__(self, options):

        self.options = options
        self.server_hostname = options.get("webdav_hostname", '')
        self.server_login = options.get("webdav_login", '')
        self.server_password = options.get("webdav_password", '')
        self.proxy_hostname = options.get("proxy_hostname", '')
        self.proxy_login = options.get("proxy_login", '')
        self.proxy_password = options.get("proxy_password", '')
        self.cert_path = options.get("cert_path", '')
        self.key_path = options.get("key_path", '')

        webdav_root = options.get("webdav_root", '')
        self.webdav_root = Urn(webdav_root).quote() if webdav_root else ''
        self.webdav_root = self.webdav_root.rstrip(Urn.separate)

        pycurl.global_init(pycurl.GLOBAL_DEFAULT)

        self.default_options = {}
    def get_json(self, url):
        # Set up the PyCurl process so we can download the results
        # directly. We do this so it is easier to pull out the data
        # later (e.g., cache it).
        pycurl.global_init(pycurl.GLOBAL_DEFAULT)

        # Retrieve the JSON for that specific movie.
        curl = pycurl.Curl()
        curl.setopt(pycurl.URL, url)
        curl.setopt(pycurl.HTTPHEADER, ["Accept: application/json"])

        # Download into a string.
        buf = StringIO.StringIO()
        curl.setopt(pycurl.WRITEFUNCTION, buf.write)
        curl.setopt(pycurl.FOLLOWLOCATION, 1)
        curl.setopt(pycurl.MAXREDIRS, 5)
        curl.perform()
        
        # Return the resulting JSON file.
        json = simplejson.loads(buf.getvalue())
        return json
示例#25
0
 def get_filesize(self):
     if hasattr(self, 'filesize') and self.filesize != None:
         return self.filesize
     
     pycurl.global_init(pycurl.GLOBAL_ALL) # GLOBAL_ALL must be set in normal
     curl = pycurl.Curl()
     curl.setopt(pycurl.HEADER, True)
     curl.setopt(pycurl.NOBODY, True)
     curl.setopt(pycurl.URL, self.url)
     curl.setopt(pycurl.TIMEOUT, HEADER_TIMEOUT)
     
     b = StringIO.StringIO()
     curl.setopt(pycurl.WRITEFUNCTION, b.write)
     curl.perform()
     try:
         size = int(re.findall("Content-Length: (\d+)", b.getvalue())[0])
     except:
         size = -1
     
     pycurl.global_cleanup()
     
     self.filesize = size
     return size
示例#26
0
    def get_filesize(self):
        if hasattr(self, 'filesize') and self.filesize != None:
            return self.filesize

        pycurl.global_init(
            pycurl.GLOBAL_ALL)  # GLOBAL_ALL must be set in normal
        curl = pycurl.Curl()
        curl.setopt(pycurl.HEADER, True)
        curl.setopt(pycurl.NOBODY, True)
        curl.setopt(pycurl.URL, self.url)
        curl.setopt(pycurl.TIMEOUT, HEADER_TIMEOUT)

        b = StringIO.StringIO()
        curl.setopt(pycurl.WRITEFUNCTION, b.write)
        curl.perform()
        try:
            size = int(re.findall("Content-Length: (\d+)", b.getvalue())[0])
        except:
            size = -1

        pycurl.global_cleanup()

        self.filesize = size
        return size
示例#27
0
def main():
	usage = "usage: %prog [options]"
	parser = OptionParser(usage=usage)
	
	#	parser.add_option('-q', '--quiet', action="store_const", const=0, dest="v", default=1, help='quiet')
	#	parser.add_option('-v', '--verbose', action="store_const", const=1, dest="v", help='verbose')
	
	parser.add_option('-c', '--config', action='store', dest='config', default=DEFAULT_CONF, help='load parameters from configfile (default: ' + DEFAULT_CONF + ')')
	
	parser.add_option('-t', '--tcp', action='store_const', dest='mode', const='tcp', help='tcp mode (default)')
	
	#parser.add_option('-u', '--udp', action='store_const', dest='t', const='udp', help='udp mode')
		
	parser.add_option('-p', '--port', action="store", type='int', dest="port", help='port to listen (default: '+ str(DEFAULT_LISTENPORT) +')')
	
	parser.add_option('--url', action="store", dest="url", help='URL of tunnelendpoint (default: '+ DEFAULT_URL +')')
	
	parser.add_option('-d', '--dest', action="store", dest="dest", help='destination to connect to (default ' + DEFAULT_TARGET + ')')
	
	parser.add_option('--proxy', action='store', dest='proxy', help='proxy to use')
	
	parser.add_option('--auth', action='store', dest='auth', help='auth with user:password')
	
	parser.add_option('-v', '--verbose', action='store_const', dest='verbose', const=1, help='verbose')
	
	
	#parser.add_option('--no-proxy', action='store_true', dest='np', default=False, help='use no proxy (default: use proxy from env)')
	
	global options
	(options, args) = parser.parse_args()
	
	cparser = ConfigParser.ConfigParser(defaults={
		'mode': 'tcp',
		'port': DEFAULT_LISTENPORT,
		'url': DEFAULT_URL,
		'dest': DEFAULT_TARGET,
		'auth': '',
		'proxy': '',
		'verbose': 0
		})

	cparser.read(options.config)
	
	if cparser.has_section('pyhstopc'):
		if not options.mode:	options.mode = cparser.get('pyhstopc', 'mode')
		if not options.port:	options.port = cparser.getint('pyhstopc', 'port')
		if not options.url:	options.url = cparser.get('pyhstopc', 'url')
		if not options.dest:	options.dest = cparser.get('pyhstopc', 'dest')
		if not options.auth:	options.auth = cparser.get('pyhstopc', 'auth')
		if not options.proxy:	options.proxy = cparser.get('pyhstopc', 'proxy')
		try:
			if not options.verbose:	options.verbose = cparser.getint('pyhstopc', 'verbose')
		except TypeError:
			options.verbose = 0
	
	cparser = None
		
	print 'pyhstopc Version: ' + VERSION
	print 'terminate with EOF'

	print 'start..'
	
	if USE_CURL:
		pycurl.global_init(pycurl.GLOBAL_ALL)
	
	sl = socketListener()
	sl.listen()
		
	input = sys.stdin.readline()
	while input:
		input = sys.stdin.readline()
		
	sl.terminate()
	if USE_CURL:
		pycurl.global_cleanup()
	print 'end..'
 def __init__(self, pool, cr):
     super(taobao_shop, self).__init__(pool, cr)
     #pycurl两个函数不是线程安全。所以在主线程中进行一次的初始化和清除
     import pycurl
     pycurl.global_init(pycurl.GLOBAL_DEFAULT)
     pycurl.global_cleanup()
示例#29
0
def get_data(
    config,
    protocol,
    host,
    port,
    rel_path,
):

    if port:
        url = '%s://%s:%s/%s' % (protocol, host, port, rel_path)
    else:
        url = '%s://%s/%s' % (protocol, host, rel_path)

    passphrase = ''
    try:
        pp_file = open(config.passphrase_file, 'r')
        passphrase = pp_file.readline().strip()
        pp_file.close()
    except:
        config.logger.error('Failed to read passprase from %s',
                            config.passphrase_file)
        return None

    # Store output in memory

    output = StringIO.StringIO()

    # Init cURL (not strictly necessary, but for symmetry with cleanup)

    pycurl.global_init(pycurl.GLOBAL_SSL)

    curl = pycurl.Curl()
    # Never use proxy
    curl.setopt(pycurl.PROXY, "")
    curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP GET'])
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.MAXREDIRS, 5)
    curl.setopt(pycurl.URL, url)
    curl.setopt(pycurl.WRITEFUNCTION, output.write)
    curl.setopt(pycurl.NOSIGNAL, 1)

    # Uncomment to get verbose cURL output including SSL negotiation
    # curl.setopt(curl.VERBOSE, 1)

    # TODO: read timeout values from config?
    # We can not allow the server to block for very long

    curl.setopt(pycurl.CONNECTTIMEOUT, 5)
    curl.setopt(pycurl.TIMEOUT, 10)
    if protocol == 'https':
        curl.setopt(curl.SSLCERT, config.server_cert)
        curl.setopt(curl.SSLKEY, config.server_key)
        if passphrase:
            curl.setopt(curl.SSLKEYPASSWD, passphrase)

        # Path to CA certificates

        # To use NorduGRID default certificate path set:
        # curl.setopt(curl.CAPATH, "/etc/grid-security/certificates")

        if config.ca_dir:
            curl.setopt(curl.CAPATH, config.ca_dir)
        elif config.ca_file:

            # We use our own demo CA file specified in the configuration for now

            curl.setopt(curl.CAINFO, config.ca_file)

        # Workaround for broken host certificates:
        # ###################################################
        # Do not use this, but fix host cert + CA instead! #
        # ###################################################
        # VERIFYHOST should be 2 (default) unless remote cert can not be
        # verified using CA cert.
        # curl.setopt(curl.SSL_VERIFYHOST,1)
        # Similarly VERIFYPEER will then probably need to be set to 0
        # curl.setopt(curl.SSL_VERIFYPEER,0)

    # Clean up after cURL

    try:
        config.logger.info('get_data: fetch %s', url)
        curl.perform()
    except pycurl.error as e:

        # pycurl.error is an (errorcode, errormsg) tuple

        config.logger.error('cURL command failed! %s', e[1])
        return ''

    http_status = curl.getinfo(pycurl.HTTP_CODE)

    curl.close()
    pycurl.global_cleanup()

    server_status = ConfigParser.ConfigParser()

    if http_status == http_success:

        # Go to start of buffer

        output.seek(0)
        try:
            server_status.readfp(output)
        except:
            config.logger.error('Failed to parse server status')
            return None
    else:
        config.logger.error('Server returned HTTP code %d, expected %d',
                            http_status, http_success)
        return None

    output.close()

    return server_status
示例#30
0
def PutFile():
    srcdir = "."
    filename = "testfile"
    protocol = "https"
    host = "mig-1.imada.sdu.dk"
    port = ""

    filepath = srcdir + "/" + filename
    try:
        inputfile = open(filepath, "rb")
    except:
        print "Error: Failed to open %s for reading!" % filepath
        return (False, "Invalid filename!")

    # Set size of file to be uploaded.

    size = os.path.getsize(filepath)

    if port:
        url = "%s://%s:%s/%s" % (protocol, host, port, filename)
    else:
        url = "%s://%s/%s" % (protocol, host, filename)

    # TODO: change to 'real' server certs
    # Just using default NorduGRID certificates for now

    os.environ["HOME"] = pwd.getpwuid(os.geteuid())[5]
    globus_dir = os.path.expanduser("~/.globus")
    cert_dir = globus_dir
    server_cert = cert_dir + "/usercert.pem"
    server_key = cert_dir + "/userkey.pem"
    passwd_file = "cert_pass"

    passwd = ""
    try:
        pw_file = open(passwd_file, "r")
        passwd = pw_file.readline().strip()
        pw_file.close()
    except:
        print "Failed to read password from file!"
        return ""

    # Store output in memory

    output = StringIO.StringIO()

    # Init cURL (not strictly necessary, but for symmetry with cleanup)

    pycurl.global_init(pycurl.GLOBAL_SSL)

    curl = pycurl.Curl()
    curl.setopt(pycurl.HTTPHEADER, ["User-Agent: MiG HTTP PUT"])
    curl.setopt(pycurl.PUT, 1)
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.MAXREDIRS, 5)
    curl.setopt(pycurl.URL, url)
    curl.setopt(pycurl.WRITEFUNCTION, output.write)
    curl.setopt(pycurl.NOSIGNAL, 1)

    # Uncomment to get verbose cURL output including SSL negotiation

    curl.setopt(curl.VERBOSE, 1)
    curl.setopt(pycurl.CONNECTTIMEOUT, 30)
    curl.setopt(pycurl.TIMEOUT, 300)

    # curl.setopt(curl.PORT, port)

    curl.setopt(pycurl.INFILE, inputfile)
    curl.setopt(pycurl.INFILESIZE, size)

    if protocol == "https":
        curl.setopt(curl.SSLCERT, server_cert)
        curl.setopt(curl.SSLKEY, server_key)
        if passwd:
            curl.setopt(curl.SSLKEYPASSWD, passwd)

        # Path to CA certificates (NorduGRID default certificate path)

        curl.setopt(curl.CAPATH, "/etc/grid-security/certificates")

        # TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)

        curl.setopt(curl.SSL_VERIFYHOST, 1)

        # Uncomment if server identity can't be verified from local hostcert or CA cert

        curl.setopt(curl.SSL_VERIFYPEER, 0)

    # TODO: uncomment the following to actually execute upload

    try:
        curl.perform()
    except pycurl.error, e:

        # pycurl.error is an (errorcode, errormsg) tuple

        print "Error: cURL command failed! %s" % e[1]
        return (404, "Error!")
示例#31
0
from urlparse import urljoin
import simplejson as json

from opendiamond.helpers import murmur, split_scheme
from opendiamond.protocol import XDR_attribute, XDR_object

ATTR_HEADER_URL = 'x-attributes'
ATTR_HEADER_PREFIX = 'x-attr-'
# Object attributes handled directly by the server
ATTR_DATA = ''
ATTR_OBJ_ID = '_ObjectID'
ATTR_DISPLAY_NAME = 'Display-Name'
ATTR_DEVICE_NAME = 'Device-Name'

# Initialize curl before multiple threads have been started
curl.global_init(curl.GLOBAL_DEFAULT)


class ObjectLoadError(Exception):
    '''Object failed to load.'''


class EmptyObject(object):
    '''An immutable Diamond object with no data and no attributes.'''

    def __init__(self):
        self._attrs = dict()
        self._signatures = dict()
        self._omit_attrs = set()

    def __str__(self):
示例#32
0
def GetFile():

    # Defaults

    url = "cgi-bin/ls.py"
    base_dir = "."

    # Just using default NorduGRID certificates for now

    os.environ["HOME"] = pwd.getpwuid(os.geteuid())[5]
    globus_dir = os.path.expanduser("~/.globus")
    cert_dir = globus_dir
    server_cert = cert_dir + "/usercert.pem"
    server_key = cert_dir + "/userkey.pem"
    passwd = ""
    MiGServer = "https://mig-1.imada.sdu.dk"
    port = "8092"

    if len(sys.argv) > 1:
        passwd = sys.argv[1]

    data = StringIO.StringIO()

    # Init cURL (not strictly necessary, but for symmetry with cleanup)

    pycurl.global_init(pycurl.GLOBAL_SSL)
    print "cURL:\t\t", pycurl.version
    curl = pycurl.Curl()
    curl.setopt(pycurl.HTTPHEADER, ["User-Agent: MiG HTTP GET"])
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.MAXREDIRS, 5)
    curl.setopt(pycurl.URL, MiGServer + ":" + port + "/" + url)
    curl.setopt(pycurl.WRITEFUNCTION, data.write)
    curl.setopt(pycurl.NOSIGNAL, 1)

    # Uncomment to get verbose cURL output including SSL negotiation

    curl.setopt(curl.VERBOSE, 1)
    curl.setopt(pycurl.CONNECTTIMEOUT, 30)
    curl.setopt(pycurl.TIMEOUT, 300)

    # curl.setopt(curl.PORT, port)

    curl.setopt(curl.SSLCERT, server_cert)
    curl.setopt(curl.SSLKEY, server_key)
    if passwd:
        curl.setopt(curl.SSLKEYPASSWD, passwd)

    # Path to CA certificates (NorduGRID default certificate path)

    curl.setopt(curl.CAPATH, "/etc/grid-security/certificates")

    # TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)

    curl.setopt(curl.SSL_VERIFYHOST, 1)

    # Uncomment if server identity can't be verified from local hostcert or CA cert

    curl.setopt(curl.SSL_VERIFYPEER, 0)

    print "fetching:\t", url
    print "cert:\t\t", server_cert
    print "key:\t\t", server_key
    print "passwd:\t\t", passwd

    # Clean up after cURL

    try:
        curl.perform()
    except pycurl.error, e:
        print "cURL command failed!:"

        # error is a (errorcode, errormsg) tuple

        print e[1]
        return False
示例#33
0
def get_data(
    protocol,
    host,
    port,
    rel_path,
    cert,
    key,
    ca_dir,
    ca_file,
    passphrase_file='',
    ):

    if port:
        url = '%s://%s:%s/%s' % (protocol, host, port, rel_path)
    else:
        url = '%s://%s/%s' % (protocol, host, rel_path)

    passphrase = ''
    if passphrase_file:
        try:
            pp_file = open(passphrase_file, 'r')
            passphrase = pp_file.readline().strip()
            pp_file.close()
        except:
            print 'Failed to read passprase from %s', passphrase_file
            return None

    # Store output in memory

    output = StringIO.StringIO()

    # Init cURL (not strictly necessary, but for symmetry with cleanup)

    pycurl.global_init(pycurl.GLOBAL_SSL)

    curl = pycurl.Curl()
    curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP GET'])
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.MAXREDIRS, 5)
    curl.setopt(pycurl.URL, url)
    curl.setopt(pycurl.WRITEFUNCTION, output.write)
    curl.setopt(pycurl.NOSIGNAL, 1)

    # Uncomment to get verbose cURL output including SSL negotiation

    curl.setopt(curl.VERBOSE, 1)
    curl.setopt(pycurl.CONNECTTIMEOUT, 30)
    curl.setopt(pycurl.TIMEOUT, 300)
    if protocol == 'https':
        curl.setopt(curl.SSLCERT, cert)
        curl.setopt(curl.SSLKEY, key)
        if passphrase:
            curl.setopt(curl.SSLKEYPASSWD, passphrase)

        # Path to CA certificates

        if ca_dir:
            curl.setopt(curl.CAPATH, ca_dir)
        elif ca_file:

            # We use our own demo CA file specified in the configuration for now

            curl.setopt(curl.CAINFO, ca_file)

        # Workaround for broken host certificates:
        # ###################################################
        # Do not use this, but fix host cert + CA instead! #
        # ###################################################
        # VERIFYHOST should be 2 (default) unless remote cert can not be
        # verified using CA cert.
        # curl.setopt(curl.SSL_VERIFYHOST,1)
        # Similarly VERIFYPEER will then probably need to be set to 0
        # curl.setopt(curl.SSL_VERIFYPEER,0)

        # TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)

        curl.setopt(curl.SSL_VERIFYHOST, 1)

        # Uncomment if server identity can't be verified from local hostcert or CA cert

        curl.setopt(curl.SSL_VERIFYPEER, 0)

    try:
        print 'get_data: fetch %s', url
        curl.perform()
    except pycurl.error, e:

        # pycurl.error is an (errorcode, errormsg) tuple

        print 'cURL command failed! %s', e[1]
        return ''
示例#34
0
def GetFile():

    # Defaults

    url = 'cgi-bin/ls.py'
    base_dir = '.'

    # Just using default NorduGRID certificates for now

    os.environ['HOME'] = pwd.getpwuid(os.geteuid())[5]
    globus_dir = os.path.expanduser('~/.globus')
    cert_dir = globus_dir
    server_cert = cert_dir + '/usercert.pem'
    server_key = cert_dir + '/userkey.pem'
    passwd = ''
    MiGServer = 'https://mig-1.imada.sdu.dk'
    port = '8092'

    if len(sys.argv) > 1:
        passwd = sys.argv[1]

    data = StringIO.StringIO()

    # Init cURL (not strictly necessary, but for symmetry with cleanup)

    pycurl.global_init(pycurl.GLOBAL_SSL)
    print 'cURL:\t\t', pycurl.version
    curl = pycurl.Curl()
    curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP GET'])
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.MAXREDIRS, 5)
    curl.setopt(pycurl.URL, MiGServer + ':' + port + '/' + url)
    curl.setopt(pycurl.WRITEFUNCTION, data.write)
    curl.setopt(pycurl.NOSIGNAL, 1)

    # Uncomment to get verbose cURL output including SSL negotiation

    curl.setopt(curl.VERBOSE, 1)
    curl.setopt(pycurl.CONNECTTIMEOUT, 30)
    curl.setopt(pycurl.TIMEOUT, 300)

    # curl.setopt(curl.PORT, port)

    curl.setopt(curl.SSLCERT, server_cert)
    curl.setopt(curl.SSLKEY, server_key)
    if passwd:
        curl.setopt(curl.SSLKEYPASSWD, passwd)

    # Path to CA certificates (NorduGRID default certificate path)

    curl.setopt(curl.CAPATH, '/etc/grid-security/certificates')

    # TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)

    curl.setopt(curl.SSL_VERIFYHOST, 1)

    # Uncomment if server identity can't be verified from local hostcert or CA cert

    curl.setopt(curl.SSL_VERIFYPEER, 0)

    print 'fetching:\t', url
    print 'cert:\t\t', server_cert
    print 'key:\t\t', server_key
    print 'passwd:\t\t', passwd

    # Clean up after cURL

    try:
        curl.perform()
    except pycurl.error, e:
        print 'cURL command failed!:'

        # error is a (errorcode, errormsg) tuple

        print e[1]
        return False
示例#35
0
def main():
	signal.signal(signal.SIGHUP, signal.SIG_IGN)
	usage = "usage: %prog [options]"
	parser = OptionParser(usage=usage)
	
	#	parser.add_option('-q', '--quiet', action="store_const", const=0, dest="v", default=1, help='quiet')
	
	parser.add_option('-c', '--config', action='store', dest='config', default=DEFAULT_CONF, help='load parameters from configfile (default: ' + DEFAULT_CONF + ')')
	
	parser.add_option('-t', '--tcp', action='store_const', dest='mode', const='tcp', help='tcp mode (default)')
	
	parser.add_option('-u', '--udp', action='store_const', dest='mode', const='udp', help='udp mode')
	
	parser.add_option('-L', action='append', dest='forward', help='forward port:remotehost:remoteport (like ssh)')
		
	parser.add_option('--url', action="store", dest="url", help='URL of tunnelendpoint')
	
	parser.add_option('--proxy', action='store', dest='proxy', help='proxy to use')
	
	parser.add_option('--auth', action='store', dest='auth', help='auth with user:password')
	
	parser.add_option('-a', '--agent', action='store', dest='agent', help='fake useragent')
	
	parser.add_option('-v', '--verbose', action='store_const', dest='verbose', const=1, help='verbose')
	
	
	parser.add_option('--no-verify-ssl', action='store_true', dest='nv', help='do not verify ssl-host')
	parser.add_option('--verify-ssl', action='store_false', dest='nv', help='do not verify ssl-host')
	
	global options
	(options, args) = parser.parse_args()
		
	cparser = ConfigParser.ConfigParser(defaults={
		'mode': 'tcp',
		'url': DEFAULT_URL,
		'auth': '',
		'proxy': '',
		'agent': '',
		'verbose': 0,
		'verify': True
		})

	cparser.read(options.config)
	
	if cparser.has_section('pyhstopc'):
		if not options.url:	options.url = cparser.get('pyhstopc', 'url')
		if not options.auth:	options.auth = cparser.get('pyhstopc', 'auth')
		if not options.agent:	options.agent = cparser.get('pyhstopc', 'agent')
		if not options.proxy:	options.proxy = cparser.get('pyhstopc', 'proxy')
		if not options.forward:
			options.forward = []
			try:
				options.forward.extend(cparser.get('pyhstopc', 'forward').split(','))
			except ConfigParser.NoOptionError:
				pass
		try:
			if not options.verbose:	options.verbose = cparser.getint('pyhstopc', 'verbose')
		except TypeError:
			options.verbose = 0
		try:
			if options.nv == None:	options.nv = not cparser.getboolean('pyhstopc', 'verify')
		except TypeError:
			options.nv = False
	
	cparser = None
	
	tmpforward = options.forward
	options.forward = []
	for i in tmpforward:
		try:
			lport, rhost, rport = i.split(':')
			options.forward.append((int(lport.strip()), rhost.strip(), int(rport.strip()),'tcp'))
		except (KeyError, ValueError):
			try:
				lport, rhost, rport, mode = i.split(':')
				options.forward.append((int(lport.strip()), rhost.strip(), int(rport.strip()), mode))
			except (KeyError, ValueError):
				print 'malformed forward option: ', i
	
	print 'pyhstopc Version: ' + VERSION
	print 'terminate with EOF'

	print 'start..'
	
	pycurl.global_init(pycurl.GLOBAL_ALL)
	
	sls = []
	for i in range(len(options.forward)):
		sl = socketListener(i)
		sl.listen()
		
	try:
		input = sys.stdin.readline()
		while input:
			input = sys.stdin.readline()
	except KeyboardInterrupt:
		print 'interrupted'
	
	for sl in sls:
		sl.terminate()
	
	pycurl.global_cleanup()
	print 'end..'
示例#36
0
 def test_global_init_bogus(self):
     # initialize libcurl with bogus flags
     with pytest.raises(ValueError):
         pycurl.global_init(0xffff)
示例#37
0
import os

from io import BytesIO

DEFAULT_OPTIONS = {
    pycurl.CAINFO: certifi.where(),
    pycurl.FOLLOWLOCATION: 1,
    pycurl.MAXREDIRS: 5,
    pycurl.SSL_VERIFYPEER: 1,
    pycurl.SSL_VERIFYHOST: 2
}

# global inits
os.environ.update({'SSLKEYLOGFILE': './sslkeys.log'})
atexit.register(pycurl.global_cleanup)
pycurl.global_init(pycurl.GLOBAL_SSL)


class Session:
    ''' Session class for api requests based on pycurl '''
    def __init__(self):
        self._session = pycurl.Curl()
        self._options = {}
        self.set_options(DEFAULT_OPTIONS)

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self._session.close()
示例#38
0
 def test_global_init_default(self):
     # initialize libcurl with DEFAULT flags
     pycurl.global_init(pycurl.GLOBAL_DEFAULT)
     pycurl.global_cleanup()
示例#39
0
文件: pycurl.py 项目: blarsson/holon
 def __init__(self, *args, **kwargs):
     super(PyCurlHttpService, self).__init__(*args, **kwargs)
     pycurl.global_init(pycurl.GLOBAL_ALL)
示例#40
0
 def test_global_init_bogus(self):
     # initialize libcurl with bogus flags
     pycurl.global_init(0xffff)
示例#41
0
def init_libcurl():
    pycurl.global_init(pycurl.GLOBAL_ALL)
示例#42
0
def main():
	signal.signal(signal.SIGHUP, signal.SIG_IGN)
	usage = "usage: %prog [options]"
	parser = OptionParser(usage=usage)
	
	#	parser.add_option('-q', '--quiet', action="store_const", const=0, dest="v", default=1, help='quiet')
	
	parser.add_option('-c', '--config', action='store', dest='config', default=DEFAULT_CONF, help='load parameters from configfile (default: ' + DEFAULT_CONF + ')')
	
	parser.add_option('-t', '--tcp', action='store_const', dest='mode', const='tcp', help='tcp mode (default)')
	
	parser.add_option('-u', '--udp', action='store_const', dest='mode', const='udp', help='udp mode')
	
	parser.add_option('-L', action='append', dest='forward', help='forward port:remotehost:remoteport (like ssh)')
		
	parser.add_option('--url', action="store", dest="url", help='URL of tunnelendpoint')
	
	parser.add_option('--proxy', action='store', dest='proxy', help='proxy to use')
	
	parser.add_option('--auth', action='store', dest='auth', help='auth with user:password')
	
	parser.add_option('-a', '--agent', action='store', dest='agent', help='fake useragent')
	
	parser.add_option('-v', '--verbose', action='store_const', dest='verbose', const=1, help='verbose')
	
	
	parser.add_option('--no-verify-ssl', action='store_true', dest='nv', help='do not verify ssl-host')
	parser.add_option('--verify-ssl', action='store_false', dest='nv', help='do not verify ssl-host')
	
	global options
	(options, args) = parser.parse_args()
		
	cparser = ConfigParser.ConfigParser(defaults={
		'mode': 'tcp',
		'url': DEFAULT_URL,
		'auth': '',
		'proxy': '',
		'agent': '',
		'verbose': 0,
		'verify': True
		})

	cparser.read(options.config)
	
	if cparser.has_section('pyhstopc'):
		if not options.url:	options.url = cparser.get('pyhstopc', 'url')
		if not options.auth:	options.auth = cparser.get('pyhstopc', 'auth')
		if not options.agent:	options.agent = cparser.get('pyhstopc', 'agent')
		if not options.proxy:	options.proxy = cparser.get('pyhstopc', 'proxy')
		if not options.forward:
			options.forward = []
			try:
				options.forward.extend(cparser.get('pyhstopc', 'forward').split(','))
			except ConfigParser.NoOptionError:
				pass
		try:
			if not options.verbose:	options.verbose = cparser.getint('pyhstopc', 'verbose')
		except TypeError:
			options.verbose = 0
		try:
			if options.nv == None:	options.nv = not cparser.getboolean('pyhstopc', 'verify')
		except TypeError:
			options.nv = False
	
	cparser = None
	
	tmpforward = options.forward
	options.forward = []
	for i in tmpforward:
		try:
			lport, rhost, rport = i.split(':')
			options.forward.append((int(lport.strip()), rhost.strip(), int(rport.strip()),'tcp'))
		except (KeyError, ValueError):
			try:
				lport, rhost, rport, mode = i.split(':')
				options.forward.append((int(lport.strip()), rhost.strip(), int(rport.strip()), mode))
			except (KeyError, ValueError):
				print 'malformed forward option: ', i
	
	print 'pyhstopc Version: ' + VERSION
	print 'terminate with EOF'

	print 'start..'
	
	pycurl.global_init(pycurl.GLOBAL_ALL)
	
	sls = []
	for i in range(len(options.forward)):
		sl = socketListener(i)
		sl.listen()
		
	try:
		input = sys.stdin.readline()
		while input:
			input = sys.stdin.readline()
	except KeyboardInterrupt:
		print 'interrupted'
	
	for sl in sls:
		sl.terminate()
	
	pycurl.global_cleanup()
	print 'end..'
示例#43
0
def put_data(
    config,
    filename,
    protocol,
    host,
    port,
    rel_path,
    ):

    try:
        inputfile = open(filename, 'rb')
    except:
        config.logger.error('Failed to open %s for reading!', filename)
        return (False, 'Invalid filename!')

    # Set size of file to be uploaded.

    size = os.path.getsize(filename)

    if port:
        url = '%s://%s:%s/%s' % (protocol, host, port, rel_path)
    else:
        url = '%s://%s/%s' % (protocol, host, rel_path)

    passphrase = ''
    try:
        pp_file = open(config.passphrase_file, 'r')
        passphrase = pp_file.readline().strip()
        pp_file.close()
    except:
        config.logger.error('Failed to read passphrase from file %s',
                            config.passphrase_file)
        return (-1, 'Failed to read passphrase from file')

    # Store output in memory

    output = StringIO.StringIO()

    # Init cURL (not strictly necessary, but for symmetry with cleanup)

    pycurl.global_init(pycurl.GLOBAL_SSL)

    curl = pycurl.Curl()
    curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP PUT'])
    curl.setopt(pycurl.PUT, 1)
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.MAXREDIRS, 5)
    curl.setopt(pycurl.URL, url)
    curl.setopt(pycurl.WRITEFUNCTION, output.write)
    curl.setopt(pycurl.NOSIGNAL, 1)

    # Uncomment to get verbose cURL output including SSL negotiation
    # curl.setopt(curl.VERBOSE, 1)

    # We can not let the server block for very long

    curl.setopt(pycurl.CONNECTTIMEOUT, 5)
    curl.setopt(pycurl.TIMEOUT, 10)

    curl.setopt(pycurl.INFILE, inputfile)
    curl.setopt(pycurl.INFILESIZE, size)

    if protocol == 'https':
        curl.setopt(curl.SSLCERT, config.server_cert)
        curl.setopt(curl.SSLKEY, config.server_key)
        if passphrase:
            curl.setopt(curl.SSLKEYPASSWD, passphrase)

        # Path to CA certificates

        # To use NorduGRID default certificate path set:
        # curl.setopt(curl.CAPATH, "/etc/grid-security/certificates")

        if config.ca_dir:
            curl.setopt(curl.CAPATH, config.ca_dir)
        elif config.ca_file:

            # We use our own demo CA file specified in the configuration for now

            curl.setopt(curl.CAINFO, config.ca_file)

        # Workaround for broken host certificates:
        # ###################################################
        # Do not use this, but fix host cert + CA instead! #
        # ###################################################
        # VERIFYHOST should be 2 (default) unless remote cert can not be
        # verified using CA cert.
        # curl.setopt(curl.SSL_VERIFYHOST,1)
        # Similarly VERIFYPEER will then probably need to be set to 0
        # curl.setopt(curl.SSL_VERIFYPEER,0)

    try:
        curl.perform()
    except pycurl.error, e:

        # pycurl.error is an (errorcode, errormsg) tuple

        config.logger.error('cURL command failed! %s', e[1])
        return (404, 'Error!')
示例#44
0
def init_libcurl():
    pycurl.global_init(pycurl.GLOBAL_ALL)
示例#45
0
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

import cPickle
import zlib
import json

# Pycurl part
import pycurl
pycurl.global_init(pycurl.GLOBAL_ALL)
import urllib
from StringIO import StringIO

from shinken.bin import VERSION
from shinken.log import logger
PYCURL_VERSION = pycurl.version_info()[1]

class HTTPException(Exception):
    pass


HTTPExceptions = (HTTPException,)

class FileReader:
    def __init__(self, fp):
示例#46
0
def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "", ["port=", "test=", "proxy=", "cwd=", "secondary-proxy-port="])
    except getopt.GetoptError as err:
        print str(err)
        sys.exit(2)
    test = None
    proxy_port = 12345
    proxy_executable = None
    proxy_cwd = None
    secondary_proxy_port = None
    for o, a in opts:
        if o == "--port":
            proxy_port = int(a)
        elif o == "--test":
            test = a
        elif o ==  "--proxy":
            proxy_executable = a
        elif o == "--cwd":
            proxy_cwd = a
        elif o == "--secondary-proxy-port":
            secondary_proxy_port = int(a)
        else:
            assert False, "unhandled option"
    assert test is not None, "test must be specified"
    assert proxy_executable is not None, "proxy executable must be specified"
    
    using_proxy_chaining = (secondary_proxy_port is not None)
    if test == "proxy-cycle":
        assert using_proxy_chaining, "The secondary-proxy-port flag is required for the proxy-cycle test"

    cache_dir = os.path.expanduser('~/.proxy-cache-' + socket.gethostname())
    shutil.rmtree(cache_dir, ignore_errors=True)
    
    # Get a place to store the stdout/stderr
    log_file = tempfile.TemporaryFile()
    
    try:
        if not using_proxy_chaining:
            proxy = subprocess.Popen([proxy_executable, "--port", str(proxy_port)], cwd=proxy_cwd, stdout=log_file, stderr=subprocess.STDOUT)
        else:
            proxy = subprocess.Popen([proxy_executable, "--port", str(proxy_port), "--proxy-server", "localhost", "--proxy-port", str(secondary_proxy_port)], cwd=proxy_cwd, stdout=log_file, stderr=subprocess.STDOUT)
    except:
        print("Couldn't start proxy")
        return

    if using_proxy_chaining:
        try:
            secondary_proxy = subprocess.Popen([proxy_executable, "--port", str(secondary_proxy_port), "--proxy-server", "localhost", "--proxy-port", str(proxy_port)], cwd=proxy_cwd, stdout=log_file, stderr=subprocess.STDOUT)
            #secondary_proxy = subprocess.Popen([proxy_executable, "--port", str(secondary_proxy_port)], cwd=proxy_cwd, stdout=log_file, stderr=subprocess.STDOUT)
        except:
            print("Couldn't start secondary proxy")
            return
    #elif test == 'proxy-chain-basic':
    #    try:
    #        exec_command = "ssh " + os.environ["USER"] + "@myth12.stanford.edu 'cd /usr/class/cs110/staff/bin/master-repos/assign6-soln/; " + proxy_executable + " --port 56565'" 
    #        print exec_command
    #        secondary_proxy = subprocess.Popen(exec_command, cwd=proxy_cwd, stdout=log_file, stderr=subprocess.STDOUT)
    #    except:
    #        print("Couldn't start secondary proxy")
    #        return
   

    # try:
    time.sleep(5) # Make sure the proxy is running
    pycurl.global_init(pycurl.GLOBAL_DEFAULT)
    test_runner = ProxyTest(proxy_port)
    test_runner.run_test(test)
    pycurl.global_cleanup()

    return_code = proxy.poll() # Get the return status if it already exited (crashed)

    if return_code is None: # Hopefully the proxy was still running
        proxy.terminate()
        proxy.wait()
    else:
        log_file.seek(0)
        print("Proxy crashed; return code: %d, output:" % return_code)
        print(log_file.read())

    if using_proxy_chaining:
        secondary_return_code = secondary_proxy.poll();
        if secondary_return_code is None:
            secondary_proxy.terminate()
            secondary_proxy.wait()

    # except: # Make sure the proxy is dead
    #     print("Something went wrong. Killing the proxy.")
    #     proxy.kill()
    #     proxy.wait()
    #     raise
    # finally:
    shutil.rmtree(cache_dir, ignore_errors=True) # Clean up
示例#47
0
 def __init__(self, pool, cr):
     super(taobao_shop, self).__init__(pool, cr)
     #pycurl两个函数不是线程安全。所以在主线程中进行一次的初始化和清除
     import pycurl
     pycurl.global_init(pycurl.GLOBAL_DEFAULT)
     pycurl.global_cleanup()
示例#48
0
 def test_global_init_default(self):
     # initialize libcurl with DEFAULT flags
     pycurl.global_init(pycurl.GLOBAL_DEFAULT)
     pycurl.global_cleanup()
示例#49
0
def PutFile():
    srcdir = '.'
    filename = 'testfile'
    protocol = 'https'
    host = 'mig-1.imada.sdu.dk'
    port = ''

    filepath = srcdir + '/' + filename
    try:
        inputfile = open(filepath, 'rb')
    except:
        print 'Error: Failed to open %s for reading!' % filepath
        return (False, 'Invalid filename!')

    # Set size of file to be uploaded.

    size = os.path.getsize(filepath)

    if port:
        url = '%s://%s:%s/%s' % (protocol, host, port, filename)
    else:
        url = '%s://%s/%s' % (protocol, host, filename)

    # TODO: change to 'real' server certs
    # Just using default NorduGRID certificates for now

    os.environ['HOME'] = pwd.getpwuid(os.geteuid())[5]
    globus_dir = os.path.expanduser('~/.globus')
    cert_dir = globus_dir
    server_cert = cert_dir + '/usercert.pem'
    server_key = cert_dir + '/userkey.pem'
    passwd_file = 'cert_pass'

    passwd = ''
    try:
        pw_file = open(passwd_file, 'r')
        passwd = pw_file.readline().strip()
        pw_file.close()
    except:
        print 'Failed to read password from file!'
        return ''

    # Store output in memory

    output = StringIO.StringIO()

    # Init cURL (not strictly necessary, but for symmetry with cleanup)

    pycurl.global_init(pycurl.GLOBAL_SSL)

    curl = pycurl.Curl()
    curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP PUT'])
    curl.setopt(pycurl.PUT, 1)
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.MAXREDIRS, 5)
    curl.setopt(pycurl.URL, url)
    curl.setopt(pycurl.WRITEFUNCTION, output.write)
    curl.setopt(pycurl.NOSIGNAL, 1)

    # Uncomment to get verbose cURL output including SSL negotiation

    curl.setopt(curl.VERBOSE, 1)
    curl.setopt(pycurl.CONNECTTIMEOUT, 30)
    curl.setopt(pycurl.TIMEOUT, 300)

    # curl.setopt(curl.PORT, port)

    curl.setopt(pycurl.INFILE, inputfile)
    curl.setopt(pycurl.INFILESIZE, size)

    if protocol == 'https':
        curl.setopt(curl.SSLCERT, server_cert)
        curl.setopt(curl.SSLKEY, server_key)
        if passwd:
            curl.setopt(curl.SSLKEYPASSWD, passwd)

        # Path to CA certificates (NorduGRID default certificate path)

        curl.setopt(curl.CAPATH, '/etc/grid-security/certificates')

        # TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)

        curl.setopt(curl.SSL_VERIFYHOST, 1)

        # Uncomment if server identity can't be verified from local hostcert or CA cert

        curl.setopt(curl.SSL_VERIFYPEER, 0)

    # TODO: uncomment the following to actually execute upload

    try:
        curl.perform()
    except pycurl.error, e:

        # pycurl.error is an (errorcode, errormsg) tuple

        print 'Error: cURL command failed! %s' % e[1]
        return (404, 'Error!')
示例#50
0
 def test_global_init_bogus(self):
     # initialize libcurl with bogus flags
     pycurl.global_init(0xffff)
示例#51
0
def put_data(
    config,
    filename,
    protocol,
    host,
    port,
    rel_path,
):

    try:
        inputfile = open(filename, 'rb')
    except:
        config.logger.error('Failed to open %s for reading!', filename)
        return (False, 'Invalid filename!')

    # Set size of file to be uploaded.

    size = os.path.getsize(filename)

    if port:
        url = '%s://%s:%s/%s' % (protocol, host, port, rel_path)
    else:
        url = '%s://%s/%s' % (protocol, host, rel_path)

    passphrase = ''
    try:
        pp_file = open(config.passphrase_file, 'r')
        passphrase = pp_file.readline().strip()
        pp_file.close()
    except:
        config.logger.error('Failed to read passphrase from file %s',
                            config.passphrase_file)
        return (-1, 'Failed to read passphrase from file')

    # Store output in memory

    output = StringIO.StringIO()

    # Init cURL (not strictly necessary, but for symmetry with cleanup)

    pycurl.global_init(pycurl.GLOBAL_SSL)

    curl = pycurl.Curl()
    # Never use proxy
    curl.setopt(pycurl.PROXY, "")
    curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP PUT'])
    curl.setopt(pycurl.PUT, 1)
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.MAXREDIRS, 5)
    curl.setopt(pycurl.URL, url)
    curl.setopt(pycurl.WRITEFUNCTION, output.write)
    curl.setopt(pycurl.NOSIGNAL, 1)

    # Uncomment to get verbose cURL output including SSL negotiation
    # curl.setopt(curl.VERBOSE, 1)

    # We can not let the server block for very long

    curl.setopt(pycurl.CONNECTTIMEOUT, 5)
    curl.setopt(pycurl.TIMEOUT, 10)

    curl.setopt(pycurl.INFILE, inputfile)
    curl.setopt(pycurl.INFILESIZE, size)

    if protocol == 'https':
        curl.setopt(curl.SSLCERT, config.server_cert)
        curl.setopt(curl.SSLKEY, config.server_key)
        if passphrase:
            curl.setopt(curl.SSLKEYPASSWD, passphrase)

        # Path to CA certificates

        # To use NorduGRID default certificate path set:
        # curl.setopt(curl.CAPATH, "/etc/grid-security/certificates")

        if config.ca_dir:
            curl.setopt(curl.CAPATH, config.ca_dir)
        elif config.ca_file:

            # We use our own demo CA file specified in the configuration for now

            curl.setopt(curl.CAINFO, config.ca_file)

        # Workaround for broken host certificates:
        # ###################################################
        # Do not use this, but fix host cert + CA instead! #
        # ###################################################
        # VERIFYHOST should be 2 (default) unless remote cert can not be
        # verified using CA cert.
        # curl.setopt(curl.SSL_VERIFYHOST,1)
        # Similarly VERIFYPEER will then probably need to be set to 0
        # curl.setopt(curl.SSL_VERIFYPEER,0)

    try:
        curl.perform()
    except pycurl.error as e:

        # pycurl.error is an (errorcode, errormsg) tuple

        config.logger.error('cURL command failed! %s', e[1])
        return (404, 'Error!')

    http_status = curl.getinfo(pycurl.HTTP_CODE)

    # Clean up after cURL

    curl.close()
    pycurl.global_cleanup()

    if http_status == http_success:
        config.logger.info('PUT request succeeded')

        # Go to start of buffer

        output.seek(0)
        msg = output.readlines()
    else:

        # print msg

        config.logger.warning('Server returned HTTP code %d, expected %d',
                              http_status, http_success)

    inputfile.close()
    output.close()

    return (http_status, 'Success!')
示例#52
0
            return response

    @classmethod
    def getServerInfo(c):
        """
        Return the information needed to send API messages to the server.

        This can be used by an external program (e.g. pdinstall).
        """
        info = {
            'authorization': 'Bearer {}'.format(c.token),
            'router_id': nexus.core.info.pdid,
            'server': nexus.core.info.pdserver
        }
        return info

    @classmethod
    def resetToken(c):
        """
        Reset the auth token, to be called if the router's identity has changed.
        """
        c.token = None


# Initialize pycurl.  Does this do anything?
pycurl.global_init(pycurl.GLOBAL_ALL)

# Set the number of connections that can be kept alive in the connection pool.
# Setting this equal to the size of the semaphore should prevent churn.
TwistedRequestDriver.pool.maxPersistentPerHost = settings.PDSERVER_MAX_CONCURRENT_REQUESTS
示例#53
0
def get_data(
    protocol,
    host,
    port,
    rel_path,
    cert,
    key,
    ca_dir,
    ca_file,
    passphrase_file='',
):

    if port:
        url = '%s://%s:%s/%s' % (protocol, host, port, rel_path)
    else:
        url = '%s://%s/%s' % (protocol, host, rel_path)

    passphrase = ''
    if passphrase_file:
        try:
            pp_file = open(passphrase_file, 'r')
            passphrase = pp_file.readline().strip()
            pp_file.close()
        except:
            print('Failed to read passprase from %s', passphrase_file)
            return None

    # Store output in memory

    output = StringIO.StringIO()

    # Init cURL (not strictly necessary, but for symmetry with cleanup)

    pycurl.global_init(pycurl.GLOBAL_SSL)

    curl = pycurl.Curl()
    curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP GET'])
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.MAXREDIRS, 5)
    curl.setopt(pycurl.URL, url)
    curl.setopt(pycurl.WRITEFUNCTION, output.write)
    curl.setopt(pycurl.NOSIGNAL, 1)

    # Uncomment to get verbose cURL output including SSL negotiation

    curl.setopt(curl.VERBOSE, 1)
    curl.setopt(pycurl.CONNECTTIMEOUT, 30)
    curl.setopt(pycurl.TIMEOUT, 300)
    if protocol == 'https':
        curl.setopt(curl.SSLCERT, cert)
        curl.setopt(curl.SSLKEY, key)
        if passphrase:
            curl.setopt(curl.SSLKEYPASSWD, passphrase)

        # Path to CA certificates

        if ca_dir:
            curl.setopt(curl.CAPATH, ca_dir)
        elif ca_file:

            # We use our own demo CA file specified in the configuration for now

            curl.setopt(curl.CAINFO, ca_file)

        # Workaround for broken host certificates:
        # ###################################################
        # Do not use this, but fix host cert + CA instead! #
        # ###################################################
        # VERIFYHOST should be 2 (default) unless remote cert can not be
        # verified using CA cert.
        # curl.setopt(curl.SSL_VERIFYHOST,1)
        # Similarly VERIFYPEER will then probably need to be set to 0
        # curl.setopt(curl.SSL_VERIFYPEER,0)

        # TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)

        curl.setopt(curl.SSL_VERIFYHOST, 1)

        # Uncomment if server identity can't be verified from local hostcert or CA cert

        curl.setopt(curl.SSL_VERIFYPEER, 0)

    try:
        print('get_data: fetch %s', url)
        curl.perform()
    except pycurl.error as e:

        # pycurl.error is an (errorcode, errormsg) tuple

        print('cURL command failed! %s', e[1])
        return ''

    http_status = curl.getinfo(pycurl.HTTP_CODE)

    # Clean up after cURL

    curl.close()
    pycurl.global_cleanup()

    server_status = ''

    if http_status == http_success:

        # Go to start of buffer

        output.seek(0)
        try:
            server_status = output.readlines()
        except:
            print('Failed to parse server status')
            return None
    else:
        print('Server returned HTTP code %d, expected %d', http_status, \
            http_success)
        return None

    output.close()

    return server_status
示例#54
0
from urllib.parse import quote, urlencode

from future import standard_library
from future.builtins import dict, int

import pycurl
from pyload.requests.base.load import LoadRequest
from pyload.requests.base.request import (BAD_HTTP_RESPONSES, Abort,
                                          ResponseException)
from pyload.requests.cookie import CookieJar
from pyload.utils.check import isiterable
from pyload.utils.convert import to_bytes

standard_library.install_aliases()

pycurl.global_init(pycurl.GLOBAL_DEFAULT)


def safequote(url):
    return quote(to_bytes(url), safe="%/:=&?~#+!$,;'@()*[]")


def safeurlencode(data):
    if isiterable(data):
        res = urlencode(
            dict((to_bytes(x), to_bytes(y)) for x, y in dict(data).items()))
    else:
        res = urlencode(to_bytes(data))
    return res

#!/usr/bin/python
import os.path
import pycurl
import sys

pycurl.global_init(pycurl.GLOBAL_DEFAULT)

def write_cb(_):
    # this should cause pycurl.WRITEFUNCTION (without any range errors)
    return -1

# download the script itself through the file:// protocol into write_cb
c = pycurl.Curl()
c.setopt(pycurl.URL, 'file://' + os.path.abspath(sys.argv[0]))
c.setopt(pycurl.WRITEFUNCTION, write_cb)
try:
    c.perform()
except pycurl.error, (err, msg):
    # we expect pycurl.E_WRITE_ERROR as the response
    assert pycurl.E_WRITE_ERROR == err

# no additional errors should be reported
assert not hasattr(sys, 'last_value')

c.close()

pycurl.global_cleanup()