def __init__(self, core):
"""Constructor"""
self.core = core
self.log = core.log
self.threads = [] # thread list
self.localThreads = [] #addon+decrypter threads
self.pause = True
self.reconnecting = Event()
self.reconnecting.clear()
self.downloaded = 0 #number of files downloaded since last cleanup
self.lock = Lock()
# some operations require to fetch url info from hoster, so we caching them so it wont be done twice
# contains a timestamp and will be purged after timeout
self.infoCache = {}
# pool of ids for online check
self.resultIDs = 0
# threads which are fetching hoster results
self.infoResults = {}
# timeout for cache purge
self.timestamp = 0
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
for i in range(self.core.config.get("download", "max_downloads")):
self.createThread()
def daemonize():
"""Properly deamonizes the process and closes file desriptors."""
sys.stderr.flush()
sys.stdout.flush()
pid = os.fork()
if pid != 0:
# Nothing more to do for the parent
sys.exit(0)
os.setsid()
pid = os.fork()
if pid != 0:
# Nothing more to do for the parent
sys.exit(0)
os.umask(0)
os.chdir('/')
dev_null = open('/dev/null', 'w')
os.dup2(dev_null.fileno(), sys.stdin.fileno())
os.dup2(dev_null.fileno(), sys.stdout.fileno())
os.dup2(dev_null.fileno(), sys.stderr.fileno())
# Re-initialize cURL. This is necessary to re-initialze the PKCS #11
# security tokens in NSS. Otherwise any use of SDK after the fork() would
# lead to the error:
#
# A PKCS #11 module returned CKR_DEVICE_ERROR, indicating that a
# problem has occurred with the token or slot.
#
pycurl.global_cleanup()
pycurl.global_init(pycurl.GLOBAL_ALL)
def main():
global saw_error
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
outf = file("/dev/null", "rb+")
cm = pycurl.CurlMulti()
# Set multi handle's options
cm.setopt(pycurl.M_PIPELINING, 1)
eh = pycurl.Curl()
for x in range(1, 20):
eh.setopt(pycurl.WRITEDATA, outf)
eh.setopt(pycurl.URL, sys.argv[1])
cm.add_handle(eh)
while 1:
ret, active_handles = cm.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
while active_handles:
ret = cm.select(1.0)
if ret == -1:
continue
while 1:
ret, active_handles = cm.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
count, good, bad = cm.info_read()
for h, en, em in bad:
print "Transfer to %s failed with %d, %s\n" % \
(h.getinfo(pycurl.EFFECTIVE_URL), en, em)
raise RuntimeError
for h in good:
httpcode = h.getinfo(pycurl.RESPONSE_CODE)
if httpcode != 200:
print "Transfer to %s failed with code %d\n" %\
(h.getinfo(pycurl.EFFECTIVE_URL), httpcode)
raise RuntimeError
else:
print "Recd %d bytes from %s" % \
(h.getinfo(pycurl.SIZE_DOWNLOAD),
h.getinfo(pycurl.EFFECTIVE_URL))
cm.remove_handle(eh)
eh.reset()
eh.close()
cm.close()
outf.close()
pycurl.global_cleanup()
def test_global_init_ack_eintr(self):
# the GLOBAL_ACK_EINTR flag was introduced in libcurl-7.30, but can also
# be backported for older versions of libcurl at the distribution level
if not util.pycurl_version_less_than(7, 30) or hasattr(pycurl, 'GLOBAL_ACK_EINTR'):
# initialize libcurl with the GLOBAL_ACK_EINTR flag
pycurl.global_init(pycurl.GLOBAL_ACK_EINTR)
pycurl.global_cleanup()
def __init__(self, core):
"""Constructor"""
self.core = core
self.log = core.log
self.threads = [] # thread list
self.localThreads = [] #addon+decrypter threads
self.pause = True
self.reconnecting = Event()
self.reconnecting.clear()
self.downloaded = 0 #number of files downloaded since last cleanup
self.lock = Lock()
# some operations require to fetch url info from hoster, so we caching them so it wont be done twice
# contains a timestamp and will be purged after timeout
self.infoCache = {}
# pool of ids for online check
self.resultIDs = 0
# threads which are fetching hoster results
self.infoResults = {}
# timeout for cache purge
self.timestamp = 0
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
for i in range(self.core.config.get("download", "max_downloads")):
self.createThread()
def main():
global saw_error
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
outf = file("/dev/null", "rb+")
cm = pycurl.CurlMulti()
# Set multi handle's options
cm.setopt(pycurl.M_PIPELINING, 1)
eh = pycurl.Curl()
for x in range(1, 20):
eh.setopt(pycurl.WRITEDATA, outf)
eh.setopt(pycurl.URL, sys.argv[1])
cm.add_handle(eh)
while 1:
ret, active_handles = cm.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
while active_handles:
ret = cm.select(1.0)
if ret == -1:
continue
while 1:
ret, active_handles = cm.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
count, good, bad = cm.info_read()
for h, en, em in bad:
print "Transfer to %s failed with %d, %s\n" % \
(h.getinfo(pycurl.EFFECTIVE_URL), en, em)
raise RuntimeError
for h in good:
httpcode = h.getinfo(pycurl.RESPONSE_CODE)
if httpcode != 200:
print "Transfer to %s failed with code %d\n" %\
(h.getinfo(pycurl.EFFECTIVE_URL), httpcode)
raise RuntimeError
else:
print "Recd %d bytes from %s" % \
(h.getinfo(pycurl.SIZE_DOWNLOAD),
h.getinfo(pycurl.EFFECTIVE_URL))
cm.remove_handle(eh)
eh.reset()
eh.close()
cm.close()
outf.close()
pycurl.global_cleanup()
def assembled(cls):
cls.needs_registration = True
cls.needs_perform = False
pycurl.global_init(pycurl.GLOBAL_ALL)
cls.multi = pycurl.CurlMulti()
cls.multi.setopt(pycurl.M_TIMERFUNCTION, cls.curl_timeout)
cls.multi.setopt(pycurl.M_SOCKETFUNCTION, cls.on_socket)
cls.instances = {}
def cleanPycurl(self):
""" make a global curl cleanup (currently unused) """
if self.processingIds():
return False
pycurl.global_cleanup()
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
self.downloaded = 0
self.log.debug("Cleaned up pycurl")
return True
def cleanPycurl(self):
""" make a global curl cleanup (currently unused) """
if self.processingIds():
return False
pycurl.global_cleanup()
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
self.downloaded = 0
self.log.debug("Cleaned up pycurl")
return True
def test_global_init_ack_eintr(self):
# the GLOBAL_ACK_EINTR flag was introduced in libcurl-7.30, but can also
# be backported for older versions of libcurl at the distribution level
if util.pycurl_version_less_than(7, 30) and not hasattr(pycurl, 'GLOBAL_ACK_EINTR'):
raise nose.plugins.skip.SkipTest('libcurl < 7.30.0 or no GLOBAL_ACK_EINTR')
# initialize libcurl with the GLOBAL_ACK_EINTR flag
pycurl.global_init(pycurl.GLOBAL_ACK_EINTR)
pycurl.global_cleanup()
def test_global_init_ack_eintr(self):
# the GLOBAL_ACK_EINTR flag was introduced in libcurl-7.30, but can also
# be backported for older versions of libcurl at the distribution level
if util.pycurl_version_less_than(
7, 30) and not hasattr(pycurl, 'GLOBAL_ACK_EINTR'):
raise unittest.SkipTest('libcurl < 7.30.0 or no GLOBAL_ACK_EINTR')
# initialize libcurl with the GLOBAL_ACK_EINTR flag
pycurl.global_init(pycurl.GLOBAL_ACK_EINTR)
pycurl.global_cleanup()
def __init__(self, options):
webdav_options = get_options(type=WebDAVSettings, from_options=options)
proxy_options = get_options(type=ProxySettings, from_options=options)
self.webdav = WebDAVSettings(webdav_options)
self.proxy = ProxySettings(proxy_options)
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
self.default_options = {}
def __init__(self, options):
webdav_options = get_options(type=WebDAVSettings, from_options=options)
proxy_options = get_options(type=ProxySettings, from_options=options)
self.webdav = WebDAVSettings(webdav_options)
self.proxy = ProxySettings(proxy_options)
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
self.default_options = {}
def wrapper(*args, **kwargs):
# curl_global_init(3) and curl_global_cleanup(3) must be called with only
# one thread running. This check is just a safety measure -- it doesn't
# cover all cases.
assert threading.activeCount() == 1, \
"Found active threads when initializing pycURL"
pycurl.global_init(pycurl.GLOBAL_ALL)
try:
return fn(*args, **kwargs)
finally:
pycurl.global_cleanup()
def __init__(self, url):
# Remove trailing slashes from url
url = url.rstrip('/')
url = url + '/json/'
self.url = url
self.is_error = 0
self.last_error = ''
self.curl_response = ''
self.curl_content = ''
self.curl_content_type = ''
self.curl_headers = ''
self.cookie = ''
pycurl.global_init(pycurl.GLOBAL_ALL)
def dj_pycurl_download(url):
pycurl.global_init(pycurl.GLOBAL_ALL)
c = pycurl.Curl()
c.setopt(pycurl.URL, url)
c.setopt(pycurl.WRITEDATA, fp)
c.setopt(pycurl.WRITEFUNCTION, dj_pycurl_writeFile)
c.setopt(pycurl.NOPROGRESS, 0)
c.setopt(pycurl.CONNECTTIMEOUT, DJ_PYCURL_CONNECTTIMEOUT)
c.setopt(pycurl.TIMEOUT, DJ_PYCURL_TIMEOUT)
c.setopt(pycurl.VERBOSE, 1)
c.perform()
c.close()
fp.close()
def dj_pycurl_download(url):
pycurl.global_init(pycurl.GLOBAL_ALL)
c = pycurl.Curl()
c.setopt(pycurl.URL, url)
c.setopt(pycurl.WRITEDATA,fp)
c.setopt(pycurl.WRITEFUNCTION,dj_pycurl_writeFile)
c.setopt(pycurl.NOPROGRESS,0)
c.setopt(pycurl.CONNECTTIMEOUT,DJ_PYCURL_CONNECTTIMEOUT)
c.setopt(pycurl.TIMEOUT,DJ_PYCURL_TIMEOUT)
c.setopt(pycurl.VERBOSE,1)
c.perform()
c.close()
fp.close()
def Init():
"""Initializes the module-global HTTP client manager.
Must be called before using any RPC function and while exactly one thread is
running.
"""
# curl_global_init(3) and curl_global_cleanup(3) must be called with only
# one thread running. This check is just a safety measure -- it doesn't
# cover all cases.
assert threading.activeCount() == 1, \
"Found more than one active thread when initializing pycURL"
logging.info("Using PycURL %s", pycurl.version)
pycurl.global_init(pycurl.GLOBAL_ALL)
def Init():
"""Initializes the module-global HTTP client manager.
Must be called before using any RPC function and while exactly one thread is
running.
"""
# curl_global_init(3) and curl_global_cleanup(3) must be called with only
# one thread running. This check is just a safety measure -- it doesn't
# cover all cases.
assert threading.activeCount() == 1, \
"Found more than one active thread when initializing pycURL"
logging.info("Using PycURL %s", pycurl.version)
pycurl.global_init(pycurl.GLOBAL_ALL)
def perform(self):
print self.version()
filesize = self.get_filesize()
pycurl.global_init(
pycurl.GLOBAL_ALL) # GLOBAL_ALL must be set in normal
if filesize == -1: # length not known, use single connection instead
c = self.gen_curl()
outfile = self.try_soutfile(self.filename)
c.setopt(pycurl.WRITEFUNCTION, outfile.write)
c.perform()
outfile.close()
else:
curlpool = []
blocksize = filesize / self.num_blocks + 1
print filesize
for p_start, p_end in [(x, x + blocksize)
for x in xrange(0, filesize, blocksize)]:
curlpool.append(self.gen_curl(p_start, p_end, filesize))
m = pycurl.CurlMulti()
m.handles = []
for c in curlpool:
m.add_handle(c)
m.handles.append(c)
try:
while True:
ret, num_handles = m.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
while num_handles:
ret = m.select(1.0)
if ret == -1:
continue
while True:
ret, num_handles = m.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
self.end_perform(normal=True)
self.event.set()
except KeyboardInterrupt:
self.end_perform(normal=False)
except SystemExit:
self.end_perform(normal=False)
pycurl.global_cleanup()
def perform(self):
print self.version()
filesize = self.get_filesize()
pycurl.global_init(pycurl.GLOBAL_ALL) # GLOBAL_ALL must be set in normal
if filesize == -1: # length not known, use single connection instead
c = self.gen_curl()
outfile = self.try_soutfile(self.filename)
c.setopt(pycurl.WRITEFUNCTION, outfile.write)
c.perform()
outfile.close()
else:
curlpool = []
blocksize = filesize / self.num_blocks + 1
print filesize
for p_start, p_end in [(x, x + blocksize) for x in xrange(0, filesize, blocksize)]:
curlpool.append(self.gen_curl(p_start, p_end, filesize))
m = pycurl.CurlMulti()
m.handles = []
for c in curlpool:
m.add_handle(c)
m.handles.append(c)
try:
while True:
ret, num_handles = m.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
while num_handles:
ret = m.select(1.0)
if ret == -1:
continue
while True:
ret, num_handles = m.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
self.end_perform(normal = True)
self.event.set()
except KeyboardInterrupt:
self.end_perform(normal = False)
except SystemExit:
self.end_perform(normal = False)
pycurl.global_cleanup()
def __init__(self, options):
self.options = options
self.server_hostname = options.get("webdav_hostname", '')
self.server_login = options.get("webdav_login", '')
self.server_password = options.get("webdav_password", '')
self.proxy_hostname = options.get("proxy_hostname", '')
self.proxy_login = options.get("proxy_login", '')
self.proxy_password = options.get("proxy_password", '')
self.cert_path = options.get("cert_path", '')
self.key_path = options.get("key_path", '')
webdav_root = options.get("webdav_root", '')
self.webdav_root = Urn(webdav_root).quote() if webdav_root else ''
self.webdav_root = self.webdav_root.rstrip(Urn.separate)
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
self.default_options = {}
def __init__(self, options):
self.options = options
self.server_hostname = options.get("webdav_hostname", '')
self.server_login = options.get("webdav_login", '')
self.server_password = options.get("webdav_password", '')
self.proxy_hostname = options.get("proxy_hostname", '')
self.proxy_login = options.get("proxy_login", '')
self.proxy_password = options.get("proxy_password", '')
self.cert_path = options.get("cert_path", '')
self.key_path = options.get("key_path", '')
webdav_root = options.get("webdav_root", '')
self.webdav_root = Urn(webdav_root).quote() if webdav_root else ''
self.webdav_root = self.webdav_root.rstrip(Urn.separate)
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
self.default_options = {}
def get_json(self, url):
# Set up the PyCurl process so we can download the results
# directly. We do this so it is easier to pull out the data
# later (e.g., cache it).
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
# Retrieve the JSON for that specific movie.
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.HTTPHEADER, ["Accept: application/json"])
# Download into a string.
buf = StringIO.StringIO()
curl.setopt(pycurl.WRITEFUNCTION, buf.write)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.perform()
# Return the resulting JSON file.
json = simplejson.loads(buf.getvalue())
return json
def get_filesize(self):
if hasattr(self, 'filesize') and self.filesize != None:
return self.filesize
pycurl.global_init(pycurl.GLOBAL_ALL) # GLOBAL_ALL must be set in normal
curl = pycurl.Curl()
curl.setopt(pycurl.HEADER, True)
curl.setopt(pycurl.NOBODY, True)
curl.setopt(pycurl.URL, self.url)
curl.setopt(pycurl.TIMEOUT, HEADER_TIMEOUT)
b = StringIO.StringIO()
curl.setopt(pycurl.WRITEFUNCTION, b.write)
curl.perform()
try:
size = int(re.findall("Content-Length: (\d+)", b.getvalue())[0])
except:
size = -1
pycurl.global_cleanup()
self.filesize = size
return size
def get_filesize(self):
if hasattr(self, 'filesize') and self.filesize != None:
return self.filesize
pycurl.global_init(
pycurl.GLOBAL_ALL) # GLOBAL_ALL must be set in normal
curl = pycurl.Curl()
curl.setopt(pycurl.HEADER, True)
curl.setopt(pycurl.NOBODY, True)
curl.setopt(pycurl.URL, self.url)
curl.setopt(pycurl.TIMEOUT, HEADER_TIMEOUT)
b = StringIO.StringIO()
curl.setopt(pycurl.WRITEFUNCTION, b.write)
curl.perform()
try:
size = int(re.findall("Content-Length: (\d+)", b.getvalue())[0])
except:
size = -1
pycurl.global_cleanup()
self.filesize = size
return size
def main():
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
# parser.add_option('-q', '--quiet', action="store_const", const=0, dest="v", default=1, help='quiet')
# parser.add_option('-v', '--verbose', action="store_const", const=1, dest="v", help='verbose')
parser.add_option('-c', '--config', action='store', dest='config', default=DEFAULT_CONF, help='load parameters from configfile (default: ' + DEFAULT_CONF + ')')
parser.add_option('-t', '--tcp', action='store_const', dest='mode', const='tcp', help='tcp mode (default)')
#parser.add_option('-u', '--udp', action='store_const', dest='t', const='udp', help='udp mode')
parser.add_option('-p', '--port', action="store", type='int', dest="port", help='port to listen (default: '+ str(DEFAULT_LISTENPORT) +')')
parser.add_option('--url', action="store", dest="url", help='URL of tunnelendpoint (default: '+ DEFAULT_URL +')')
parser.add_option('-d', '--dest', action="store", dest="dest", help='destination to connect to (default ' + DEFAULT_TARGET + ')')
parser.add_option('--proxy', action='store', dest='proxy', help='proxy to use')
parser.add_option('--auth', action='store', dest='auth', help='auth with user:password')
parser.add_option('-v', '--verbose', action='store_const', dest='verbose', const=1, help='verbose')
#parser.add_option('--no-proxy', action='store_true', dest='np', default=False, help='use no proxy (default: use proxy from env)')
global options
(options, args) = parser.parse_args()
cparser = ConfigParser.ConfigParser(defaults={
'mode': 'tcp',
'port': DEFAULT_LISTENPORT,
'url': DEFAULT_URL,
'dest': DEFAULT_TARGET,
'auth': '',
'proxy': '',
'verbose': 0
})
cparser.read(options.config)
if cparser.has_section('pyhstopc'):
if not options.mode: options.mode = cparser.get('pyhstopc', 'mode')
if not options.port: options.port = cparser.getint('pyhstopc', 'port')
if not options.url: options.url = cparser.get('pyhstopc', 'url')
if not options.dest: options.dest = cparser.get('pyhstopc', 'dest')
if not options.auth: options.auth = cparser.get('pyhstopc', 'auth')
if not options.proxy: options.proxy = cparser.get('pyhstopc', 'proxy')
try:
if not options.verbose: options.verbose = cparser.getint('pyhstopc', 'verbose')
except TypeError:
options.verbose = 0
cparser = None
print 'pyhstopc Version: ' + VERSION
print 'terminate with EOF'
print 'start..'
if USE_CURL:
pycurl.global_init(pycurl.GLOBAL_ALL)
sl = socketListener()
sl.listen()
input = sys.stdin.readline()
while input:
input = sys.stdin.readline()
sl.terminate()
if USE_CURL:
pycurl.global_cleanup()
print 'end..'
def __init__(self, pool, cr):
super(taobao_shop, self).__init__(pool, cr)
#pycurl两个函数不是线程安全。所以在主线程中进行一次的初始化和清除
import pycurl
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
pycurl.global_cleanup()
def get_data(
config,
protocol,
host,
port,
rel_path,
):
if port:
url = '%s://%s:%s/%s' % (protocol, host, port, rel_path)
else:
url = '%s://%s/%s' % (protocol, host, rel_path)
passphrase = ''
try:
pp_file = open(config.passphrase_file, 'r')
passphrase = pp_file.readline().strip()
pp_file.close()
except:
config.logger.error('Failed to read passprase from %s',
config.passphrase_file)
return None
# Store output in memory
output = StringIO.StringIO()
# Init cURL (not strictly necessary, but for symmetry with cleanup)
pycurl.global_init(pycurl.GLOBAL_SSL)
curl = pycurl.Curl()
# Never use proxy
curl.setopt(pycurl.PROXY, "")
curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP GET'])
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, output.write)
curl.setopt(pycurl.NOSIGNAL, 1)
# Uncomment to get verbose cURL output including SSL negotiation
# curl.setopt(curl.VERBOSE, 1)
# TODO: read timeout values from config?
# We can not allow the server to block for very long
curl.setopt(pycurl.CONNECTTIMEOUT, 5)
curl.setopt(pycurl.TIMEOUT, 10)
if protocol == 'https':
curl.setopt(curl.SSLCERT, config.server_cert)
curl.setopt(curl.SSLKEY, config.server_key)
if passphrase:
curl.setopt(curl.SSLKEYPASSWD, passphrase)
# Path to CA certificates
# To use NorduGRID default certificate path set:
# curl.setopt(curl.CAPATH, "/etc/grid-security/certificates")
if config.ca_dir:
curl.setopt(curl.CAPATH, config.ca_dir)
elif config.ca_file:
# We use our own demo CA file specified in the configuration for now
curl.setopt(curl.CAINFO, config.ca_file)
# Workaround for broken host certificates:
# ###################################################
# Do not use this, but fix host cert + CA instead! #
# ###################################################
# VERIFYHOST should be 2 (default) unless remote cert can not be
# verified using CA cert.
# curl.setopt(curl.SSL_VERIFYHOST,1)
# Similarly VERIFYPEER will then probably need to be set to 0
# curl.setopt(curl.SSL_VERIFYPEER,0)
# Clean up after cURL
try:
config.logger.info('get_data: fetch %s', url)
curl.perform()
except pycurl.error as e:
# pycurl.error is an (errorcode, errormsg) tuple
config.logger.error('cURL command failed! %s', e[1])
return ''
http_status = curl.getinfo(pycurl.HTTP_CODE)
curl.close()
pycurl.global_cleanup()
server_status = ConfigParser.ConfigParser()
if http_status == http_success:
# Go to start of buffer
output.seek(0)
try:
server_status.readfp(output)
except:
config.logger.error('Failed to parse server status')
return None
else:
config.logger.error('Server returned HTTP code %d, expected %d',
http_status, http_success)
return None
output.close()
return server_status
def PutFile():
srcdir = "."
filename = "testfile"
protocol = "https"
host = "mig-1.imada.sdu.dk"
port = ""
filepath = srcdir + "/" + filename
try:
inputfile = open(filepath, "rb")
except:
print "Error: Failed to open %s for reading!" % filepath
return (False, "Invalid filename!")
# Set size of file to be uploaded.
size = os.path.getsize(filepath)
if port:
url = "%s://%s:%s/%s" % (protocol, host, port, filename)
else:
url = "%s://%s/%s" % (protocol, host, filename)
# TODO: change to 'real' server certs
# Just using default NorduGRID certificates for now
os.environ["HOME"] = pwd.getpwuid(os.geteuid())[5]
globus_dir = os.path.expanduser("~/.globus")
cert_dir = globus_dir
server_cert = cert_dir + "/usercert.pem"
server_key = cert_dir + "/userkey.pem"
passwd_file = "cert_pass"
passwd = ""
try:
pw_file = open(passwd_file, "r")
passwd = pw_file.readline().strip()
pw_file.close()
except:
print "Failed to read password from file!"
return ""
# Store output in memory
output = StringIO.StringIO()
# Init cURL (not strictly necessary, but for symmetry with cleanup)
pycurl.global_init(pycurl.GLOBAL_SSL)
curl = pycurl.Curl()
curl.setopt(pycurl.HTTPHEADER, ["User-Agent: MiG HTTP PUT"])
curl.setopt(pycurl.PUT, 1)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, output.write)
curl.setopt(pycurl.NOSIGNAL, 1)
# Uncomment to get verbose cURL output including SSL negotiation
curl.setopt(curl.VERBOSE, 1)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
# curl.setopt(curl.PORT, port)
curl.setopt(pycurl.INFILE, inputfile)
curl.setopt(pycurl.INFILESIZE, size)
if protocol == "https":
curl.setopt(curl.SSLCERT, server_cert)
curl.setopt(curl.SSLKEY, server_key)
if passwd:
curl.setopt(curl.SSLKEYPASSWD, passwd)
# Path to CA certificates (NorduGRID default certificate path)
curl.setopt(curl.CAPATH, "/etc/grid-security/certificates")
# TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)
curl.setopt(curl.SSL_VERIFYHOST, 1)
# Uncomment if server identity can't be verified from local hostcert or CA cert
curl.setopt(curl.SSL_VERIFYPEER, 0)
# TODO: uncomment the following to actually execute upload
try:
curl.perform()
except pycurl.error, e:
# pycurl.error is an (errorcode, errormsg) tuple
print "Error: cURL command failed! %s" % e[1]
return (404, "Error!")
from urlparse import urljoin
import simplejson as json
from opendiamond.helpers import murmur, split_scheme
from opendiamond.protocol import XDR_attribute, XDR_object
ATTR_HEADER_URL = 'x-attributes'
ATTR_HEADER_PREFIX = 'x-attr-'
# Object attributes handled directly by the server
ATTR_DATA = ''
ATTR_OBJ_ID = '_ObjectID'
ATTR_DISPLAY_NAME = 'Display-Name'
ATTR_DEVICE_NAME = 'Device-Name'
# Initialize curl before multiple threads have been started
curl.global_init(curl.GLOBAL_DEFAULT)
class ObjectLoadError(Exception):
'''Object failed to load.'''
class EmptyObject(object):
'''An immutable Diamond object with no data and no attributes.'''
def __init__(self):
self._attrs = dict()
self._signatures = dict()
self._omit_attrs = set()
def __str__(self):
def GetFile():
# Defaults
url = "cgi-bin/ls.py"
base_dir = "."
# Just using default NorduGRID certificates for now
os.environ["HOME"] = pwd.getpwuid(os.geteuid())[5]
globus_dir = os.path.expanduser("~/.globus")
cert_dir = globus_dir
server_cert = cert_dir + "/usercert.pem"
server_key = cert_dir + "/userkey.pem"
passwd = ""
MiGServer = "https://mig-1.imada.sdu.dk"
port = "8092"
if len(sys.argv) > 1:
passwd = sys.argv[1]
data = StringIO.StringIO()
# Init cURL (not strictly necessary, but for symmetry with cleanup)
pycurl.global_init(pycurl.GLOBAL_SSL)
print "cURL:\t\t", pycurl.version
curl = pycurl.Curl()
curl.setopt(pycurl.HTTPHEADER, ["User-Agent: MiG HTTP GET"])
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.URL, MiGServer + ":" + port + "/" + url)
curl.setopt(pycurl.WRITEFUNCTION, data.write)
curl.setopt(pycurl.NOSIGNAL, 1)
# Uncomment to get verbose cURL output including SSL negotiation
curl.setopt(curl.VERBOSE, 1)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
# curl.setopt(curl.PORT, port)
curl.setopt(curl.SSLCERT, server_cert)
curl.setopt(curl.SSLKEY, server_key)
if passwd:
curl.setopt(curl.SSLKEYPASSWD, passwd)
# Path to CA certificates (NorduGRID default certificate path)
curl.setopt(curl.CAPATH, "/etc/grid-security/certificates")
# TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)
curl.setopt(curl.SSL_VERIFYHOST, 1)
# Uncomment if server identity can't be verified from local hostcert or CA cert
curl.setopt(curl.SSL_VERIFYPEER, 0)
print "fetching:\t", url
print "cert:\t\t", server_cert
print "key:\t\t", server_key
print "passwd:\t\t", passwd
# Clean up after cURL
try:
curl.perform()
except pycurl.error, e:
print "cURL command failed!:"
# error is a (errorcode, errormsg) tuple
print e[1]
return False
def get_data(
protocol,
host,
port,
rel_path,
cert,
key,
ca_dir,
ca_file,
passphrase_file='',
):
if port:
url = '%s://%s:%s/%s' % (protocol, host, port, rel_path)
else:
url = '%s://%s/%s' % (protocol, host, rel_path)
passphrase = ''
if passphrase_file:
try:
pp_file = open(passphrase_file, 'r')
passphrase = pp_file.readline().strip()
pp_file.close()
except:
print 'Failed to read passprase from %s', passphrase_file
return None
# Store output in memory
output = StringIO.StringIO()
# Init cURL (not strictly necessary, but for symmetry with cleanup)
pycurl.global_init(pycurl.GLOBAL_SSL)
curl = pycurl.Curl()
curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP GET'])
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, output.write)
curl.setopt(pycurl.NOSIGNAL, 1)
# Uncomment to get verbose cURL output including SSL negotiation
curl.setopt(curl.VERBOSE, 1)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
if protocol == 'https':
curl.setopt(curl.SSLCERT, cert)
curl.setopt(curl.SSLKEY, key)
if passphrase:
curl.setopt(curl.SSLKEYPASSWD, passphrase)
# Path to CA certificates
if ca_dir:
curl.setopt(curl.CAPATH, ca_dir)
elif ca_file:
# We use our own demo CA file specified in the configuration for now
curl.setopt(curl.CAINFO, ca_file)
# Workaround for broken host certificates:
# ###################################################
# Do not use this, but fix host cert + CA instead! #
# ###################################################
# VERIFYHOST should be 2 (default) unless remote cert can not be
# verified using CA cert.
# curl.setopt(curl.SSL_VERIFYHOST,1)
# Similarly VERIFYPEER will then probably need to be set to 0
# curl.setopt(curl.SSL_VERIFYPEER,0)
# TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)
curl.setopt(curl.SSL_VERIFYHOST, 1)
# Uncomment if server identity can't be verified from local hostcert or CA cert
curl.setopt(curl.SSL_VERIFYPEER, 0)
try:
print 'get_data: fetch %s', url
curl.perform()
except pycurl.error, e:
# pycurl.error is an (errorcode, errormsg) tuple
print 'cURL command failed! %s', e[1]
return ''
def GetFile():
# Defaults
url = 'cgi-bin/ls.py'
base_dir = '.'
# Just using default NorduGRID certificates for now
os.environ['HOME'] = pwd.getpwuid(os.geteuid())[5]
globus_dir = os.path.expanduser('~/.globus')
cert_dir = globus_dir
server_cert = cert_dir + '/usercert.pem'
server_key = cert_dir + '/userkey.pem'
passwd = ''
MiGServer = 'https://mig-1.imada.sdu.dk'
port = '8092'
if len(sys.argv) > 1:
passwd = sys.argv[1]
data = StringIO.StringIO()
# Init cURL (not strictly necessary, but for symmetry with cleanup)
pycurl.global_init(pycurl.GLOBAL_SSL)
print 'cURL:\t\t', pycurl.version
curl = pycurl.Curl()
curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP GET'])
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.URL, MiGServer + ':' + port + '/' + url)
curl.setopt(pycurl.WRITEFUNCTION, data.write)
curl.setopt(pycurl.NOSIGNAL, 1)
# Uncomment to get verbose cURL output including SSL negotiation
curl.setopt(curl.VERBOSE, 1)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
# curl.setopt(curl.PORT, port)
curl.setopt(curl.SSLCERT, server_cert)
curl.setopt(curl.SSLKEY, server_key)
if passwd:
curl.setopt(curl.SSLKEYPASSWD, passwd)
# Path to CA certificates (NorduGRID default certificate path)
curl.setopt(curl.CAPATH, '/etc/grid-security/certificates')
# TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)
curl.setopt(curl.SSL_VERIFYHOST, 1)
# Uncomment if server identity can't be verified from local hostcert or CA cert
curl.setopt(curl.SSL_VERIFYPEER, 0)
print 'fetching:\t', url
print 'cert:\t\t', server_cert
print 'key:\t\t', server_key
print 'passwd:\t\t', passwd
# Clean up after cURL
try:
curl.perform()
except pycurl.error, e:
print 'cURL command failed!:'
# error is a (errorcode, errormsg) tuple
print e[1]
return False
def main():
signal.signal(signal.SIGHUP, signal.SIG_IGN)
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
# parser.add_option('-q', '--quiet', action="store_const", const=0, dest="v", default=1, help='quiet')
parser.add_option('-c', '--config', action='store', dest='config', default=DEFAULT_CONF, help='load parameters from configfile (default: ' + DEFAULT_CONF + ')')
parser.add_option('-t', '--tcp', action='store_const', dest='mode', const='tcp', help='tcp mode (default)')
parser.add_option('-u', '--udp', action='store_const', dest='mode', const='udp', help='udp mode')
parser.add_option('-L', action='append', dest='forward', help='forward port:remotehost:remoteport (like ssh)')
parser.add_option('--url', action="store", dest="url", help='URL of tunnelendpoint')
parser.add_option('--proxy', action='store', dest='proxy', help='proxy to use')
parser.add_option('--auth', action='store', dest='auth', help='auth with user:password')
parser.add_option('-a', '--agent', action='store', dest='agent', help='fake useragent')
parser.add_option('-v', '--verbose', action='store_const', dest='verbose', const=1, help='verbose')
parser.add_option('--no-verify-ssl', action='store_true', dest='nv', help='do not verify ssl-host')
parser.add_option('--verify-ssl', action='store_false', dest='nv', help='do not verify ssl-host')
global options
(options, args) = parser.parse_args()
cparser = ConfigParser.ConfigParser(defaults={
'mode': 'tcp',
'url': DEFAULT_URL,
'auth': '',
'proxy': '',
'agent': '',
'verbose': 0,
'verify': True
})
cparser.read(options.config)
if cparser.has_section('pyhstopc'):
if not options.url: options.url = cparser.get('pyhstopc', 'url')
if not options.auth: options.auth = cparser.get('pyhstopc', 'auth')
if not options.agent: options.agent = cparser.get('pyhstopc', 'agent')
if not options.proxy: options.proxy = cparser.get('pyhstopc', 'proxy')
if not options.forward:
options.forward = []
try:
options.forward.extend(cparser.get('pyhstopc', 'forward').split(','))
except ConfigParser.NoOptionError:
pass
try:
if not options.verbose: options.verbose = cparser.getint('pyhstopc', 'verbose')
except TypeError:
options.verbose = 0
try:
if options.nv == None: options.nv = not cparser.getboolean('pyhstopc', 'verify')
except TypeError:
options.nv = False
cparser = None
tmpforward = options.forward
options.forward = []
for i in tmpforward:
try:
lport, rhost, rport = i.split(':')
options.forward.append((int(lport.strip()), rhost.strip(), int(rport.strip()),'tcp'))
except (KeyError, ValueError):
try:
lport, rhost, rport, mode = i.split(':')
options.forward.append((int(lport.strip()), rhost.strip(), int(rport.strip()), mode))
except (KeyError, ValueError):
print 'malformed forward option: ', i
print 'pyhstopc Version: ' + VERSION
print 'terminate with EOF'
print 'start..'
pycurl.global_init(pycurl.GLOBAL_ALL)
sls = []
for i in range(len(options.forward)):
sl = socketListener(i)
sl.listen()
try:
input = sys.stdin.readline()
while input:
input = sys.stdin.readline()
except KeyboardInterrupt:
print 'interrupted'
for sl in sls:
sl.terminate()
pycurl.global_cleanup()
print 'end..'
def test_global_init_bogus(self):
# initialize libcurl with bogus flags
with pytest.raises(ValueError):
pycurl.global_init(0xffff)
import os
from io import BytesIO
DEFAULT_OPTIONS = {
pycurl.CAINFO: certifi.where(),
pycurl.FOLLOWLOCATION: 1,
pycurl.MAXREDIRS: 5,
pycurl.SSL_VERIFYPEER: 1,
pycurl.SSL_VERIFYHOST: 2
}
# global inits
os.environ.update({'SSLKEYLOGFILE': './sslkeys.log'})
atexit.register(pycurl.global_cleanup)
pycurl.global_init(pycurl.GLOBAL_SSL)
class Session:
''' Session class for api requests based on pycurl '''
def __init__(self):
self._session = pycurl.Curl()
self._options = {}
self.set_options(DEFAULT_OPTIONS)
def __enter__(self):
return self
def __exit__(self, *args):
self._session.close()
def test_global_init_default(self):
# initialize libcurl with DEFAULT flags
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
pycurl.global_cleanup()
def __init__(self, *args, **kwargs):
super(PyCurlHttpService, self).__init__(*args, **kwargs)
pycurl.global_init(pycurl.GLOBAL_ALL)
def test_global_init_bogus(self):
# initialize libcurl with bogus flags
pycurl.global_init(0xffff)
def init_libcurl():
pycurl.global_init(pycurl.GLOBAL_ALL)
def main():
signal.signal(signal.SIGHUP, signal.SIG_IGN)
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
# parser.add_option('-q', '--quiet', action="store_const", const=0, dest="v", default=1, help='quiet')
parser.add_option('-c', '--config', action='store', dest='config', default=DEFAULT_CONF, help='load parameters from configfile (default: ' + DEFAULT_CONF + ')')
parser.add_option('-t', '--tcp', action='store_const', dest='mode', const='tcp', help='tcp mode (default)')
parser.add_option('-u', '--udp', action='store_const', dest='mode', const='udp', help='udp mode')
parser.add_option('-L', action='append', dest='forward', help='forward port:remotehost:remoteport (like ssh)')
parser.add_option('--url', action="store", dest="url", help='URL of tunnelendpoint')
parser.add_option('--proxy', action='store', dest='proxy', help='proxy to use')
parser.add_option('--auth', action='store', dest='auth', help='auth with user:password')
parser.add_option('-a', '--agent', action='store', dest='agent', help='fake useragent')
parser.add_option('-v', '--verbose', action='store_const', dest='verbose', const=1, help='verbose')
parser.add_option('--no-verify-ssl', action='store_true', dest='nv', help='do not verify ssl-host')
parser.add_option('--verify-ssl', action='store_false', dest='nv', help='do not verify ssl-host')
global options
(options, args) = parser.parse_args()
cparser = ConfigParser.ConfigParser(defaults={
'mode': 'tcp',
'url': DEFAULT_URL,
'auth': '',
'proxy': '',
'agent': '',
'verbose': 0,
'verify': True
})
cparser.read(options.config)
if cparser.has_section('pyhstopc'):
if not options.url: options.url = cparser.get('pyhstopc', 'url')
if not options.auth: options.auth = cparser.get('pyhstopc', 'auth')
if not options.agent: options.agent = cparser.get('pyhstopc', 'agent')
if not options.proxy: options.proxy = cparser.get('pyhstopc', 'proxy')
if not options.forward:
options.forward = []
try:
options.forward.extend(cparser.get('pyhstopc', 'forward').split(','))
except ConfigParser.NoOptionError:
pass
try:
if not options.verbose: options.verbose = cparser.getint('pyhstopc', 'verbose')
except TypeError:
options.verbose = 0
try:
if options.nv == None: options.nv = not cparser.getboolean('pyhstopc', 'verify')
except TypeError:
options.nv = False
cparser = None
tmpforward = options.forward
options.forward = []
for i in tmpforward:
try:
lport, rhost, rport = i.split(':')
options.forward.append((int(lport.strip()), rhost.strip(), int(rport.strip()),'tcp'))
except (KeyError, ValueError):
try:
lport, rhost, rport, mode = i.split(':')
options.forward.append((int(lport.strip()), rhost.strip(), int(rport.strip()), mode))
except (KeyError, ValueError):
print 'malformed forward option: ', i
print 'pyhstopc Version: ' + VERSION
print 'terminate with EOF'
print 'start..'
pycurl.global_init(pycurl.GLOBAL_ALL)
sls = []
for i in range(len(options.forward)):
sl = socketListener(i)
sl.listen()
try:
input = sys.stdin.readline()
while input:
input = sys.stdin.readline()
except KeyboardInterrupt:
print 'interrupted'
for sl in sls:
sl.terminate()
pycurl.global_cleanup()
print 'end..'
def put_data(
config,
filename,
protocol,
host,
port,
rel_path,
):
try:
inputfile = open(filename, 'rb')
except:
config.logger.error('Failed to open %s for reading!', filename)
return (False, 'Invalid filename!')
# Set size of file to be uploaded.
size = os.path.getsize(filename)
if port:
url = '%s://%s:%s/%s' % (protocol, host, port, rel_path)
else:
url = '%s://%s/%s' % (protocol, host, rel_path)
passphrase = ''
try:
pp_file = open(config.passphrase_file, 'r')
passphrase = pp_file.readline().strip()
pp_file.close()
except:
config.logger.error('Failed to read passphrase from file %s',
config.passphrase_file)
return (-1, 'Failed to read passphrase from file')
# Store output in memory
output = StringIO.StringIO()
# Init cURL (not strictly necessary, but for symmetry with cleanup)
pycurl.global_init(pycurl.GLOBAL_SSL)
curl = pycurl.Curl()
curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP PUT'])
curl.setopt(pycurl.PUT, 1)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, output.write)
curl.setopt(pycurl.NOSIGNAL, 1)
# Uncomment to get verbose cURL output including SSL negotiation
# curl.setopt(curl.VERBOSE, 1)
# We can not let the server block for very long
curl.setopt(pycurl.CONNECTTIMEOUT, 5)
curl.setopt(pycurl.TIMEOUT, 10)
curl.setopt(pycurl.INFILE, inputfile)
curl.setopt(pycurl.INFILESIZE, size)
if protocol == 'https':
curl.setopt(curl.SSLCERT, config.server_cert)
curl.setopt(curl.SSLKEY, config.server_key)
if passphrase:
curl.setopt(curl.SSLKEYPASSWD, passphrase)
# Path to CA certificates
# To use NorduGRID default certificate path set:
# curl.setopt(curl.CAPATH, "/etc/grid-security/certificates")
if config.ca_dir:
curl.setopt(curl.CAPATH, config.ca_dir)
elif config.ca_file:
# We use our own demo CA file specified in the configuration for now
curl.setopt(curl.CAINFO, config.ca_file)
# Workaround for broken host certificates:
# ###################################################
# Do not use this, but fix host cert + CA instead! #
# ###################################################
# VERIFYHOST should be 2 (default) unless remote cert can not be
# verified using CA cert.
# curl.setopt(curl.SSL_VERIFYHOST,1)
# Similarly VERIFYPEER will then probably need to be set to 0
# curl.setopt(curl.SSL_VERIFYPEER,0)
try:
curl.perform()
except pycurl.error, e:
# pycurl.error is an (errorcode, errormsg) tuple
config.logger.error('cURL command failed! %s', e[1])
return (404, 'Error!')
def init_libcurl():
pycurl.global_init(pycurl.GLOBAL_ALL)
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken. If not, see <http://www.gnu.org/licenses/>.
import cPickle
import zlib
import json
# Pycurl part
import pycurl
pycurl.global_init(pycurl.GLOBAL_ALL)
import urllib
from StringIO import StringIO
from shinken.bin import VERSION
from shinken.log import logger
PYCURL_VERSION = pycurl.version_info()[1]
class HTTPException(Exception):
pass
HTTPExceptions = (HTTPException,)
class FileReader:
def __init__(self, fp):
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], "", ["port=", "test=", "proxy=", "cwd=", "secondary-proxy-port="])
except getopt.GetoptError as err:
print str(err)
sys.exit(2)
test = None
proxy_port = 12345
proxy_executable = None
proxy_cwd = None
secondary_proxy_port = None
for o, a in opts:
if o == "--port":
proxy_port = int(a)
elif o == "--test":
test = a
elif o == "--proxy":
proxy_executable = a
elif o == "--cwd":
proxy_cwd = a
elif o == "--secondary-proxy-port":
secondary_proxy_port = int(a)
else:
assert False, "unhandled option"
assert test is not None, "test must be specified"
assert proxy_executable is not None, "proxy executable must be specified"
using_proxy_chaining = (secondary_proxy_port is not None)
if test == "proxy-cycle":
assert using_proxy_chaining, "The secondary-proxy-port flag is required for the proxy-cycle test"
cache_dir = os.path.expanduser('~/.proxy-cache-' + socket.gethostname())
shutil.rmtree(cache_dir, ignore_errors=True)
# Get a place to store the stdout/stderr
log_file = tempfile.TemporaryFile()
try:
if not using_proxy_chaining:
proxy = subprocess.Popen([proxy_executable, "--port", str(proxy_port)], cwd=proxy_cwd, stdout=log_file, stderr=subprocess.STDOUT)
else:
proxy = subprocess.Popen([proxy_executable, "--port", str(proxy_port), "--proxy-server", "localhost", "--proxy-port", str(secondary_proxy_port)], cwd=proxy_cwd, stdout=log_file, stderr=subprocess.STDOUT)
except:
print("Couldn't start proxy")
return
if using_proxy_chaining:
try:
secondary_proxy = subprocess.Popen([proxy_executable, "--port", str(secondary_proxy_port), "--proxy-server", "localhost", "--proxy-port", str(proxy_port)], cwd=proxy_cwd, stdout=log_file, stderr=subprocess.STDOUT)
#secondary_proxy = subprocess.Popen([proxy_executable, "--port", str(secondary_proxy_port)], cwd=proxy_cwd, stdout=log_file, stderr=subprocess.STDOUT)
except:
print("Couldn't start secondary proxy")
return
#elif test == 'proxy-chain-basic':
# try:
# exec_command = "ssh " + os.environ["USER"] + "@myth12.stanford.edu 'cd /usr/class/cs110/staff/bin/master-repos/assign6-soln/; " + proxy_executable + " --port 56565'"
# print exec_command
# secondary_proxy = subprocess.Popen(exec_command, cwd=proxy_cwd, stdout=log_file, stderr=subprocess.STDOUT)
# except:
# print("Couldn't start secondary proxy")
# return
# try:
time.sleep(5) # Make sure the proxy is running
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
test_runner = ProxyTest(proxy_port)
test_runner.run_test(test)
pycurl.global_cleanup()
return_code = proxy.poll() # Get the return status if it already exited (crashed)
if return_code is None: # Hopefully the proxy was still running
proxy.terminate()
proxy.wait()
else:
log_file.seek(0)
print("Proxy crashed; return code: %d, output:" % return_code)
print(log_file.read())
if using_proxy_chaining:
secondary_return_code = secondary_proxy.poll();
if secondary_return_code is None:
secondary_proxy.terminate()
secondary_proxy.wait()
# except: # Make sure the proxy is dead
# print("Something went wrong. Killing the proxy.")
# proxy.kill()
# proxy.wait()
# raise
# finally:
shutil.rmtree(cache_dir, ignore_errors=True) # Clean up
def __init__(self, pool, cr):
super(taobao_shop, self).__init__(pool, cr)
#pycurl两个函数不是线程安全。所以在主线程中进行一次的初始化和清除
import pycurl
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
pycurl.global_cleanup()
def test_global_init_default(self):
# initialize libcurl with DEFAULT flags
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
pycurl.global_cleanup()
def PutFile():
srcdir = '.'
filename = 'testfile'
protocol = 'https'
host = 'mig-1.imada.sdu.dk'
port = ''
filepath = srcdir + '/' + filename
try:
inputfile = open(filepath, 'rb')
except:
print 'Error: Failed to open %s for reading!' % filepath
return (False, 'Invalid filename!')
# Set size of file to be uploaded.
size = os.path.getsize(filepath)
if port:
url = '%s://%s:%s/%s' % (protocol, host, port, filename)
else:
url = '%s://%s/%s' % (protocol, host, filename)
# TODO: change to 'real' server certs
# Just using default NorduGRID certificates for now
os.environ['HOME'] = pwd.getpwuid(os.geteuid())[5]
globus_dir = os.path.expanduser('~/.globus')
cert_dir = globus_dir
server_cert = cert_dir + '/usercert.pem'
server_key = cert_dir + '/userkey.pem'
passwd_file = 'cert_pass'
passwd = ''
try:
pw_file = open(passwd_file, 'r')
passwd = pw_file.readline().strip()
pw_file.close()
except:
print 'Failed to read password from file!'
return ''
# Store output in memory
output = StringIO.StringIO()
# Init cURL (not strictly necessary, but for symmetry with cleanup)
pycurl.global_init(pycurl.GLOBAL_SSL)
curl = pycurl.Curl()
curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP PUT'])
curl.setopt(pycurl.PUT, 1)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, output.write)
curl.setopt(pycurl.NOSIGNAL, 1)
# Uncomment to get verbose cURL output including SSL negotiation
curl.setopt(curl.VERBOSE, 1)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
# curl.setopt(curl.PORT, port)
curl.setopt(pycurl.INFILE, inputfile)
curl.setopt(pycurl.INFILESIZE, size)
if protocol == 'https':
curl.setopt(curl.SSLCERT, server_cert)
curl.setopt(curl.SSLKEY, server_key)
if passwd:
curl.setopt(curl.SSLKEYPASSWD, passwd)
# Path to CA certificates (NorduGRID default certificate path)
curl.setopt(curl.CAPATH, '/etc/grid-security/certificates')
# TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)
curl.setopt(curl.SSL_VERIFYHOST, 1)
# Uncomment if server identity can't be verified from local hostcert or CA cert
curl.setopt(curl.SSL_VERIFYPEER, 0)
# TODO: uncomment the following to actually execute upload
try:
curl.perform()
except pycurl.error, e:
# pycurl.error is an (errorcode, errormsg) tuple
print 'Error: cURL command failed! %s' % e[1]
return (404, 'Error!')
def test_global_init_bogus(self):
# initialize libcurl with bogus flags
pycurl.global_init(0xffff)
def put_data(
config,
filename,
protocol,
host,
port,
rel_path,
):
try:
inputfile = open(filename, 'rb')
except:
config.logger.error('Failed to open %s for reading!', filename)
return (False, 'Invalid filename!')
# Set size of file to be uploaded.
size = os.path.getsize(filename)
if port:
url = '%s://%s:%s/%s' % (protocol, host, port, rel_path)
else:
url = '%s://%s/%s' % (protocol, host, rel_path)
passphrase = ''
try:
pp_file = open(config.passphrase_file, 'r')
passphrase = pp_file.readline().strip()
pp_file.close()
except:
config.logger.error('Failed to read passphrase from file %s',
config.passphrase_file)
return (-1, 'Failed to read passphrase from file')
# Store output in memory
output = StringIO.StringIO()
# Init cURL (not strictly necessary, but for symmetry with cleanup)
pycurl.global_init(pycurl.GLOBAL_SSL)
curl = pycurl.Curl()
# Never use proxy
curl.setopt(pycurl.PROXY, "")
curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP PUT'])
curl.setopt(pycurl.PUT, 1)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, output.write)
curl.setopt(pycurl.NOSIGNAL, 1)
# Uncomment to get verbose cURL output including SSL negotiation
# curl.setopt(curl.VERBOSE, 1)
# We can not let the server block for very long
curl.setopt(pycurl.CONNECTTIMEOUT, 5)
curl.setopt(pycurl.TIMEOUT, 10)
curl.setopt(pycurl.INFILE, inputfile)
curl.setopt(pycurl.INFILESIZE, size)
if protocol == 'https':
curl.setopt(curl.SSLCERT, config.server_cert)
curl.setopt(curl.SSLKEY, config.server_key)
if passphrase:
curl.setopt(curl.SSLKEYPASSWD, passphrase)
# Path to CA certificates
# To use NorduGRID default certificate path set:
# curl.setopt(curl.CAPATH, "/etc/grid-security/certificates")
if config.ca_dir:
curl.setopt(curl.CAPATH, config.ca_dir)
elif config.ca_file:
# We use our own demo CA file specified in the configuration for now
curl.setopt(curl.CAINFO, config.ca_file)
# Workaround for broken host certificates:
# ###################################################
# Do not use this, but fix host cert + CA instead! #
# ###################################################
# VERIFYHOST should be 2 (default) unless remote cert can not be
# verified using CA cert.
# curl.setopt(curl.SSL_VERIFYHOST,1)
# Similarly VERIFYPEER will then probably need to be set to 0
# curl.setopt(curl.SSL_VERIFYPEER,0)
try:
curl.perform()
except pycurl.error as e:
# pycurl.error is an (errorcode, errormsg) tuple
config.logger.error('cURL command failed! %s', e[1])
return (404, 'Error!')
http_status = curl.getinfo(pycurl.HTTP_CODE)
# Clean up after cURL
curl.close()
pycurl.global_cleanup()
if http_status == http_success:
config.logger.info('PUT request succeeded')
# Go to start of buffer
output.seek(0)
msg = output.readlines()
else:
# print msg
config.logger.warning('Server returned HTTP code %d, expected %d',
http_status, http_success)
inputfile.close()
output.close()
return (http_status, 'Success!')
return response
@classmethod
def getServerInfo(c):
"""
Return the information needed to send API messages to the server.
This can be used by an external program (e.g. pdinstall).
"""
info = {
'authorization': 'Bearer {}'.format(c.token),
'router_id': nexus.core.info.pdid,
'server': nexus.core.info.pdserver
}
return info
@classmethod
def resetToken(c):
"""
Reset the auth token, to be called if the router's identity has changed.
"""
c.token = None
# Initialize pycurl. Does this do anything?
pycurl.global_init(pycurl.GLOBAL_ALL)
# Set the number of connections that can be kept alive in the connection pool.
# Setting this equal to the size of the semaphore should prevent churn.
TwistedRequestDriver.pool.maxPersistentPerHost = settings.PDSERVER_MAX_CONCURRENT_REQUESTS
def get_data(
protocol,
host,
port,
rel_path,
cert,
key,
ca_dir,
ca_file,
passphrase_file='',
):
if port:
url = '%s://%s:%s/%s' % (protocol, host, port, rel_path)
else:
url = '%s://%s/%s' % (protocol, host, rel_path)
passphrase = ''
if passphrase_file:
try:
pp_file = open(passphrase_file, 'r')
passphrase = pp_file.readline().strip()
pp_file.close()
except:
print('Failed to read passprase from %s', passphrase_file)
return None
# Store output in memory
output = StringIO.StringIO()
# Init cURL (not strictly necessary, but for symmetry with cleanup)
pycurl.global_init(pycurl.GLOBAL_SSL)
curl = pycurl.Curl()
curl.setopt(pycurl.HTTPHEADER, ['User-Agent: MiG HTTP GET'])
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, output.write)
curl.setopt(pycurl.NOSIGNAL, 1)
# Uncomment to get verbose cURL output including SSL negotiation
curl.setopt(curl.VERBOSE, 1)
curl.setopt(pycurl.CONNECTTIMEOUT, 30)
curl.setopt(pycurl.TIMEOUT, 300)
if protocol == 'https':
curl.setopt(curl.SSLCERT, cert)
curl.setopt(curl.SSLKEY, key)
if passphrase:
curl.setopt(curl.SSLKEYPASSWD, passphrase)
# Path to CA certificates
if ca_dir:
curl.setopt(curl.CAPATH, ca_dir)
elif ca_file:
# We use our own demo CA file specified in the configuration for now
curl.setopt(curl.CAINFO, ca_file)
# Workaround for broken host certificates:
# ###################################################
# Do not use this, but fix host cert + CA instead! #
# ###################################################
# VERIFYHOST should be 2 (default) unless remote cert can not be
# verified using CA cert.
# curl.setopt(curl.SSL_VERIFYHOST,1)
# Similarly VERIFYPEER will then probably need to be set to 0
# curl.setopt(curl.SSL_VERIFYPEER,0)
# TODO: Should not be necessary but mig-1 host cert has wrong subject (vcr)
curl.setopt(curl.SSL_VERIFYHOST, 1)
# Uncomment if server identity can't be verified from local hostcert or CA cert
curl.setopt(curl.SSL_VERIFYPEER, 0)
try:
print('get_data: fetch %s', url)
curl.perform()
except pycurl.error as e:
# pycurl.error is an (errorcode, errormsg) tuple
print('cURL command failed! %s', e[1])
return ''
http_status = curl.getinfo(pycurl.HTTP_CODE)
# Clean up after cURL
curl.close()
pycurl.global_cleanup()
server_status = ''
if http_status == http_success:
# Go to start of buffer
output.seek(0)
try:
server_status = output.readlines()
except:
print('Failed to parse server status')
return None
else:
print('Server returned HTTP code %d, expected %d', http_status, \
http_success)
return None
output.close()
return server_status
from urllib.parse import quote, urlencode
from future import standard_library
from future.builtins import dict, int
import pycurl
from pyload.requests.base.load import LoadRequest
from pyload.requests.base.request import (BAD_HTTP_RESPONSES, Abort,
ResponseException)
from pyload.requests.cookie import CookieJar
from pyload.utils.check import isiterable
from pyload.utils.convert import to_bytes
standard_library.install_aliases()
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
def safequote(url):
return quote(to_bytes(url), safe="%/:=&?~#+!$,;'@()*[]")
def safeurlencode(data):
if isiterable(data):
res = urlencode(
dict((to_bytes(x), to_bytes(y)) for x, y in dict(data).items()))
else:
res = urlencode(to_bytes(data))
return res
#!/usr/bin/python
import os.path
import pycurl
import sys
pycurl.global_init(pycurl.GLOBAL_DEFAULT)
def write_cb(_):
# this should cause pycurl.WRITEFUNCTION (without any range errors)
return -1
# download the script itself through the file:// protocol into write_cb
c = pycurl.Curl()
c.setopt(pycurl.URL, 'file://' + os.path.abspath(sys.argv[0]))
c.setopt(pycurl.WRITEFUNCTION, write_cb)
try:
c.perform()
except pycurl.error, (err, msg):
# we expect pycurl.E_WRITE_ERROR as the response
assert pycurl.E_WRITE_ERROR == err
# no additional errors should be reported
assert not hasattr(sys, 'last_value')
c.close()
pycurl.global_cleanup()