def initSearchClient(self):
print 'initialize search....'
http = HttpTransport()
opener = urllib2.build_opener(HTTPSudsPreprocessor(self.SID))
http.urlopener = opener
self.client['search'] = Client(self.url['search'], transport = http)
print 'searching initializtion done'
def __init__(self, **kwargs):
"""
@param kwargs: Keyword arguments.
- B{proxy} - An http proxy to be specified on requests.
The proxy is defined as {protocol:proxy,}
- type: I{dict}
- default: {}
- B{timeout} - Set the url open timeout (seconds).
- type: I{float}
- default: 90
- B{username} - The username used for http authentication.
- type: I{str}
- default: None
- B{password} - The password used for http authentication.
- type: I{str}
- default: None
- B{unverified_context} - Use an unverified context for the
connection, i.e. disabling HTTPS certificate validation.
- type: I{bool}
- default: False
"""
HttpTransport.__init__(self, **kwargs)
self.pm = urllib.request.HTTPPasswordMgrWithDefaultRealm()
if self.options.unverified_context:
import ssl
self.HTTPSHandler = urllib.request.HTTPSHandler(context=ssl._create_unverified_context())
else:
self.HTTPSHandler = urllib.request.HTTPSHandler()
def _add_sid(self):
"""Create URL opener with authentication token as header."""
opener = urllib2.build_opener()
opener.addheaders = [('Cookie', 'SID="'+self.sid_token+'"')]
http = HttpTransport()
http.urlopener = opener
self._establish_search_client(http)
def __init__(self, slug=None, session=None, related_objects=None, timeout=None):
self.related_objects = related_objects or ()
self.slug = slug
self.timeout = timeout
# super won't work because not using new style class
HttpTransport.__init__(self)
self._session = session or security_requests.SecuritySession()
def auth(ver, type, username, password):
if ver == 2:
if type == 'up':
auth_client = Client(AUTH_URL_ver2, username = username, password = password) #version 2.0
SID = auth_client.service.authenticate()
return SID
elif type == 'ip':
auth_client = Client(AUTH_URL_ver2)
SID = auth_client.service.authenticate()
return SID
else:
print "Type either 'ip' or 'up'. 'ip' for ip access, 'up' for username/password access"
elif ver == 3:
if type == 'up':
username_password = ['username_password'][username]
#Your username/password from a purchased subscription should go here!
auth = HttpTransport()
auth_sender = urllib2.build_opener(HTTPAuthSender(username_password))
auth.urlopener = auth_sender
auth_client = Client(AUTH_URL_ver3, transport = auth)
SID = auth_client.service.authenticate()
return SID
elif type == 'ip':
auth_client = Client(AUTH_URL_ver3) #comment out if using username, password. #version 3.0
SID = auth_client.service.authenticate()
return SID
else:
print "Type either 'ip' or 'up'. 'ip' for ip access, 'up' for username/password access. username is an integer, password can be a null value for version 3."
else:
print "Authentication failed. Invalid version. Your request was not supported by this authentication module."
def search_lite(fieldtag, searchterm, ver, count, SID):
#fieldtag must be 'PN = ' or 'CD ='
http = HttpTransport()
opener = urllib2.build_opener(HTTPSudsPreprocessor(SID))
http.urlopener = opener
client_obj = Client(SEARCH_LITE_URL[int(ver - 2)], transport = http, retxml = True)
query = fieldtag + '=' + searchterm
#construct query and retrieve field parameters
qparams = {
'databaseId' : 'DIIDW',
'userQuery' : query,
'queryLanguage' : 'en',
'editions' : [{
'collection' : 'DIIDW',
'edition' : 'CDerwent',
},{
'collection' : 'DIIDW',
'edition' : 'MDerwent',
},{
'collection' : 'DIIDW',
'edition' : 'EDerwent',
}]
}
rparams = {
'count' : count, # 1-500
'firstRecord' : 1,
'sortField' : [{
'name' : 'TC',
'sort' : 'D',
}]
}
result = client_obj.service.search(qparams, rparams)
return result
def __init__(self, key, cert, *args, **kwargs):
"""
@param key: full path for the client's private key file
@param cert: full path for the client's PEM certificate file
"""
HttpTransport.__init__(self, *args, **kwargs)
self.key = key
self.cert = cert
def __init__(self, key, cert, *args, **kwargs):
"""
@param key: full path for the client's private key file
@param cert: full path for the client's PEM certificate file
"""
HttpTransport.__init__(self, *args, **kwargs)
self.key = key
self.cert = cert
self.urlopener = u2.build_opener(self._get_auth_handler())
def u2handlers(self):
try:
from ntlm import HTTPNtlmAuthHandler
except ImportError:
raise Exception("Cannot import python-ntlm module")
handlers = HttpTransport.u2handlers(self)
handlers.append(HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(self.pm))
return handlers
def open(self, request):
url = request.url
if url.startswith('file:'):
return HttpTransport.open(self, request)
else:
resp = self._session.get(url)
resp.raise_for_status()
return BytesIO(resp.content)
def send(self, request):
self.addcredentials(request)
r = XMLParser.parse(request.message.decode('ascii'))
data = dump_records(r)
request.message = data
request.headers['Content-Type'] = 'application/soap+msbin1'
# request.message = request.message()
return HttpTransport.send(self, request)
def send(self, request):
request.headers['Accept-encoding'] = 'gzip'
result = HttpTransport.send(self, request)
if ('content-encoding' in result.headers and
result.headers['content-encoding'] == 'gzip') :
buf = BytesIO(result.message)
f = gzip.GzipFile(fileobj=buf)
result.message = f.read()
return result
def search(fieldtag, searchterm, ver, count, SID):
'''
Makes a search query to the database.
:param fieldtag: fieldtag must be 'PN = ' or 'CD ='
:param searchterm: search term. in this case, the patent number.
:param ver: version 2 or 3. refer to the thomson reuters documentation.
:param count: requested retrieval count.
:param SID: Session ID.
:return: output xml.
'''
http = HttpTransport()
opener = urllib2.build_opener(HTTPSudsPreprocessor(SID))
http.urlopener = opener
client_obj = Client(SEARCH_URL[int(ver - 2)], transport = http, retxml = True)
query = fieldtag + '=' + searchterm
#construct query and retrieve field parameters
qparams = {
'databaseId' : 'DIIDW',
'userQuery' : query,
'queryLanguage' : 'en',
'editions' : [{
'collection' : 'DIIDW',
'edition' : 'CDerwent',
},{
'collection' : 'DIIDW',
'edition' : 'MDerwent',
},{
'collection' : 'DIIDW',
'edition' : 'EDerwent',
}]
}
rparams = {
'count' : count, # 1-500
'firstRecord' : 1,
'sortField' : [{
'name' : 'Relevance',
'sort' : 'D',
}]
}
result = client_obj.service.search(qparams, rparams)
return result
def __init__(self, **kwargs):
"""
@param kwargs: Keyword arguments.
- B{proxy} - An http proxy to be specified on requests.
The proxy is defined as {protocol:proxy,}
- type: I{dict}
- default: {}
- B{timeout} - Set the url open timeout (seconds).
- type: I{float}
- default: 90
- B{username} - The username used for http authentication.
- type: I{str}
- default: None
- B{password} - The password used for http authentication.
- type: I{str}
- default: None
"""
HttpTransport.__init__(self, **kwargs)
self.pm = u2.HTTPPasswordMgrWithDefaultRealm()
def u2handlers(self):
# try to import ntlm support
try:
from ntlm3 import HTTPNtlmAuthHandler
except ImportError:
raise Exception("Cannot import python-ntlm3 module")
handlers = HttpTransport.u2handlers(self)
handlers.append(HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(
password_mgr=self.pm,header='Negotiate')
)
return handlers
def send(self, request):
request.headers['Accept-encoding'] = 'gzip'
result = HttpTransport.send(self, request)
if result.headers['content-encoding'] == 'gzip':
try:
result.message = gzip.decompress(result.message)
except OSError:
pass
return result
def __init__(self, **kwargs):
"""
Provides a throttled HTTP transport for respecting rate limits
on rate-restricted SOAP APIs using :mod:`suds`.
This class is a :class:`suds.transport.Transport` subclass
based on the default ``HttpAuthenticated`` transport.
:param minimum_spacing: Minimum number of seconds between requests.
Default 0.
:type minimum_spacing: int
.. todo::
Use redis or so to coordinate between threads to allow a
maximum requests per hour/day limit.
"""
self._minumum_spacing = kwargs.pop('minimum_spacing', 0)
self._last_called = int(time.time())
HttpTransport.__init__(self, **kwargs)
def search(query, SID):
url = client = {}
http = HttpTransport()
opener = urllib2.build_opener(HTTPSudsPreprocessor(SID))
http.urlopener = opener
url['search'] = 'http://search.webofknowledge.com/esti/wokmws/ws/WokSearch?wsdl'
client['search'] = Client(url['search'], transport = http)
qparams = {
'databaseId' : 'WOS',
'userQuery' : query,
'queryLanguage' : 'en'
}
rparams = {
'count' : 100, # 1-100
'firstRecord' : 1
}
return client['search'].service.search(qparams, rparams)
def __init__(self, **kwargs):
"""
@param kwargs: Keyword arguments.
- B{proxy} - An http proxy to be specified on requests.
The proxy is defined as {protocol:proxy,}
- type: I{dict}
- default: {}
- B{cache} - The http I{transport} cache. May be set (None) for no caching.
- type: L{Cache}
- default: L{NoCache}
- B{username} - The username used for http authentication.
- type: I{str}
- default: None
- B{password} - The password used for http authentication.
- type: I{str}
- default: None
"""
HttpTransport.__init__(self, **kwargs)
self.pm = u2.HTTPPasswordMgrWithDefaultRealm()
self.handler = u2.HTTPBasicAuthHandler(self.pm)
self.urlopener = u2.build_opener(self.handler)
def retrieve_by_id_lite(input_term, ver, start_record, max_count, SID):
'''
This is a method used for lite services.
:param input_term: input search term.
:param ver: Version 2 or 3. Refer to Thomson Reuters Documentation for details.
:param start_record: starting record point
:param max_count: maximum request count
:param SID: Session ID.
:return: output xml
'''
http = HttpTransport()
opener = urllib2.build_opener(HTTPSudsPreprocessor(SID))
http.urlopener = opener
client_obj = Client(SEARCH_LITE_URL[int(ver - 2)], transport = http, retxml = True)
rparams = {
'count' : max_count, # 1-100
'firstRecord' : start_record,
'sortField' : [{
'name' : 'Relevance',
'sort' : 'D',
}]
}
cited_results = client_obj.service.retrieveById('DIIDW', input_term, 'en', rparams)
return cited_results
class Parser:
""" sax parser """
@classmethod
def saxparser(cls):
p = make_parser()
p.setFeature(feature_external_ges, 0)
h = Handler()
p.setContentHandler(h)
return (p, h)
def __init__(self, transport=None):
if transport is None:
self.transport = HttpTransport()
else:
self.transport = transport
def parse(self, file=None, url=None, string=None):
timer = metrics.Timer()
timer.start()
sax, handler = self.saxparser()
if file is not None:
sax.parse(file)
timer.stop()
metrics.log.debug('sax (%s) duration: %s', file, timer)
return handler.nodes[0]
if url is not None:
fp = self.transport.open(Request(url))
sax.parse(fp)
timer.stop()
metrics.log.debug('sax (%s) duration: %s', url, timer)
return handler.nodes[0]
if string is not None:
source = InputSource(None)
source.setByteStream(StringIO(string))
sax.parse(source)
timer.stop()
metrics.log.debug('%s\nsax duration: %s', string, timer)
return handler.nodes[0]
def send(self, request):
return HttpTransport.send(self, request)
def __init__(self, key, cert, *args, **kwargs):
HttpTransport.__init__(self, *args, **kwargs)
self.key = key
self.cert = cert
def __init__(self, cert, key, options=Options()):
HttpTransport.__init__(self)
self.handler = HTTPSClientAuthHandler(cert, key)
self.urlopener = build_opener(self.handler)
def open(self, request):
return HttpTransport.open(self, request)
def send(self, request):
self.addcredentials(request)
return HttpTransport.send(self, request)
def __init__(self, options):
HttpTransport.__init__(self, options)
self.pm = u2.HTTPPasswordMgrWithDefaultRealm()
self.handler = u2.HTTPBasicAuthHandler(self.pm)
self.urlopener = u2.build_opener(self.handler)
def __init__(self, context, *args, **kwargs):
HttpTransport.__init__(self, *args, **kwargs)
self.context = context
def send(self, request, timeout=None):
self.addcredentials(request)
return HttpTransport.send(self, request, timeout)
def initSearchClient(self):
http = HttpTransport()
opener = urllib2.build_opener(HTTPSudsPreprocessor(self.SID))
http.urlopener = opener
self.client['search'] = Client(self.url['search'], transport=http)
def __init__(self, *args, **kwargs):
HttpTransport.__init__(self, *args, **kwargs)
def open(self, request):
self.addcredentials(request)
return HttpTransport.open(self, request)
def __init__(self, key, cert, **kwargs):
HttpTransport.__init__(self, **kwargs)
self.urlopener = urllib2.build_opener(HTTPSClientAuthHandler(
key, cert))
def __init__(self, key, cert, *args, **kwargs):
HttpTransport.__init__(self, *args, **kwargs)
self.key = key
self.cert = cert
def __init__(self,
server=None,
username=None,
password=None,
wsdl_location="local",
timeout=30,
plugins=[],
sslcontext=None):
self._logged_in = False
if server is None:
server = _config_value("general", "server")
if username is None:
username = _config_value("general", "username")
if password is None:
password = _config_value("general", "password")
if server is None:
raise ConfigError("server must be set in config file or Client()")
if username is None:
raise ConfigError(
"username must be set in config file or Client()")
if password is None:
raise ConfigError(
"password must be set in config file or Client()")
if sslcontext is not None:
self.transport = HTTPSClientContextTransport(sslcontext)
else:
self.transport = HttpTransport()
self.server = server
self.username = username
self.password = password
url = "https://%s/sdk" % self.server
if wsdl_location == "local":
current_path = os.path.abspath(os.path.dirname(__file__))
current_path = current_path.replace('\\', '/')
if not current_path.startswith('/'):
current_path = '/' + current_path
if current_path.endswith('/'):
current_path = current_path[:-1]
wsdl_uri = ("file://%s/wsdl/vimService.wsdl" % current_path)
elif wsdl_location == "remote":
wsdl_uri = url + "/vimService.wsdl"
else:
raise ValueError("wsdl_location must be \"local\" or \"remote\"")
# Init the base class
try:
# Add ExtraConfigPlugin to the plugins
plugins.append(ExtraConfigPlugin())
suds.client.Client.__init__(self,
wsdl_uri,
plugins=plugins,
transport=self.transport)
except URLError:
logger.critical("Failed to connect to %s", self.server)
raise
except IOError:
logger.critical("Failed to load the local WSDL from %s", wsdl_uri)
raise
except TransportError:
logger.critical("Failed to load the remote WSDL from %s", wsdl_uri)
raise
self.options.transport.options.timeout = timeout
self.set_options(location=url, transport=self.transport)
mo_ref = soap.ManagedObjectReference("ServiceInstance",
"ServiceInstance")
self.si = ServiceInstance(mo_ref, self)
try:
self.sc = self.si.RetrieveServiceContent()
except URLError, e:
logger.critical("Failed to connect to %s" % self.server)
logger.critical("urllib2 said: %s" % e.reason)
raise
def initSearchClient(self):
http = HttpTransport()
opener = urllib2.build_opener(HTTPSudsPreprocessor(self.SID))
http.urlopener = opener
self.client['search'] = Client(self.url['search'], transport = http)
def __init__(self, context, *args, **kwargs):
HttpTransport.__init__(self, *args, **kwargs)
self.context = context
def __init__(self,options,config_file=None,session="DWE"):
"""Datastream SOAP Client
>>> ds=Datastream()
"""
_set_debug(logging.WARN,
'suds.resolver',
'suds.metrics',
'suds.xsd.sxbasic',
'suds.xsd.query',
'suds.client',
'suds.wsdl',
'suds.transport.http',
'suds.mx.core',
'suds.umx.typed',
'suds.xsd.schema',
'suds.xsd.sxbase',
'suds.mx.literal')
if config_file is None:
if exists(expanduser('~/.dwe.conf')):
config_file = expanduser('~/.dwe.conf')
os.chmod(config_file,stat.S_IRUSR | stat.S_IWUSR)
else:
config_file = DEFAULT_DSTREAM_CONF_FILE
config_file = expanduser(expandvars(config_file))
self._options = options
if not exists(config_file):
LOGGER.info('no configuration for datastream (%s). using BITS path',config_file)
config_file = join(BITS_ETC,DEFAULT_BITS_CONF_FILE)
self.config_file = config_file
if not exists(self.config_file):
raise IOError, 'Datastream Configuration File not Found %s' % config_file
dstream_info = get_session(config_file,session,
secure=True,
url='http://dataworks.thomson.com/dataworks/enterprise/1.0/webServiceClient.asmx?WSDL',
user=(None,ValueError),
password=(None,ValueError),
realm=''
)
self._auth=dstream_info
t = HttpTransport()
### Proxy if any
#
proxy_info = get_proxy()
if proxy_info:
pip={
'http':proxy_info['proxy'],
'https':proxy_info['proxy'],
}
proxy = urllib2.ProxyHandler(pip)
# opener = urllib2.build_opener(SocksiPyHandler(socks.PROXY_TYPE_SOCKS4, 'localhost', 8080))
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
t.urlopener = opener
LOGGER.debug('proxy info for http: %s',pip['http'])
#
#
###
try:
Client.__init__(self,self._auth['url'],transport=t)
except urllib2.URLError, exc:
LOGGER.error('{}: No connection to {}'.format(self._options.job,self._auth['url']))
raise exc
def send(self, request):
return HttpTransport.send(self, request)
def open(self, request):
self.addcredentials(request)
return HttpTransport.open(self, request)
def u2handlers(self):
r = HttpTransport.u2handlers(self)
r.append(HTTPSHandler())
return r
def u2handlers(self):
handlers = HttpTransport.u2handlers(self)
handlers.append(u2.HTTPBasicAuthHandler(self.pm))
return handlers
def send(self, request):
self.addcredentials(request)
return HttpTransport.send(self, request)
def __init__(self, transport=None):
if transport is None:
self.transport = HttpTransport()
else:
self.transport = transport
def u2handlers(self):
handlers = HttpTransport.u2handlers(self)
handlers.append(urllib.request.HTTPBasicAuthHandler(self.pm))
return handlers
