def __init__(self,config_file=None):
"""Excel Provider Client
>>> ds=Excel()
"""
### Proxy if any
#
proxy_info = get_proxy()
if proxy_info:
pip={
'http':proxy_info['proxy'],
'https':proxy_info['proxy'],
}
proxy = urllib2.ProxyHandler(pip)
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
def __init__(self,options=Options()):
"""Excel Provider Client
>>> ds=Excel()
"""
self._options = options
### Proxy if any
#
proxy_info = get_proxy()
if proxy_info:
pip={
'http':proxy_info['proxy'],
'https':proxy_info['proxy'],
}
proxy = urllib2.ProxyHandler(pip)
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
def __init__(self,config_file=None,session="FRED"):
"""FRED REST Client
>>> fr=FRED()
"""
self.config_file = discover_config_file(self._name,
config_file,
'~/.fred.conf',
DEFAULT_FRED_CONF_FILE,
join(BITS_ETC,
DEFAULT_BITS_CONF_FILE))
fred_info = get_session(config_file,session,
secure=True,
url='http://api.stlouisfed.org/fred',
api_key=(None,ValueError),
)
self._auth=fred_info
### Proxy if any
#
proxy_info = get_proxy()
if proxy_info:
pip={
'http':proxy_info['proxy'],
'https':proxy_info['proxy'],
}
proxy = urllib2.ProxyHandler(pip)
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
#
#
###
Fred.__init__(self,self._auth['api_key'])
def execute(self):
logger.debug('begin')
_newds = self._dataset
### Proxy if any
#
proxy_info = get_proxy()
if proxy_info:
pip={
'http':proxy_info['proxy'],
'https':proxy_info['proxy'],
}
proxy = urllib2.ProxyHandler(pip)
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
page = urllib2.urlopen(self._url)
logger.debug('Got %s',self._url)
soup = BeautifulSoup(page)
### Questo blocco cerca la tabella tab2a
tTags = soup.findAll('table')
found = False
for i,tTag in enumerate(tTags):
#print '====',i,str(tTag.tr.td.text)[0:200]
if re.match('^Table %s' % self._table,str(tTag.tr.td.text),re.I):
found=True
break
# for tTag in tTags:
#tA = tTd.findNext('a')
#tAv = tTag.find(attrs={'name':self._table})
#if tAv:
# found=True
# break
# print "--------------------"
if not found:
logger.error('Table %s not in page',self._table)
return
raise ValueError, "%s not found" % self._table
tTrs = tTag.findAll('tr')
# print ">>>>>>>>"
i = 0
inBlock = False
res = []
for tTr in tTrs:
tTds = tTr.findAll('td')
v = []
for i,td in enumerate(tTds):
txt = td.text
if txt != ' ':
if re.match('^--$',txt):
txt = ''
elif re.match(',',txt):
txt = txt.replace(',','')
elif i==0 and re.match('^.+([0-9]\/?)$',txt):
m = re.match('^.+([0-9]\/?)$',txt)
m1 = m.group(1)
txt = txt[:-len(m1)]
elif re.match('^\*.+$',txt):
while txt[0]!='*':
txt = txt[1:]
v.append(txt)
res.append(v)
i += 1
titles = [ 'Stand-by Arrangements',
'Extended Arrangements',
'Flexible Credit Line',
'Precautionary' ]
ttl = titles.pop(0)
lk = -1
v = {}
t = {}
for l in res:
if len(l)==1 and re.match("^%s"%ttl,l[0],re.I):
lk += 1
if len(titles)>0:
ttl = titles.pop(0)
logger.debug('Next Title %s',ttl)
elif len(l)==7:
if not v.has_key(lk):
v[lk]=[]
v[lk].append(l)
elif len(l)==4 and re.search('Arrangements?$',l[0]):
m = re.match('^([0-9]+) Arrangements?$',l[0])
if m:
npx = m.group(1)
npt = False
if not t.has_key(lk):
t[lk]=[]
if len(v[lk])==int(npx):
logger.debug("Wow right arrangement found")
npt=True
l.append(npx)
l.append(npt)
t[lk].extend(l)
else:
if len(l)>0:
logger.debug('LINE NOT INSERTED %d > %s', len(l), l)
N = np.sum([ int(x[4]) for k,x in t.items()])
# print "ROWS ARE=",N
# Make the result vectors
#
_ds = DataSet()
l = 0
for k,w in sorted(v.items()):
for i,n in enumerate( ('MEMBER',
'EFDATE',
'EXDATE',
'AGREED',
'UNDRAWN',
'OUTSTANDING',
'POQ',
) ):
name = "%s_%d" % (n,k)
if not _ds.has_key(name):
_ds[name]=[]
for m in w:
# print "WL=",m
if n=='POQ' and len(m[i])>0:
_ds[name].append(m[i]+"\\%")
else:
_ds[name].append(m[i].lstrip('*'))
l += 1
# pprint(_ds)
return _ds
def __init__(self,options,config_file=None,session="DWE"):
"""Datastream SOAP Client
>>> ds=Datastream()
"""
_set_debug(logging.WARN,
'suds.resolver',
'suds.metrics',
'suds.xsd.sxbasic',
'suds.xsd.query',
'suds.client',
'suds.wsdl',
'suds.transport.http',
'suds.mx.core',
'suds.umx.typed',
'suds.xsd.schema',
'suds.xsd.sxbase',
'suds.mx.literal')
if config_file is None:
if exists(expanduser('~/.dwe.conf')):
config_file = expanduser('~/.dwe.conf')
os.chmod(config_file,stat.S_IRUSR | stat.S_IWUSR)
else:
config_file = DEFAULT_DSTREAM_CONF_FILE
config_file = expanduser(expandvars(config_file))
self._options = options
if not exists(config_file):
LOGGER.info('no configuration for datastream (%s). using BITS path',config_file)
config_file = join(BITS_ETC,DEFAULT_BITS_CONF_FILE)
self.config_file = config_file
if not exists(self.config_file):
raise IOError, 'Datastream Configuration File not Found %s' % config_file
dstream_info = get_session(config_file,session,
secure=True,
url='http://dataworks.thomson.com/dataworks/enterprise/1.0/webServiceClient.asmx?WSDL',
user=(None,ValueError),
password=(None,ValueError),
realm=''
)
self._auth=dstream_info
t = HttpTransport()
### Proxy if any
#
proxy_info = get_proxy()
if proxy_info:
pip={
'http':proxy_info['proxy'],
'https':proxy_info['proxy'],
}
proxy = urllib2.ProxyHandler(pip)
# opener = urllib2.build_opener(SocksiPyHandler(socks.PROXY_TYPE_SOCKS4, 'localhost', 8080))
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
t.urlopener = opener
LOGGER.debug('proxy info for http: %s',pip['http'])
#
#
###
try:
Client.__init__(self,self._auth['url'],transport=t)
except urllib2.URLError, exc:
LOGGER.error('{}: No connection to {}'.format(self._options.job,self._auth['url']))
raise exc
