def __init__(self,config_file=None): """Excel Provider Client >>> ds=Excel() """ ### Proxy if any # proxy_info = get_proxy() if proxy_info: pip={ 'http':proxy_info['proxy'], 'https':proxy_info['proxy'], } proxy = urllib2.ProxyHandler(pip) opener = urllib2.build_opener(proxy) urllib2.install_opener(opener)
def __init__(self,options=Options()): """Excel Provider Client >>> ds=Excel() """ self._options = options ### Proxy if any # proxy_info = get_proxy() if proxy_info: pip={ 'http':proxy_info['proxy'], 'https':proxy_info['proxy'], } proxy = urllib2.ProxyHandler(pip) opener = urllib2.build_opener(proxy) urllib2.install_opener(opener)
def __init__(self,config_file=None,session="FRED"): """FRED REST Client >>> fr=FRED() """ self.config_file = discover_config_file(self._name, config_file, '~/.fred.conf', DEFAULT_FRED_CONF_FILE, join(BITS_ETC, DEFAULT_BITS_CONF_FILE)) fred_info = get_session(config_file,session, secure=True, url='http://api.stlouisfed.org/fred', api_key=(None,ValueError), ) self._auth=fred_info ### Proxy if any # proxy_info = get_proxy() if proxy_info: pip={ 'http':proxy_info['proxy'], 'https':proxy_info['proxy'], } proxy = urllib2.ProxyHandler(pip) opener = urllib2.build_opener(proxy) urllib2.install_opener(opener) # # ### Fred.__init__(self,self._auth['api_key'])
def execute(self): logger.debug('begin') _newds = self._dataset ### Proxy if any # proxy_info = get_proxy() if proxy_info: pip={ 'http':proxy_info['proxy'], 'https':proxy_info['proxy'], } proxy = urllib2.ProxyHandler(pip) opener = urllib2.build_opener(proxy) urllib2.install_opener(opener) page = urllib2.urlopen(self._url) logger.debug('Got %s',self._url) soup = BeautifulSoup(page) ### Questo blocco cerca la tabella tab2a tTags = soup.findAll('table') found = False for i,tTag in enumerate(tTags): #print '====',i,str(tTag.tr.td.text)[0:200] if re.match('^Table %s' % self._table,str(tTag.tr.td.text),re.I): found=True break # for tTag in tTags: #tA = tTd.findNext('a') #tAv = tTag.find(attrs={'name':self._table}) #if tAv: # found=True # break # print "--------------------" if not found: logger.error('Table %s not in page',self._table) return raise ValueError, "%s not found" % self._table tTrs = tTag.findAll('tr') # print ">>>>>>>>" i = 0 inBlock = False res = [] for tTr in tTrs: tTds = tTr.findAll('td') v = [] for i,td in enumerate(tTds): txt = td.text if txt != ' ': if re.match('^--$',txt): txt = '' elif re.match(',',txt): txt = txt.replace(',','') elif i==0 and re.match('^.+([0-9]\/?)$',txt): m = re.match('^.+([0-9]\/?)$',txt) m1 = m.group(1) txt = txt[:-len(m1)] elif re.match('^\*.+$',txt): while txt[0]!='*': txt = txt[1:] v.append(txt) res.append(v) i += 1 titles = [ 'Stand-by Arrangements', 'Extended Arrangements', 'Flexible Credit Line', 'Precautionary' ] ttl = titles.pop(0) lk = -1 v = {} t = {} for l in res: if len(l)==1 and re.match("^%s"%ttl,l[0],re.I): lk += 1 if len(titles)>0: ttl = titles.pop(0) logger.debug('Next Title %s',ttl) elif len(l)==7: if not v.has_key(lk): v[lk]=[] v[lk].append(l) elif len(l)==4 and re.search('Arrangements?$',l[0]): m = re.match('^([0-9]+) Arrangements?$',l[0]) if m: npx = m.group(1) npt = False if not t.has_key(lk): t[lk]=[] if len(v[lk])==int(npx): logger.debug("Wow right arrangement found") npt=True l.append(npx) l.append(npt) t[lk].extend(l) else: if len(l)>0: logger.debug('LINE NOT INSERTED %d > %s', len(l), l) N = np.sum([ int(x[4]) for k,x in t.items()]) # print "ROWS ARE=",N # Make the result vectors # _ds = DataSet() l = 0 for k,w in sorted(v.items()): for i,n in enumerate( ('MEMBER', 'EFDATE', 'EXDATE', 'AGREED', 'UNDRAWN', 'OUTSTANDING', 'POQ', ) ): name = "%s_%d" % (n,k) if not _ds.has_key(name): _ds[name]=[] for m in w: # print "WL=",m if n=='POQ' and len(m[i])>0: _ds[name].append(m[i]+"\\%") else: _ds[name].append(m[i].lstrip('*')) l += 1 # pprint(_ds) return _ds
def __init__(self,options,config_file=None,session="DWE"): """Datastream SOAP Client >>> ds=Datastream() """ _set_debug(logging.WARN, 'suds.resolver', 'suds.metrics', 'suds.xsd.sxbasic', 'suds.xsd.query', 'suds.client', 'suds.wsdl', 'suds.transport.http', 'suds.mx.core', 'suds.umx.typed', 'suds.xsd.schema', 'suds.xsd.sxbase', 'suds.mx.literal') if config_file is None: if exists(expanduser('~/.dwe.conf')): config_file = expanduser('~/.dwe.conf') os.chmod(config_file,stat.S_IRUSR | stat.S_IWUSR) else: config_file = DEFAULT_DSTREAM_CONF_FILE config_file = expanduser(expandvars(config_file)) self._options = options if not exists(config_file): LOGGER.info('no configuration for datastream (%s). using BITS path',config_file) config_file = join(BITS_ETC,DEFAULT_BITS_CONF_FILE) self.config_file = config_file if not exists(self.config_file): raise IOError, 'Datastream Configuration File not Found %s' % config_file dstream_info = get_session(config_file,session, secure=True, url='http://dataworks.thomson.com/dataworks/enterprise/1.0/webServiceClient.asmx?WSDL', user=(None,ValueError), password=(None,ValueError), realm='' ) self._auth=dstream_info t = HttpTransport() ### Proxy if any # proxy_info = get_proxy() if proxy_info: pip={ 'http':proxy_info['proxy'], 'https':proxy_info['proxy'], } proxy = urllib2.ProxyHandler(pip) # opener = urllib2.build_opener(SocksiPyHandler(socks.PROXY_TYPE_SOCKS4, 'localhost', 8080)) opener = urllib2.build_opener(proxy) urllib2.install_opener(opener) t.urlopener = opener LOGGER.debug('proxy info for http: %s',pip['http']) # # ### try: Client.__init__(self,self._auth['url'],transport=t) except urllib2.URLError, exc: LOGGER.error('{}: No connection to {}'.format(self._options.job,self._auth['url'])) raise exc