def read_from_file(self,fname):
global LINENO,LINE
if exists(fname):
# logger.debug('reading from file %s',str(fname))
for n,line in enumerate(codecs.open(fname,'r','utf-8').readlines()):
LINENO=n
LINE = unicode(stripcomments(line).strip('\n '))
if re.match('^[a-z0-9]+://.+$',LINE,re.I):
self.append(LINE)
def append(self, url):
"""
Append a data request URL to the list
>>> dp = DataProvider()
>>> dp.append('dstream://Datastream/AGINDEMIF')
>>> dp.append('dstream://Datastream/PCH#(AGINDEMIF, 1Y)')
>>> dp.append('option://param/TEST?VALUE')
"""
url = stripcomments(url.rstrip('\n ')) # .upper()
if len(url) == 0:
return
# Needs double parsing because urlparse function
# cant parse params (?...&...)
# if schema is not http:
# management of # character
URL = url.replace('#', '__A~~~~A__')
# first pass of parsing ot get scheme
up = urlparse(URL)
# substitute parameters when scheme is not option (CHECK)
if not re.search('^option$', up.scheme, re.IGNORECASE) and '$' in url:
url = Template(url).safe_substitute(self._options.define_set)
# reset #
url = url.replace('#', '__A~~~~A__')
# ...second urlparse using a fake http: schema
h_url = re.sub("^%s://" % up.scheme, "http://", url, flags=re.I)
h_pa = urlparse(h_url)
parsed = list(h_pa)
parsed[0] = unicode(up.scheme)
# A = parsed[2]
parsed[2] = parsed[2].replace('__A~~~~A__', '#') # ?!?
parsed[2] = parsed[2].replace('\\', '/')
# for option URI insert in the option list
if re.search('^option$', up.scheme, re.I):
self._append_option(parsed)
return
#LOGGER.debug('%s (%d)',
# parsed,
# len(self.requesting))
# otherwise add to requesting list
self.requesting.append(parsed)
