class streamer(object):
def __init__(self):
"""
Reads in credentials from the user's .netrc file and creates
a Datastream object to be used in fetching data.
"""
rc = netrc()
uname, account, passwd = rc.authenticators('datastream')
self.DWE = Datastream(username=uname, password=passwd)
def _fetch_individual_code(self, code, fields=None, **kwargs):
"""
Internal function to fetch the data.
It offers two advantages to the original:
(1) It raises a warning, not an Exception when it has problems.
This is primarily useful when you have a list of codes and
want to know when one fails, but move on.
(2) It renames the colums so that they have the code as a part
of the column name, not just the field. This
"""
print(code) #TEMP
try:
df = self.DWE.fetch(code, fields=fields, **kwargs)
except Exception, e:
df = None
warnings.warn(("Unable to load {} \n".format(code)
+ str(e.message) + '\n'))
else:
DATATYPES = ['P', 'MV', 'PE', 'VA', 'VO', 'PTBV', 'MTBV']
shenzhen_equities_symbols = []
with open('shenzhen-equities-rmb.csv', 'r') as shenzhen_equities_csv:
reader = csv.DictReader(shenzhen_equities_csv)
shenzhen_equities_symbols = [line['Symbol'] for line in reader]
for stock_mnem in shenzhen_equities_symbols:
filename = 'stock_%s_%s.csv' % ('shenzhen', stock_mnem)
filename = filename.replace(':', '-')
if os.path.isfile('csvs/%s' % filename):
print 'Stock data already exists for %s' % (stock_mnem)
continue
print 'Querying stocks data for %s' % (stock_mnem)
try:
stock_data = DWE.fetch([stock_mnem], DATATYPES, date_from='1990-01-01', freq='W')
filename = 'stock_%s_%s.csv' % ('shenzhen', stock_mnem)
filename = filename.replace(':', '-')
print 'Writing %s' % filename
stock_data.to_csv('csvs/%s' % (filename))
except KeyError, e:
print 'Key Error occured for %s_%s' % ('shenzhen', stock_mnem)
except pydatastream.pydatastream.DatastreamException, e:
print 'DatastreamException occured for %s_%s' % ('shenzhen', stock_mnem)
shenzhen_equities_inactive_dates = DWE.fetch(shenzhen_equities_symbols, INACTIVE_DATE, static=True)
shenzhen_equities_inactive_dates.to_csv('csvs/shenzhen_equities_inactive_dates.csv')
custom_fields = list()
for code_key,code_value in group_value.iteritems():
for key,value in code_value.iteritems():
if(key == 'code'):
search_code = value
search_symbol = {'Custom_Ticker' : value}
if(key == 'start_date'):
start_date = value
if(key == 'custom_field'):
custom_fields[:] = []
custom_fields.append(value)
startTime = datetime.datetime.now()
#send request to retrieve the data from Datastream
req = DWE.fetch(str(search_code),custom_fields,date_from=str(start_date),only_data=False)
time_taken = time_taken + datetime.datetime.now() - startTime
#format date and convert to json
raw_json = req[0].to_json(date_format='iso')
raw_metadata = req[1].to_json()
#Data cleaning and processing
#remove the time component including the '.' char from the key values of datetime in the data
raw_json = raw_json.replace("T00:00:00.000Z","")
#replace the metadata's keys from "0" to "default_ws_key"
raw_metadata = raw_metadata.replace("\"0\"","\"Custom_WS_Key\"")
#combine the data and the metadata about the code
allData_str = json.loads(raw_json)