class streamer(object): def __init__(self): """ Reads in credentials from the user's .netrc file and creates a Datastream object to be used in fetching data. """ rc = netrc() uname, account, passwd = rc.authenticators('datastream') self.DWE = Datastream(username=uname, password=passwd) def _fetch_individual_code(self, code, fields=None, **kwargs): """ Internal function to fetch the data. It offers two advantages to the original: (1) It raises a warning, not an Exception when it has problems. This is primarily useful when you have a list of codes and want to know when one fails, but move on. (2) It renames the colums so that they have the code as a part of the column name, not just the field. This """ print(code) #TEMP try: df = self.DWE.fetch(code, fields=fields, **kwargs) except Exception, e: df = None warnings.warn(("Unable to load {} \n".format(code) + str(e.message) + '\n')) else:
DATATYPES = ['P', 'MV', 'PE', 'VA', 'VO', 'PTBV', 'MTBV'] shenzhen_equities_symbols = [] with open('shenzhen-equities-rmb.csv', 'r') as shenzhen_equities_csv: reader = csv.DictReader(shenzhen_equities_csv) shenzhen_equities_symbols = [line['Symbol'] for line in reader] for stock_mnem in shenzhen_equities_symbols: filename = 'stock_%s_%s.csv' % ('shenzhen', stock_mnem) filename = filename.replace(':', '-') if os.path.isfile('csvs/%s' % filename): print 'Stock data already exists for %s' % (stock_mnem) continue print 'Querying stocks data for %s' % (stock_mnem) try: stock_data = DWE.fetch([stock_mnem], DATATYPES, date_from='1990-01-01', freq='W') filename = 'stock_%s_%s.csv' % ('shenzhen', stock_mnem) filename = filename.replace(':', '-') print 'Writing %s' % filename stock_data.to_csv('csvs/%s' % (filename)) except KeyError, e: print 'Key Error occured for %s_%s' % ('shenzhen', stock_mnem) except pydatastream.pydatastream.DatastreamException, e: print 'DatastreamException occured for %s_%s' % ('shenzhen', stock_mnem) shenzhen_equities_inactive_dates = DWE.fetch(shenzhen_equities_symbols, INACTIVE_DATE, static=True) shenzhen_equities_inactive_dates.to_csv('csvs/shenzhen_equities_inactive_dates.csv')
custom_fields = list() for code_key,code_value in group_value.iteritems(): for key,value in code_value.iteritems(): if(key == 'code'): search_code = value search_symbol = {'Custom_Ticker' : value} if(key == 'start_date'): start_date = value if(key == 'custom_field'): custom_fields[:] = [] custom_fields.append(value) startTime = datetime.datetime.now() #send request to retrieve the data from Datastream req = DWE.fetch(str(search_code),custom_fields,date_from=str(start_date),only_data=False) time_taken = time_taken + datetime.datetime.now() - startTime #format date and convert to json raw_json = req[0].to_json(date_format='iso') raw_metadata = req[1].to_json() #Data cleaning and processing #remove the time component including the '.' char from the key values of datetime in the data raw_json = raw_json.replace("T00:00:00.000Z","") #replace the metadata's keys from "0" to "default_ws_key" raw_metadata = raw_metadata.replace("\"0\"","\"Custom_WS_Key\"") #combine the data and the metadata about the code allData_str = json.loads(raw_json)