Ejemplo n.º 1
0
class streamer(object):
    def __init__(self):
        """
        Reads in credentials from the user's .netrc file and creates
        a Datastream object to be used in fetching data.
        """
        rc = netrc()
        uname, account, passwd = rc.authenticators('datastream')
        self.DWE = Datastream(username=uname, password=passwd)

    def _fetch_individual_code(self, code, fields=None, **kwargs):
        """
        Internal function to fetch the data.
        It offers two advantages to the original:
            (1) It raises a warning, not an Exception when it has problems.
                This is primarily useful when you have a list of codes and
                want to know when one fails, but move on.
            (2) It renames the colums so that they have the code as a part
                of the column name, not just the field.  This 
        """
        print(code) #TEMP
        try:
            df = self.DWE.fetch(code, fields=fields, **kwargs)
        except Exception, e:
            df = None
            warnings.warn(("Unable to load {} \n".format(code) 
                + str(e.message) + '\n'))
        else:
DATATYPES = ['P', 'MV', 'PE', 'VA', 'VO', 'PTBV', 'MTBV']

shenzhen_equities_symbols = []

with open('shenzhen-equities-rmb.csv', 'r') as shenzhen_equities_csv:
    reader = csv.DictReader(shenzhen_equities_csv)
    shenzhen_equities_symbols = [line['Symbol'] for line in reader]

    for stock_mnem in shenzhen_equities_symbols:
      filename = 'stock_%s_%s.csv' % ('shenzhen', stock_mnem)
      filename = filename.replace(':', '-')

      if os.path.isfile('csvs/%s' % filename):
        print 'Stock data already exists for %s' % (stock_mnem)
        continue

      print 'Querying stocks data for %s' % (stock_mnem)
      try:
        stock_data = DWE.fetch([stock_mnem], DATATYPES, date_from='1990-01-01', freq='W')
        filename = 'stock_%s_%s.csv' % ('shenzhen', stock_mnem)
        filename = filename.replace(':', '-')
        print 'Writing %s' % filename
        stock_data.to_csv('csvs/%s' % (filename))
      except KeyError, e:
        print 'Key Error occured for %s_%s' % ('shenzhen', stock_mnem)
      except pydatastream.pydatastream.DatastreamException, e:
        print 'DatastreamException occured for %s_%s' % ('shenzhen', stock_mnem)

shenzhen_equities_inactive_dates = DWE.fetch(shenzhen_equities_symbols, INACTIVE_DATE, static=True)
shenzhen_equities_inactive_dates.to_csv('csvs/shenzhen_equities_inactive_dates.csv')
Ejemplo n.º 3
0
		custom_fields = list()
		for code_key,code_value in group_value.iteritems():
		
			for key,value in code_value.iteritems():
				if(key == 'code'):
					search_code = value
					search_symbol = {'Custom_Ticker' : value}
				if(key == 'start_date'):
					start_date = value
				if(key == 'custom_field'):
					custom_fields[:] = []
					custom_fields.append(value)

			startTime = datetime.datetime.now()	
			#send request to retrieve the data from Datastream
			req = DWE.fetch(str(search_code),custom_fields,date_from=str(start_date),only_data=False)
			
			time_taken = time_taken + datetime.datetime.now() - startTime
							
			#format date and convert to json 
			raw_json = req[0].to_json(date_format='iso')
			raw_metadata = req[1].to_json()
			
			#Data cleaning and processing
			#remove the time component including the '.' char from the key values of datetime in the data
			raw_json = raw_json.replace("T00:00:00.000Z","")
			#replace the metadata's keys from "0" to "default_ws_key"
			raw_metadata = raw_metadata.replace("\"0\"","\"Custom_WS_Key\"")
			
			#combine the data and the metadata about the code
			allData_str = json.loads(raw_json)