Beispiel #1
0
def bloomberg_data_query(response_key, prepared_entries, use_terminal):
    cache.set(response_key, 0.0)
    all_fields = BloombergDataContainerMapping.objects.all().values_list('short_name__code', flat=True)
    response = BloombergTasks.send_bloomberg_get_data(prepared_entries, ticker_type='TICKER', use_terminal=use_terminal, fields=all_fields)
    cache.set('data_' + response_key, response)
    cache.set('type_' + response_key, 'securities')
    cache.set(response_key, 0.5)
    securities, final_tickers, errors = populate_security_from_bloomberg_protobuf(response)
    result = []
    
    new_securities_count = 0
    
    for security in securities.keys():
        with_isin = []
        with_bloomberg = []
        
        isin_field = securities[security].aliases.filter(alias_type__name='ISIN')
        isin_code = None
        if isin_field.exists():
            with_isin = sorted(securities[security].__class__.objects.filter(aliases__alias_type__name='ISIN', aliases__alias_value=isin_field[0].alias_value), key=lambda x: x.id)
            isin_code = isin_field[0].alias_value
            
        bloomberg_field = securities[security].aliases.filter(alias_type__name='BLOOMBERG')
        bloomberg_code = None
        if bloomberg_field.exists():
            with_bloomberg = sorted(securities[security].__class__.objects.filter(aliases__alias_type__name='BLOOMBERG', aliases__alias_value=bloomberg_field[0].alias_value), key=lambda x: x.id)
            bloomberg_code = bloomberg_field[0].alias_value
            
        if len(with_bloomberg)>1:
            securities[security].delete()
            result.append(with_bloomberg[0])
        elif len(with_isin)>1 and (bloomberg_code==None or bloomberg_code==isin_code):
            securities[security].delete()
            result.append(with_isin[0])
        else:
            new_securities_count += 1
            result.append(securities[security])
    cache.set('securities_' + response_key, result)
    cache.set('errors_' + response_key, errors)
    cache.set(response_key, 1.0)
    # Getting all kind of containers
    all_containers = {}
    for security in result:
        if not all_containers.has_key(security.type.identifier):
            all_containers[security.type.identifier] = []
        try:
            all_containers[security.type.identifier].append(security.aliases.get(alias_type__name='BLOOMBERG').alias_value)
        except:
            all_containers[security.type.identifier].append(security.aliases.get(alias_type__name='ISIN').alias_value)
    
    for key in all_containers.keys():
        fields = BloombergTrackContainerMapping.objects.filter(Q(container__identifier='CONT_SECURITY') | Q(container__identifier=key), Q(active=True)).values_list('short_name__code', flat=True)
        all_containers[key] = [to_bloomberg_code(ticker,use_terminal) for ticker in all_containers[key]]
        history_key = uuid.uuid4().get_hex()
        bb_thread = threading.Thread(None, bloomberg_history_query, history_key, (history_key, all_containers[key], fields, True))
        bb_thread.start()
        bb_thread.join()
Beispiel #2
0
def bloomberg_wizard_execute(request, entity):
    # TODO: Check user
    # TODO: Check Bloomberg method
    entries = [str(entry).strip() for entry in request.POST['bloombergList'].split('\r')]
    try:
        use_terminal = request.POST['bloombergSource']=='True'
    except:
        use_terminal = False
    prepared_entries = [to_bloomberg_code(entry,use_terminal) for entry in entries]

    response_key = uuid.uuid4().get_hex()
    # TODO: Implement CONSTANTS and dynamic choice
    if entity=='financials':
        bb_thread = threading.Thread(None, bloomberg_data_query, response_key, (response_key, prepared_entries, use_terminal))
    bb_thread.start()
    context = {'response_key': response_key}
    return render(request,entity + '/bloomberg/wizard_waiting.html', context)
Beispiel #3
0
def bloomberg_update(request):
    # TODO: Check user
    # TODO: Check Bloomberg method (Terminal/DL)
    bloomberg_company = CompanyContainer.objects.get(name='Bloomberg LP')
    bloomberg_fields = {entry[0]: entry[1] for entry in BloombergTrackContainerMapping.objects.values_list('name__name', 'short_name__name')}
    try:
        universe_id = request.GET['universe_id']
    except:
        universe_id = None
    if universe_id!=None:
        universe = Universe.objects.filter(Q(id=universe_id), Q(public=True)|Q(owner__id=request.user.id))
        if universe.exists():
            universe = universe[0]
            all_tracks = TrackContainer.objects.filter(effective_container_id__in=universe.members.all().values_list('id', flat=True), source__id=bloomberg_company.id).order_by('end_date')
    else:
        all_tracks = TrackContainer.objects.filter(source__id=bloomberg_company.id).order_by('end_date')
    bulk_information = {}
    count = 0
    for track in all_tracks:
        if track.end_date==None or dates.AddDay(track.end_date,1)<datetime.datetime.combine(datetime.date.today(), datetime.datetime.min.time()):
            if track.end_date!=None:
                key = dates.AddDay(track.end_date,1)
            else:
                key = 'None'
            if bloomberg_fields.has_key(track.type.name):
                track_field = bloomberg_fields[track.type.name]
                if not bulk_information.has_key(track_field):
                    bulk_information[track_field] = {}
                if not bulk_information[track_field].has_key(key):
                    bulk_information[track_field][key] = []
                try:
                    bb_code = track.effective_container.aliases.get(alias_type__name='BLOOMBERG').alias_value
                    bulk_information[track_field][key].append(to_bloomberg_code(bb_code, True))
                    count += 1
                except:
                    #traceback.print_exc()
                    LOGGER.error("No associated BLOOMBERG information for " + str(track.effective_container.name))
    if count>0:
        history_key = uuid.uuid4().get_hex()
        update_thread = threading.Thread(None, bloomberg_update_query, history_key, (history_key, bulk_information, True))
        update_thread.start()
    # TODO: Return error message
    return redirect('universes.html')
def import_external_data(data_source, data_type):
    LOGGER.info('Loading working data')
    query = QUERIES[data_source][data_type]['query']
    group_by = QUERIES[data_source][data_type]['group_by']
    external_alias = Attributes.objects.get(identifier='ALIAS_' + data_source.upper())
    
    if data_type=='securities':
        bloomberg_alias = Attributes.objects.get(identifier='ALIAS_BLOOMBERG')
        isin_alias = Attributes.objects.get(identifier='ALIAS_ISIN')
        sec_container_type = Attributes.objects.get(identifier='CONT_SECURITY')
        fund_container_type = Attributes.objects.get(identifier='CONT_FUND')
        bond_container_type = Attributes.objects.get(identifier='CONT_BOND')
        stock_container_type = Attributes.objects.get(identifier='CONT_SECURITY')
        security_type = Attributes.objects.get(identifier='SECTYP_SECURITY')
        fund_type = Attributes.objects.get(identifier='SECTYP_FUND')
        bond_type = Attributes.objects.get(identifier='SECTYP_BOND')
        stock_type = Attributes.objects.get(identifier='SECTYP_STOCK')

        daily = Attributes.objects.get(identifier='FREQ_DAILY', active=True)
        
        bloomberg_provider = get_bloomberg_provider()
        
        universe = get_universe_from_datasource(data_source)
        
        LOGGER.info('Cleaning already imported aliases for ' + data_source)
        securities = SecurityContainer.objects.filter(aliases__alias_type__short_name=data_source.upper()).distinct()
        for security in securities:
            for alias in security.aliases.all():
                if alias.alias_type.short_name==data_source.upper():
                    security.aliases.remove(alias)
            security.save()
        LOGGER.info('\tCleaning done')
    
    LOGGER.info('Importing ' + str(data_type) + ' from ' + str(data_source))
    LOGGER.info('Using query:' + str(query))
    results = dbutils.query_to_dicts(query, data_source)
    database = getattr(client, data_source)
    database[data_type].drop()
    
    all_tickers = []
    
    for result in results:
        # Clean the data        
        new_entry = convert_to_mongo(result)
        # LOGGER.info('Removing entries with name '  + new_entry[QUERIES[data_source][data_type]['name']])
        # SecurityContainer.objects.filter(name=new_entry[QUERIES[data_source][data_type]['name']]).delete()
        for group_id in group_by:
            if new_entry[group_id]!=None and new_entry[group_id]!='':
                # LOADING INTO MONGO
                LOGGER.info('Adding entry [' + group_id + '=' + str(new_entry[group_id]) + "]")
                new_entry['_id'] = new_entry[group_id]
                if QUERIES[data_source][data_type].has_key('joins'):
                    for join_info in QUERIES[data_source][data_type]['joins']:
                        values = [new_entry[identifier] for identifier in join_info['on']]
                        underlying_query = join_info['query']
                        for value in values:
                            underlying_query = underlying_query%value
                        new_entry[join_info['name']] = []
                        under_results = dbutils.query_to_dicts(underlying_query, data_source)
                        for under_result in under_results:
                            LOGGER.info('\tAdding underlying [' + join_info['name'] + ', ' + group_id + '=' + str(new_entry[group_id]) + "]")
                            # Clean the data        
                            under_entry = convert_to_mongo(under_result)
                            new_entry[join_info['name']].append(under_entry)
                try:
                    database[data_type].save(new_entry)
                except DuplicateKeyError:
                    LOGGER.error("The following entry already exists:")
            # CONVERTING TO FINALE
            if group_id==QUERIES[data_source][data_type]['EXTERNAL'] and data_type=='securities':
                name_field = QUERIES[data_source][data_type]['name']
                short_name_field = QUERIES[data_source][data_type]['short_name']
                bloomberg_ticker = result[QUERIES[data_source][data_type]['BLOOMBERG']]
                isin_code = result[QUERIES[data_source][data_type]['ISIN']]
                currency = result[QUERIES[data_source][data_type]['currency']]
                currency = Attributes.objects.filter(type='currency', short_name=currency)
                if currency.exists():
                    currency = currency[0]
                else:
                    currency = None
                
                security = SecurityContainer.objects.filter(aliases__alias_type__short_name=data_source.upper(), aliases__alias_value=result[group_id])
                external_append = False
                additional = ''
                if not security.exists():
                    security = SecurityContainer.objects.filter(aliases__alias_type__id=bloomberg_alias.id, aliases__alias_value=bloomberg_ticker)
                    additional = result[name_field]
                    external_append = True
                if is_fund(data_source, result):
                    container = fund_container_type
                    stype = fund_type
                elif is_bond(data_source, result):
                    container = bond_container_type
                    stype = bond_type
                elif is_stock(data_source, result):
                    container = stock_container_type
                    stype = stock_type
                else:
                    container = sec_container_type
                    stype = security_type
                if not security.exists():
                    LOGGER.info("Creating security with " + data_source + " id [" + str(result[group_id]) + "]")
                    security = SecurityContainer()
                    security.name = result[name_field]
                    security.short_name = result[short_name_field] if result[short_name_field]!=None and result[short_name_field]!='' else result[name_field]
                    security.save()
                    security.associated_companies.add(bloomberg_provider)
                    security.save()
                else:
                    LOGGER.info("Security with " + data_source + " id [" + str(result[group_id]) + "] already exists.")
                    security = security[0]
                    if not security.associated_companies.filter(role__identifier='SCR_DP').exists():
                        security.associated_companies.add(bloomberg_provider)
                        security.save()
                security.update_alias(external_alias, result[group_id], additional, external_append)
                security.currency = currency
                security.frequency = daily
                security.type = container
                security.security_type = stype
                security.save()
                if bloomberg_ticker!=None and bloomberg_ticker!='':
                    all_tickers.append(bloomberg_ticker)
                elif isin_code!=None and isin_code!='':
                    all_tickers.append(isin_code)
                    
                if bloomberg_ticker!=None and bloomberg_ticker!='':
                    security.update_alias(bloomberg_alias, bloomberg_ticker)
                if isin_code!=None and isin_code!='':
                    security.update_alias(isin_alias, isin_code)
                universe.members.add(security)
    if data_type=='securities':
        universe.save()
        
        all_containers = {}
        for member in universe.members.all():
            member = get_effective_instance(member)
            if not all_containers.has_key(member.type.identifier):
                all_containers[member.type.identifier] = []
            try:
                all_containers[member.type.identifier].append(member.aliases.get(alias_type__name='BLOOMBERG').alias_value)
            except:
                try:
                    all_containers[member.type.identifier].append(member.aliases.get(alias_type__name='ISIN').alias_value)
                except:
                    LOGGER.info("There is no Bloomberg nor ISIN code available for this security ["  + member.name + "]")
            
        for key in all_containers.keys():
            fields = BloombergTrackContainerMapping.objects.filter(Q(container__identifier='CONT_SECURITY') | Q(container__identifier=key), Q(active=True)).values_list('short_name__code', flat=True)
            all_containers[key] = [to_bloomberg_code(ticker,True) for ticker in all_containers[key]]
            history_key = uuid.uuid4().get_hex()
            bb_thread = threading.Thread(None, bloomberg_history_query, history_key, (history_key, all_containers[key], fields, True))
            bb_thread.start()
            bb_thread.join()