def _importBMFNew(urlbase, fileBase, outPath, suffix, dti, dtf): fer_bz = uu.load_obj('holidays_bz') dts = pd.bdate_range(dti, dtf, freq='C', holidays=fer_bz) for d in dts: file1 = '{}{:%y%m%d}.{}'.format(fileBase, d, suffix) fileout = '{}{:%y%m%d}.zip'.format( fileBase, d) #bmf weird: always a zip is coming urlfile = urlbase + file1 fp = outPath + '/' + fileout if not os.path.exists(fp): print('download {}'.format(file1)) # print(urlfile) try: urllib.request.urlretrieve(urlfile, fp) except: try: urllib.request.urlretrieve(urlfile, fp) except: print('UNABLE TO DOWNLOAD - MOVING ON') # if os.path.getsize(fp) < 30: # warn('file: {} is empty. Deleting'.format(fp)) # os.remove(fp) else: print('{} exists'.format(fileout))
def writePremioREF(file1='b3_premioREF'): from .globalMongo import mds from .mongo import metadataAsset, metadataOption dfs = uu.load_obj(file1) dfs = pd.concat(dfs) dfs.dt = pd.to_datetime(dfs.dt, format='%Y%m%d') dfs.venc = pd.to_datetime(dfs.venc, format='%Y%m%d') del dfs['id'] del dfs['compl'] del dfs['reg'] del dfs['tipoUnderlying'] del dfs['numCasasDec'] yearStr = pd.DatetimeIndex(dfs.venc).year.astype(str) dfs['codeSerie'] = dfs.code + dfs.serie + yearStr dfs.set_index('codeSerie', inplace=True) uCode = pd.unique(dfs.index) assetLst = np.array(mds.b3VS.list_symbols()) assetLst = [s.replace('_b3_opc_1BDay', '') for s in assetLst] #uCode = np.setdiff1d(uCode,np.array(assetLst)) uCode = [u for u in uCode if not u in assetLst] for i, code1 in enumerate(uCode): df_ = ds.mDataFrame(dfs.loc[code1].copy()) if isinstance(dfs.loc[code1], pd.Series): continue df_.set_index('dt', inplace=True) #del df_['codeSerie'] df_.md = metadataOption( code1 + '_b3_opc', 'option', df_.index[0].to_pydatetime(), None, 'BRL', False, df_.venc[-1].to_pydatetime(), strike=df_.strike[-1], underlying=df_.code[-1] + uu.futCodeFromDate(df_.venc[-1].to_pydatetime()), cp=df_.tipoOpc[-1].lower()) df_.md.subtype = 'option' df_.md.optModel = df_.modeloOpc[-1].lower() df1 = df_[['premio']] df1.columns = ['close'] print('writing {} - {}/{}'.format(code1, i, len(uCode))) try: mds.delete(code1[:-4], ds.freqHelper.bday, mds.b3VS) except: pass df1 = uu.drop_duplicates_index(df1) mds.write(df1, mds.b3VS, check_metadata=False, prune_previous_version=True)
def saveCleanXML(fileCadastro='b3_cadastroXML', filePx='b3_pesqXML'): cad = uu.load_obj(fileCadastro) px = uu.load_obj(filePx) codes = list(set.union(*[set(p.keys()) for p in px])) codes1 = [ c for c in codes if c in cad and 'code' in cad[c] and ( any([cad[c]['code'].startswith(m) for m in mercList + otherMerc]) or ('nmInf' in cad[c] and cad[c]['nmInf'] == 'OptnOnEqtsInf')) ] cad_clean = Dict({c: cad[c] for c in codes1}) uu.save_obj(cad_clean, 'b3_cadastroXML_clean') #px_clean = [{c: p[c] for c in p if c in codes1} for p in px] px_clean = [ Dict({c: p[c] for c in set.intersection(set(p.keys()), set(codes1))}) for p in px ] uu.save_obj(px_clean, 'b3_pesqXML_clean')
def writePremioEq(file1='b3_premioEq'): from .globalMongo import mds from .mongo import metadataAsset, metadataOption dfs = uu.load_obj(file1) dfs = pd.concat(dfs) dfs.dt = pd.to_datetime(dfs.dt, format='%Y%m%d') dfs.venc = pd.to_datetime(dfs.venc, format='%Y%m%d') yearStr = pd.DatetimeIndex(dfs.venc).year.astype(str) dfs.index = dfs.index + yearStr uCode = pd.unique(dfs.index) assetLst = np.array(mds.b3VS.list_symbols()) assetLst = [s.replace('_b3_opc_1BDay', '') for s in assetLst] #uCode = np.setdiff1d(uCode,np.array(assetLst)) #uCode = [u for u in uCode if not u in assetLst] uCode = list(set(uCode) - set(assetLst)) for i, code1 in enumerate(uCode): df_ = ds.mDataFrame(dfs.loc[code1].copy()) if isinstance(dfs.loc[code1], pd.Series): continue df_.set_index('dt', inplace=True) df_.md = metadataOption( code1 + '_b3_opc', 'option', df_.index[0].to_pydatetime(), None, 'BRL', False, df_.venc[-1].to_pydatetime(), strike=df_.strike[-1], underlying=code1[:4] + uu.futCodeFromDate(df_.venc[-1].to_pydatetime()), cp=df_.code[-1].lower()) df_.md.subtype = 'equity_option' df_.md.optModel = df_.type[-1].lower() df1 = df_[['premio', 'vol', 'strike']] df1.columns = ['close', 'vol', 'strike'] print('writing {} - {}/{}'.format(code1, i, len(uCode))) df1 = uu.drop_duplicates_index(df1) mds.write(df1, mds.b3VS, check_metadata=False, prune_previous_version=True)
def writeFromXML(fileCadastro='b3_cadastroXML', filePx='b3_pesqXML'): from .globalMongo import mds from .mongo import metadataAsset, metadataOption cad = uu.load_obj(fileCadastro) px = uu.load_obj(filePx) #must also get dates codes = list(set.union(*[set(p.keys()) for p in px])) if saveClean: saveCleanXML(fileCadastro, filePx) #px_ = {{k:l[k] for k in l} for l in px} #keys1 = [s for s in cad.keys()] lst_b3 = mds.find(library=[mds.b3VS, mds.b3VS_XML], metadata=False) lst_b3 = [l[1].replace('_1BDay', '') for l in lst_b3] flds = ('dt', 'FrstPric', 'MinPric', 'MaxPric', 'LastPric', 'TradAvrgPric', 'FinInstrmQty', 'OpnIntrst') cols = ['open', 'low', 'high', 'close', 'vwap', 'volume', 'open_interest'] map1 = {flds[i + 1]: c for i, c in enumerate(cols)} for j, c in enumerate(codes): #must use dates here if c in cad: cad1 = cad[c] if c in cad and any([ cad1['code'].startswith(m) for m in mercList + otherMerc ]) or cad1['nmInf'] == 'OptnOnEqtsInf': px1 = [{k: v for (k, v) in p[c].items() if k in flds} for p in px] px1 = [p for p in px1 if len(p) > 0] df1 = ds.mDataFrame(pd.DataFrame.from_dict(px1)) df1.set_index('dt', inplace=True) df1.sort_index(inplace=True) df1 = df1[(~np.all(np.isnan(df1.values), 1))] if df1.shape[0] == 0: continue isOpt = 'ExrcPric' in cad1 if isOpt: code1 = cad1['code'] + '_b3_opc' optSubtype = 'equity_option' if ( 'nmInf' in cad1 and cad1['nmInf'] == 'OptnOnEqtsInf') else 'option' strike = cad1['ExrcPric'] if not isinstance( cad1['ExrcPric'], list) else cad1['ExrcPric'][-1][-1] cp = 'p' if 'PUT' in cad1['OptnTp'].upper() else 'c' underlyingId = str( cad1['UndrlygInstrmId']['OthrId']['Id'])[:-2] underlying = cad[underlyingId]['code'] df1.md = metadataOption(code1, 'option', df1.index[0], fut_like=False, maturity=cad1['XprtnDt'], strike=strike, underlying=underlying, cp=cp) df1.md.underlyingId = underlyingId df1.md.subtype = optSubtype df1.md.optModel = 'e' if cad1['OptnStyle'] == 'EURO' else 'a' if optSubtype == 'equity_option': strikes = cad1['ExrcPric'] if not isinstance(strikes, list): strikes = [[None, strikes]] strikes = np.array(strikes) for i, s in enumerate(strikes[:, 0]): strikes[i, 0] = dt(1900, 1, 1) if s is None else s strikeSrs = pd.Series(strikes[:, 1].astype(np.float64), strikes[:, 0]) strikeSrs1 = strikeSrs.reindex(df1.index, method='ffill') df1['strike'] = strikeSrs1 df1.md.strikes = strikes.tolist() # strikeSrs.index[0]=None else: code1 = cad1['code'] + '_b3' df1.md = metadataAsset(code1, 'future', df1.index[0], fut_like=True, maturity=cad1['XprtnDt']) df1.md.id = c df1 = df1.rename(columns=map1) if code1 in lst_b3: df0 = mds.read(code1, library=[mds.b3VS, mds.b3VS_XML]) df0_ = df0.copy() df1_ = df1.copy() try: del df0_['strike'] except: pass try: del df0_['open_interest'] except: pass try: del df1_['strike'] except: pass try: del df1_['open_interest'] except: pass df0 = df0.loc[(~np.all(pd.isnull(df0_.values), 1))] df1 = df1.loc[~df1.index.isin(df0.index) & (~np.all(pd.isnull(df1_.values), 1))] if df1.shape[0] == 0: continue md = df1.md df1 = ds.mDataFrame(pd.concat([df0, df1], 0, sort=True)) df1.md = md df1 = df1.loc[(~np.all(pd.isnull(df1.values), 1))] if df1.shape[0] == 0: continue #check if exists and append !!! print('{} = {}/{}'.format(code1, j, len(codes))) df1 = uu.drop_duplicates_index(df1) if isOpt: mds.write(df1, mds.b3VS_XML, check_metadata=False, prune_previous_version=True) else: mds.write(df1, mds.b3VS, check_metadata=False, prune_previous_version=True)
def writeDFSPregao(file1='b3_pregaoFinal'): from .globalMongo import mds from .mongo import metadataAsset, metadataOption dfs = uu.load_obj(file1) df = pd.concat(dfs) flds = [ 'dt', 'code', 'serieVenc', 'venc', 'strike', 'valPonto', 'volRS', 'qtdAberto', 'qtdNeg', 'qtdTrans', 'ultOCP', 'ultOVD', 'abtNeg', 'minNeg', 'maxNeg', 'medNeg', 'ultNeg', 'dtUltNeg', 'horaUltNeg', 'aju', 'du', 'dtLimiteNeg', 'dtLiqFin', 'vencContratoObj', 'codeGTS', 'tipoSerie' ] newFlds = [ 'dt', 'codeMerc', 'srsCode', 'maturity', 'strike', 'multiplier', 'volume_fin', 'open_interest', 'neg', 'volume', 'bid', 'ask', 'open', 'low', 'high', 'vwap', 'last', 'dtLast', 'timeLast', 'close', 'du', 'dtTradeLimit', 'dtFinSettle', 'underlyingSrsCode', 'code', 'cp' ] df = df[flds] df.columns = newFlds mdfFields = [ 'dt', 'code', 'bid', 'ask', 'open', 'low', 'high', 'vwap', 'last', 'dtLast', 'timeLast', 'close', 'du', 'volume_fin', 'open_interest', 'neg', 'volume' ] descFields = np.setdiff1d(newFlds, mdfFields).tolist() + ['dt', 'code'] df = df.loc[df.codeMerc.isin(mercList)] df0 = df[mdfFields] dfMeta = df[descFields] uCode = np.unique(df0.code) for i, code1 in enumerate(uCode): df_ = ds.mDataFrame(df0.loc[df0.code == code1]) df_.set_index('dt', inplace=True) del df_['code'] dfm_ = dfMeta.loc[dfMeta.code == code1].copy() dfm_.set_index('dt', inplace=True) del dfm_['code'] # for c in dfm_.columns: # # try: # if dfm_[c].dtype==object: # dfm_[c] = pd.to_numeric(dfm_[c]) # except: # pass # if dfm_[c].dtype==np.float64: # assert((np.all(dfm_[c]==dfm_[c][0]) | np.isnan(dfm_[c][0]) )) # else: # assert(np.all(dfm_[c] == dfm_[c][0])) if dfm_.cp[-1] in ['C', 'V']: df_.md = metadataOption(code1 + '_b3', 'option', df_.index[0].to_pydatetime(), None, 'BRL', False, dfm_.maturity[-1].to_pydatetime(), strike=dfm_.strike[-1], underlying=dfm_.codeMerc[-1] + dfm_.underlyingSrsCode[-1], cp=dfm_.cp[-1].lower()) df_.md.subtype = 'option' else: df_.md = metadataAsset(code1 + '_b3', 'future', df_.index[0].to_pydatetime(), None, 'BRL', False, dfm_.maturity[-1].to_pydatetime()) df_.md.subtype = 'future' df_.md.source = 'B3' print('writing {}. {}/{}'.format(code1, i, uCode.size)) if not df_.index.is_unique: nm = df_.index.name if not df_.index.name is None else 'index' df_ = df_.reset_index().drop_duplicates(subset=nm, keep='last').set_index(nm) mds.write(df_, mds.b3VS, check_metadata=False, prune_previous_version=True)