def outputpermutations(self, ):
        #--------------------------------------------------------
        import config
        import mytools
        import datetime
        import os
        mycachefolder = config.mycachefolder
        df_perms = self.EfficientFrontierObject.PermutationsDataframe

        list_of_dicts = []
        for index, row in df_perms.iterrows():

            randomweightseries = row['value']['randomweightseries']
            dict_rows = {}
            for idx in randomweightseries.iteritems():
                dict_rows[str(idx[0])] = str(idx[1])
            dict_rows['portfolioreturn'] = row['value']['portfolioreturn']
            dict_rows['portfoliostandarddeviation'] = row['value'][
                'portfoliostandarddeviation']
            list_of_dicts.append(dict_rows)

        import pandas as pd
        df_final = pd.DataFrame(list_of_dicts)
        mytools.general().make_sure_path_exists(mycachefolder)
        date14 = str(datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
        cachedfilepathname = mycachefolder
        cachedfilepathname = os.path.join(cachedfilepathname,
                                          date14 + ' permutations.csv')
        df_final[
            'returnoverrisk'] = df_final.portfolioreturn / df_final.portfoliostandarddeviation
        df_final.to_csv(cachedfilepathname,
                        columns=(list(df_final.columns.values)))
        print 'find your permutations output here:', cachedfilepathname
        return cachedfilepathname
Exemplo n.º 2
0
def stockhistorynobackfilltodataframeusingcache(symbol, fromdate, todate):
    print('--------------------------')
    print('Initialized pullprices.stockhistorydailytodataframeusingcache')
    import pandas as pd
    #import numpy as np
    from pandas.io.data import DataReader
    #from datetime import datetime, timedelta

    import config
    mycachefolder = config.mycachefolder
    import mytools
    mytools.general().make_sure_path_exists(mycachefolder)

    cachedfilepathname = mycachefolder + '\\stockhistorynobackfill ' + symbol + ' ' + fromdate + ' ' + todate + '.csv'
    import os
    if os.path.isfile(cachedfilepathname):

        print('   Found cached file:  ' + cachedfilepathname)
        df_hist = pd.read_csv(cachedfilepathname, index_col=0)
    else:
        print('   Getting new file:' + cachedfilepathname)
        df_hist = DataReader(symbol, "yahoo", fromdate, todate)
        df_hist.to_csv(cachedfilepathname,
                       columns=('Open', 'High', 'Low', 'Close', 'Volume',
                                'Adj Close'))

    return df_hist
    def execute(self, symbols_and_signs_dataframe, startdate, enddate,
                permutations, annualized_or_cumulative):
        import config
        import mytools
        import datetime
        import os
        import outputefficientfrontier as oef
        a = list([
            symbols_and_signs_dataframe.index.tolist(),
            symbols_and_signs_dataframe['longshort'].tolist()
        ])
        symbols_and_signs_list = zip(*a)

        #print symbols_and_signs_list
        #stop
        #symbols = [i[0] for i in symbols_and_signs_list]
        #longshortlist = [i[1] for i in symbols_and_signs_list]
        #symbols_and_signs_list = xxx
        o = oef.output(symbols_and_signs_list, startdate, enddate,
                       permutations, annualized_or_cumulative)
        d = o.DictionaryOfOutputFiles
        for k, v in d.items():
            #print k, v
            (f_path, filename) = os.path.split(v)
            date14 = filename.split(' ')[0]
        print date14
        mycachefolder = config.mycachefolder
        mytools.general().make_sure_path_exists(mycachefolder)
        #date14 = str(datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
        cachedfilepathname = mycachefolder
        cachedfilepathname = os.path.join(cachedfilepathname,
                                          date14 + ' compiled.xlsx')

        import xlsxwriter
        import glob
        import csv
        import numbers
        workbook = xlsxwriter.Workbook(cachedfilepathname,
                                       {'strings_to_numbers': True})
        #for filename in glob.glob("*.csv"):
        for k, v in d.items():
            #print k, v
            (f_path, filename) = os.path.split(v)
            worksheetname = str(filename.split('.')[0])
            worksheetname = worksheetname.split(' ')[1]
            ws = workbook.add_worksheet(worksheetname)
            spamReader = csv.reader(open(v, 'rb'), delimiter=',')
            row_count = 0
            #print filename
            for row in spamReader:
                for col in range(len(row)):
                    n = row[col]
                    ws.write(row_count, col, n)
                row_count += 1

        workbook.close()
        self.PathnameToCompiledWorkbook = cachedfilepathname
        print 'you can find your compileclass file here: ', cachedfilepathname
Exemplo n.º 4
0
    def __init__(
            self,
            symbols='MSFT,BA,BX,CRM',
            startdate_string='2014-12-31',
            period='monthly'  #or daily
        ,
            source='Yahoo'
        #, enddate_string='xxx'
    ):
        self.Symbols = symbols
        self.StartDateString = startdate_string
        self.Period = period
        self.Source = source
        list_of_symbols = symbols.split(',')
        dict_of_filenames = {}
        mycachefolder = config.mycachefolder
        mytools.general().make_sure_path_exists(mycachefolder)

        for symbol in list_of_symbols:
            o = pullreturns.perform(symbol, startdate_string, period, source)
            #self.ReturnsDataframe = o.ReturnsDataframe
            df = o.ReturnsDataframe()
            print 'Length of returns dataframe', len(df)
            enddate = df.tail(1)['b_monthend'].values[0]
            print 'enddate', enddate

            cachedfilepathname = mycachefolder + '\\procoutput returnsyahoo ' + symbol + ' ' + startdate_string + ' ' + enddate + '.csv'
            watchfolderpathname = config.mywatcherfolder + '\\procoutput returnsyahoo ' + symbol + ' ' + startdate_string + ' ' + enddate + '.csv'
            df.to_csv(cachedfilepathname,
                      columns=('a_symbol', 'b_monthend', 'd_end',
                               'e_pctchange'))

            if os.path.isfile(cachedfilepathname):
                print('   Found cached file:  ' + cachedfilepathname)
                dict_of_filenames[len(dict_of_filenames)] = cachedfilepathname
                print '------ copying file to watchfolder ------'
                shutil.copy(cachedfilepathname, config.mywatcherfolder)
                if os.path.isfile(watchfolderpathname):
                    print('   Found watch file:  ' + watchfolderpathname)
                    #df_hist = pd.read_csv(watchfolderpathname,index_col=0)
                else:
                    print('   Did not find watch file:  ' +
                          watchfolderpathname)

                #df_hist = pd.read_csv(cachedfilepathname,index_col=0)
            else:
                print('   Getting new file:' + cachedfilepathname)
                #df_hist = DataReader(symbol,  "yahoo", fromdate,todate)

        #os.system("C:/Batches/AutomationProjects/Watcher/code/bat/$execute-run-sql-script.bat     ")
        print('RUNNING BATCH FILE:  ' + config.mywatchbatchfolder +
              '\\$execute-run-sql-script.bat     ')
        os.system(config.mywatchbatchfolder +
                  '\\$execute-run-sql-script.bat     ')
Exemplo n.º 5
0
 def find(self
         ,  symbol = 'SPY'
         ,  mycomparesym = '^VIX'
         ,  numberofweeksahead = 8
         ,  expirationday = 'friday' #'friday' #'wednesday for index'
         ,  daysbackmid = 0
         ,  myspreadindollars = 1
         ,  mycumprobthreshold = 80 #Percent in whole number 80 = 80%
         ,  mycumprob_to_sell_price_lowrange = 0
         ,  mycumprob_to_sell_price_highrange = 100 # was 95
         ,  numberofweekstolookback = 150
         ,  RollingNumberOfPeriods = 120
         ,  showresults = 0
     ):
     import config
     
     
     # ##########
     # Parameters    
 #    symbol = 'SPY'
 #    mycomparesym = '^VIX'
 #    numberofweeksahead = 8
 #    expirationday = 'friday' #'friday' #'wednesday for index'
 #    daysbackmid = 0
 #    myspreadindollars = 1
 #    mycumprobthreshold = 80 #Percent in whole number 80 = 80%
 #    mycumprob_to_sell_price_lowrange = 0
 #    mycumprob_to_sell_price_highrange = 100 # was 95
 #    numberofweekstolookback = 150
 #    RollingNumberOfPeriods = 120
 #    showresults = 0
     
     ThreshholdAbove = 0.0001 #Percent change above
     ThreshholdBelow = -0.0001  #Percent change below
     
     mycandidatesfolder = config.mycandidatesfolder #'C:\\Batches\\rts\\output\\condor\\candidates'
     mysourcedatafolder = config.mysourcedatafolder #'C:\\Batches\\rts\\output\\condor\\candidates'
     myselectedcandidatesfolder = config.myselectedcandidatesfolder #'C:\\Batches\\rts\\output\\condor\\selectedcandidates'
     mycachefolder = config.mycachefolder #'C:\\Batches\\rts\\output\\cache'
     print('mycandidatesfolder',mycandidatesfolder)
     #mycandidatesfolder = 'C:\\Batches\\MyPython\\active\\output'
     #mycandidatesfolder = 'C:\\Documents and Settings\\jmalinchak\\My Documents\\My Python\\Active\\New Folder\\output'
     
     import ntpath
     def path_leaf(path):
         head, tail = ntpath.split(path)
         return tail or ntpath.basename(head)
     
     def path_base(path):
         head, tail = ntpath.split(path)
         return head or ntpath.basename(head)    
     #def getdictitem(thedict,key):
     def dayofweek_int(dayofweek_word):
         rv = int(-1)
         if dayofweek_word.lower() == 'friday':
             rv = int(4)
         if dayofweek_word.lower() == 'saturday':
             rv = int(5)
         if dayofweek_word.lower() == 'sunday':
             rv = int(6)
         if dayofweek_word.lower() == 'monday':
             rv = int(0)
         if dayofweek_word.lower() == 'tuesday':
             rv = int(1)
         if dayofweek_word.lower() == 'wednesday':
             rv = int(2)
         if dayofweek_word.lower() == 'thursday':
             rv = int(3)
         return rv
         
     #import csv
     import os
     candidatesfolderwithsymbol = os.path.join(mycandidatesfolder,symbol)
     sourcedatafolderwithsymbol = os.path.join(mysourcedatafolder,symbol)
     import mytools
     mygeneral = mytools.general()
     mygeneral.make_sure_path_exists(candidatesfolderwithsymbol) #candidatesfolderwithsymbol
     mygeneral.make_sure_path_exists(sourcedatafolderwithsymbol) #sourcedatafolderwithsymbol
     ## ##########
     ## Date setup
     #import datetime
     #
     #today_datetime = datetime.datetime.today()
     #today_date = datetime.date.today()
     
     
     # ##########
     # Date setup
     import datetime
     today_datetime = datetime.datetime.today()
     today_date = datetime.date.today()
     iter_date = today_date
     
     for expirationcounter in range(numberofweeksahead):
         
         while iter_date.weekday() != dayofweek_int(expirationday):
             iter_date += datetime.timedelta(1)    
         expirationdate_string = str(iter_date)
         iter_date += datetime.timedelta(1)
         print('Doing...',expirationcounter,expirationdate_string)
     
         while True :
            # today_date = today_date
             expire_date = datetime.datetime.strptime(expirationdate_string,'%Y-%m-%d').date()
             if today_date != expire_date:
                 break
             today_date = today_date - datetime.timedelta(hours=24)
         
         delta = expire_date - today_date
         
         # ####################################################
         # Get Option Prices
         pricingsymbol = symbol
         if pricingsymbol in ['VIX','RUT']:
             pricingsymbol = '^'+symbol
             
         import pullprices as pp
         df_optionpricescurrent = pp.options_to_dataframe(pricingsymbol,expirationdate_string,0)
         #print('$$$$$$$$$$$$$ cvcvcvcvc $$$$$$$$$$$$$$$$$')
         #print(df_optionpricescurrent)
         if showresults == 1:
             # ==========
             print('-----',symbol,'Option Prices','-----')
             #print(df_optionpricescurrent)
             # ==========
         if len(df_optionpricescurrent) == 0:
             print('-:-:-:-:-:--:-:-:-:-:--:-:-:-:-:- no option prices found for',pricingsymbol,expirationdate_string)
         else:
             # ########
             # Initialize notes
             print('Initialized:','calculatecumulativeprobabilityofpricechangebasedonexpiration.py')
             print('-----------')
             print('Symbol:',symbol)
             print('  Compared to:',mycomparesym)
             print('  Today:',today_date)
             print('  Expire Date:',expire_date )
             print('  Number of Days to Expiration:',delta.days)
             
             # ##########
             # Date setup
             import time
             millis = int(round(time.time() * 1000))
             datestringforfilename = today_datetime.strftime('%Y-%m-%d %H.%M.%S ') + str(millis)
             datestringforcsv = today_datetime.strftime('%Y-%m-%d %H:%M:%S.%f')
             datestringforprinting = today_datetime.strftime('%Y-%m-%d %H:%M') 
             
             daysbackfar = delta.days
             
             import builddataframeofrefdateminusd2tod1stockpricechanges
              
             df_stockpricechanges_unfiltered = builddataframeofrefdateminusd2tod1stockpricechanges.perform(pricingsymbol,numberofweekstolookback,daysbackmid,daysbackfar,showresults).DataFrameResult
             df_stockpricechanges = df_stockpricechanges_unfiltered.dropna(subset=['priceDaysBackFar'])    
             #print(df_stockpricechanges)
             number_of_observations = len(df_stockpricechanges.index)
             print('  Number Of Observations Found',number_of_observations)
             print('runtime_delta',datetime.datetime.today() - today_datetime)
             # ---------------------------------------------------------------------------------
             comparesym = mycomparesym
             
             #import datetime
             #today_date = datetime.date.today()
             
             #today_datetime = datetime.datetime.today()
             #print('today_date',today_date)
             
             datedelta = datetime.timedelta(weeks=numberofweekstolookback+3)
             startdatecalculatedf_datetime = today_datetime - datedelta
             startdatecalculatedf_string = str(startdatecalculatedf_datetime.date())
             print('  Start Date:',startdatecalculatedf_string)
             
             
             # ###########################
             # Get VIX or comparable stock
             import pullprices
             df_comparestockpricehistory = pullprices.stockhistorybackfilledtodatframeofstockhistoryinstances(comparesym, startdatecalculatedf_string, str(today_date))
             compare_stock_price = pullprices.stock(mycomparesym)
             print('runtime_delta',datetime.datetime.today() - today_datetime)
             
             #print(f.loc[f.index == '2015-06-18'])
             #f1 = f.loc[f.index.isin(['2015-06-18'])][['Adj Close']]
             
             
             # ################################################################################################################
             # performs some general statistics
             import pandas as pd
             df_std = pd.rolling_std(df_stockpricechanges[['DrawDownPctChange', 'DrawUpPctChange']], RollingNumberOfPeriods)
             df_mean = pd.rolling_mean(df_stockpricechanges[['DrawDownPctChange', 'DrawUpPctChange']], RollingNumberOfPeriods)
             
             # #########################################################################
             # Adds a column to dataframe to Compare data to something (VIX for example)
             df_stockpricechanges['comppratfar'] = float('NaN')
             df_stockpricechanges['breachedaboveorbelow'] = int(0)
             
             #print(df_stockpricechanges)
             # #######################################################################################
             # Counts number of observations that hit above and below threshold during trading period
             idrawbeyondf_upabove = 0
             idrawbeyondf_downbelow = 0
             
             #icountfartomidbeyondf_above = 0
             #icountfartomidbeyondf_below = 0
             
             #zzzzz error was here
             #print df_comparestockpricehistory
             
             for index, row in df_stockpricechanges.iterrows():
                 try:
                     fartomidpricechangedelta = (row['priceDaysBackMid'] - row['priceDaysBackFar']) / row['priceDaysBackFar']
                 except:
                     fartomidpricechangedelta = float('NaN')
                 if row['DrawUpPctChange'] > ThreshholdAbove:
                     idrawbeyondf_upabove = idrawbeyondf_upabove + 1
                 if row['DrawDownPctChange'] > abs(ThreshholdBelow):
                     idrawbeyondf_downbelow = idrawbeyondf_downbelow + 1
                 # #################################################
                 # Populates the comppratfar field (VIX for example)
                 #row['comppratfar'] = df_comparestockpricehistory.ix[row.index, 'Adj Close']
                 #df_stockpricechanges['comppratfar'][str(index.date())] = df_comparestockpricehistory.ix[row['dateDaysBackFar'], 'Adj Close']
                 #print row['dateDaysBackFar'],str(index.date())
                 #zzzzz error was here
                 if row['dateDaysBackFar'] in df_comparestockpricehistory.index:
                     df_stockpricechanges['comppratfar'][str(index.date())] = df_comparestockpricehistory['Adj Close'][row['dateDaysBackFar']]
                     #df_stockpricechanges.set_value(str(index.date()), 'comppratfar',  df_comparestockpricehistory[df_comparestockpricehistory.'Adj Close' == row['dateDaysBackFar']])
                    # df_stockpricechanges['comppratfar'][str(index.date())] = 
                 
                 #str(index.date())
             
             
         #    if showresults == 1:
         #        # ==========
         #        print(df_stockpricechanges)    
         #       # ==========
                    
             if showresults == 1:
                 print('Last DrawUpPctChange Mean',df_mean.ix[len(df_mean.index)-1,'DrawUpPctChange'])
                 print('Last DrawUpPctChange Std',df_std.ix[len(df_std.index)-1,'DrawUpPctChange'])
                 
                 print('---------------------------------')
                 print('Percent Beyond Draw Up')
                 print('---------------------------------')
                 print('  ',symbol
                         ,'{percent:.2%}'.format(percent=idrawbeyondf_upabove/len(df_stockpricechanges.index))
                         ,'of the'
                         ,len(df_stockpricechanges.index)
                         ,'observations closed above the '
                         ,'{percent:.2%}'.format(percent=ThreshholdAbove)
                         ,'threshold between t-',daysbackfar,'and t-',daysbackmid,', a total of'
                         ,idrawbeyondf_upabove
                         ,'observations'
                     )
                 
                 print('---------------------------------')
                 print('Percent Beyond Draw Down')
                 print('---------------------------------')
                 print('  ',symbol
                         ,'{percent:.2%}'.format(percent=idrawbeyondf_downbelow/len(df_stockpricechanges.index))
                         ,'of the'
                         ,len(df_stockpricechanges.index)
                         ,'observations closed below the'
                         ,'{percent:.2%}'.format(percent=ThreshholdBelow)
                         ,'threshold between t-',daysbackfar,'and t-',daysbackmid,', a total of'
                         ,idrawbeyondf_downbelow
                         ,'observations'
                     )
             
             
             
             
             #////////////////////////////////////////////////////
             #               Draw Up and Draw Down analysis 
             import scipy.stats as ss
             import numpy as np
             import matplotlib.pyplot as plt
             #////////////////////////////////////////////////////
             #               Draw Up analysis
             print('')
             print(symbol,'Draw Up:',len(df_stockpricechanges.index), 'observations',idrawbeyondf_upabove,'breached',ThreshholdAbove)
             print('   ','DaysBackFar',daysbackfar,'   DaysBackMid:',daysbackmid)
             serDrawUp = pd.Series(df_stockpricechanges['DrawUpPctChange'])
             
             serDrawUp.hist(cumulative=True, normed=1, bins=idrawbeyondf_upabove)
             maxpercent_drawup = 0
             for n in np.linspace(0,1,1000,endpoint=False):
                 cumprob_to_sell_price = ss.percentileofscore(serDrawUp, n)    
                 if cumprob_to_sell_price >= mycumprobthreshold:
                     maxpercent_drawup = n
                     print('   ',round(cumprob_to_sell_price,1),'percent of observations closed up inside of','{percent:.2%}'.format(percent=n)),'percent'        
                     break
             plt.show()
             
             #////////////////////////////////////////////////////
             #               Draw Down analysis
             print('')
             print(symbol,'Draw Down:',len(df_stockpricechanges.index), 'observations',idrawbeyondf_upabove,'breached',ThreshholdBelow)
             print('   ','DaysBackFar',daysbackfar,'   DaysBackMid:',daysbackmid)
             serDrawDown = pd.Series(df_stockpricechanges['DrawDownPctChange'])
             serDrawDown.hist(cumulative=True, normed=1, bins=idrawbeyondf_downbelow)
             maxpercent_drawdown = 0
             for n in np.linspace(0,1,1000,endpoint=False):
                 cumprob_to_sell_price = ss.percentileofscore(serDrawDown, n)    
                 if cumprob_to_sell_price >= mycumprobthreshold:
                     maxpercent_drawdown = n
                     print('   ',round(cumprob_to_sell_price,1),'percent of observations closed down inside of','{percent:.2%}'.format(percent=(-1.0)*n)),'percent'
                     break
             plt.show()
             
                 
             stockprice = pp.stock(pricingsymbol)
             
             # ##########################
             # Calculate Analysis Results
             #import mytools
             osymbol = mytools.get_from_optionsymbol()
             rows_optionpricescurrent    = []        
             #rows_optionpricescurrent.append(['optionsymbol','stockprice','strike','pdeltapct_to_sell_price','cumprob_to_sell_price','bid','ask','last'])
             rows_optionpricescurrent.append(['optionsymbol','exdate','symbol','ty','st','strike_at_sell_price','strike_at_buy_price','pdeltapct_to_sell_price','cumprob_to_sell_price','cumprob_to_buy_price','bid','ask','iv','iscandidate'])
             
             #df_candidates = {}
             pdeltapct_atthreshold_calloption = float('NaN')
             pdeltapct_atthreshold_putoption = float('NaN')
             previous_pdeltapct_for_put = float('NaN')
             
             # ##################################
             # Loop through current option prices
             
             #print(df_optionpricescurrent)
             #print('++++++++++++++++++++++++++++++++++++++++++ df_optionpricescurrent')
             if df_optionpricescurrent is None:
                 print('There are no option prices for ' + symbol + ' expdate: '+expirationdate_string)
                
             else:
                 df_optionpricescurrent['optiontype'] = 'X'
                 
                 
                 for index, row in df_optionpricescurrent.iterrows():
                     optionsymbol = row['optionsymbol']
                     df_optionpricescurrent['optiontype'][index] = mytools.get_from_optionsymbol().optiontype(optionsymbol)
                     #print(optionsymbol)
                     strike_at_sell_price = float(row['strike'])
                     strike_at_sell_price_formatted = "%.2f" % strike_at_sell_price
                     
                     optiontype = mytools.get_from_optionsymbol().optiontype(row['optionsymbol'])
                 #    if optiontype == 'C':
                 #        strike_at_buy_price = float(strike_at_sell_price) + float(myspreadindollars)
                 #    else:
                 #        strike_at_buy_price = float(strike_at_sell_price) - float(myspreadindollars)
                 
                     if optiontype == 'C':
                         strike_at_buy_price = float(strike_at_sell_price) + float(1)
                     else:
                         strike_at_buy_price = float(strike_at_sell_price) - float(1)
                         
                     exdate = osymbol.expirationdate(row['optionsymbol'])
                     vsymbol = osymbol.symbol(row['optionsymbol'])
                     #print('stockprice',stockprice)
                     pdeltapct_to_sell_price = (float(strike_at_sell_price) - float(stockprice)) / float(stockprice)
                     #print('pdeltapct_to_sell_price',pdeltapct_to_sell_price)
                     pdeltapct_to_buy_price = (float(strike_at_buy_price) - float(stockprice)) / float(stockprice)
                     cumprob_to_sell_price = float('NaN')
                     cumprob_to_buy_price = float('NaN')
                     iscandidate = 0
                     # ========================
                 #    if abs(pdeltapct_to_sell_price) > 0.10:
                 #        print('++++++++++++++++++++++++more than 10',row)
                     # ========================
                     #print('+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')
                     if optiontype == 'C':
                         if pdeltapct_to_sell_price > 0:
                             cumprob_to_sell_price = ss.percentileofscore(serDrawUp, pdeltapct_to_sell_price)
                             cumprob_to_buy_price =  ss.percentileofscore(serDrawUp, pdeltapct_to_buy_price)
                             #print('cumprob_to_sell_price',cumprob_to_sell_price,mycumprobthreshold)
                             if str(pdeltapct_atthreshold_calloption).lower() == 'nan':
                                 if float(cumprob_to_sell_price) >= float(mycumprobthreshold):
                                     print('Yes Call threshold found %%%%%%%%%%%%%%%%%%%%%%%%%%',cumprob_to_sell_price)                    
                                     pdeltapct_atthreshold_calloption = pdeltapct_to_sell_price
                                     #capturedspreadf_at_call_thresholdf_cross
                                     print('Call',str(pdeltapct_atthreshold_calloption),'cumprob_to_sell_price > mycumprobthreshold',str(cumprob_to_sell_price),str(mycumprobthreshold),pdeltapct_atthreshold_calloption,pdeltapct_atthreshold_putoption)
                 #                    # #####################
                                 else:
                                     print('  No Call threshold found :::::::::::','float(cumprob_to_sell_price) >= float(mycumprobthreshold)',cumprob_to_sell_price,mycumprobthreshold)
                                 
                 #bbbbbb
                     elif optiontype == 'P':
                         if pdeltapct_to_sell_price < 0:
                             cumprob_to_sell_price = ss.percentileofscore(serDrawUp,(-1.0) * pdeltapct_to_sell_price)
                             cumprob_to_buy_price = ss.percentileofscore(serDrawUp,(-1.0) * pdeltapct_to_buy_price)
                             
                             if str(pdeltapct_atthreshold_putoption) == 'nan':
                                 
                                 if float(round(cumprob_to_sell_price,2)) <= float(round(mycumprobthreshold,2)):
                                     print('Yes Put threshold found $$$$$$$$$$$$$$$$$$$$$$$$$$',cumprob_to_sell_price)
                                     pdeltapct_atthreshold_putoption = previous_pdeltapct_for_put
                                     print('Put',str(pdeltapct_atthreshold_putoption),'cumprob_to_sell_price > mycumprobthreshold',str(cumprob_to_sell_price),str(mycumprobthreshold))
                 #                    # #####################
                 
                 
                                 previous_pdeltapct_for_put = pdeltapct_to_sell_price
                                     
                     if cumprob_to_sell_price != float('NaN'):
                         if cumprob_to_sell_price > mycumprob_to_sell_price_lowrange and cumprob_to_sell_price < mycumprob_to_sell_price_highrange:
                             iscandidate = 1
                             # ##########
                             # candidates are found and put into a dictionary, you might want some other data storage
                             #df_candidates[str(strike_at_sell_price)+optiontype] = row,rows[len(rows)-1]  
                         #vvvvvvv
                         rows_optionpricescurrent.append([optionsymbol,exdate,vsymbol,optiontype,stockprice,strike_at_sell_price,strike_at_buy_price,'{percent:.2%}'.format(percent=pdeltapct_to_sell_price),'{percent:.2%}'.format(percent=cumprob_to_sell_price/100),'{percent:.2%}'.format(percent=cumprob_to_buy_price/100),row['bid'],row['ask'],row['impliedvolatility'],iscandidate])
                         #df_candidates[strike_at_sell_price,optiontype] = [symbol,optiontype,stockprice,strike_at_sell_price,'{percent:.2%}'.format(percent=pdeltapct_to_sell_price),round(cumprob_to_sell_price,1),row['bid'],row['ask'],row['impliedvolatility'],iscandidate]
                         
                     #print(symbol,'price change from ',stockprice,'to strike_at_sell_price',strike_at_sell_price,'(','{percent:.2%}'.format(percent=pdeltapct_to_sell_price),') exp',exdate,cumprob_to_sell_price)
                 headers = rows_optionpricescurrent.pop(0)
                 df_cumprobsbystrikeranges = pd.DataFrame(rows_optionpricescurrent,columns=headers)
                 
                 # qqqqqq
                 #print(df_cumprobsbystrikeranges)
                 
                 print('----------------------------- ok got here -------------------------------------')
                 
                 # #####################
                 # Based on mycumprobthreshold, how many breached total, up and down
                 df_stockpricechanges['breachedaboveorbelow'] = int(0)
                 breachedmycumprobthresholdf_total = 0
                 breachedmycumprobthresholdf_up = 0
                 breachedmycumprobthresholdf_down = 0
                 
                 icountfartomidbeyondf_above = 0
                 icountfartomidbeyondf_below = 0
                 
                 for index, row in df_stockpricechanges.iterrows():
                     #fartomidpricechangedelta = (row['priceDaysBackMid'] - row['priceDaysBackFar']) / row['priceDaysBackFar']
                     priceToBreachFarToMidf_FinishUp = float(row['priceDaysBackFar']) + (float(row['priceDaysBackFar']) * pdeltapct_atthreshold_calloption)
                     priceToBreachFarToMidf_FinishDown = float(row['priceDaysBackFar']) + (float(row['priceDaysBackFar']) * pdeltapct_atthreshold_putoption)
                     if float(row['priceDaysBackMid']) > priceToBreachFarToMidf_FinishUp:
                         icountfartomidbeyondf_above = icountfartomidbeyondf_above + 1
                     if float(row['priceDaysBackMid']) < priceToBreachFarToMidf_FinishDown:
                         icountfartomidbeyondf_below = icountfartomidbeyondf_below + 1
                     #print('priceToBreachFarToMid',round(float(row['priceDaysBackMid']),2),'by',round(priceToBreachFarToMidf_FinishUp,2),round(priceToBreachFarToMidf_FinishDown,2))
                     
                     isbeyondmycumprobthresholdf_total = 0
                     if row['DrawUpPctChange'] > pdeltapct_atthreshold_calloption:
                         isbeyondmycumprobthresholdf_total = 1
                         breachedmycumprobthresholdf_up = breachedmycumprobthresholdf_up + 1
                     if row['DrawDownPctChange'] > abs(pdeltapct_atthreshold_putoption):
                         isbeyondmycumprobthresholdf_total = 1
                         breachedmycumprobthresholdf_down = breachedmycumprobthresholdf_down + 1
                     if isbeyondmycumprobthresholdf_total != 0:
                         breachedmycumprobthresholdf_total = breachedmycumprobthresholdf_total + 1
                         #df_stockpricechanges['breachedaboveorbelow'][str(index.date())] = 'breached mycumprobthreshold (' + str(mycumprobthreshold) + '%) ' + str(pdeltapct_atthreshold_calloption) +  ' ' + str(pdeltapct_atthreshold_putoption) + ' ' + str(isbeyondmycumprobthresholdf_total) #breachedaboveorbelow
                     df_stockpricechanges['breachedaboveorbelow'][str(index.date())] = isbeyondmycumprobthresholdf_total
                 
                 df_stockpricechanges['pdeltapct_atthreshold_calloption'] = pdeltapct_atthreshold_calloption
                 df_stockpricechanges['pdeltapct_atthreshold_putoption'] = pdeltapct_atthreshold_putoption
                 
                 '''
                 /////////////////////////////////////////////////////////////////////////////////////////
                                         SourceData CSV
                 /////////////////////////////////////////////////////////////////////////////////////////
                 '''
                 print('Len of df_stockpricechanges',len(df_stockpricechanges))
                 df_stockpricechanges.to_csv(sourcedatafolderwithsymbol + "\\ironcondor sourcedata (" + expirationdate_string + ') '+ symbol +  " " + datestringforfilename + ".csv",columns=('dateDaysBackMid','dateDaysBackFar','priceRefDate','priceDaysBackMid','priceDaysBackFar','DeltaFartoMid','DrawDownMax','DrawUpMax','DrawDownPctChange','DrawUpPctChange','comppratfar','pdeltapct_atthreshold_calloption','pdeltapct_atthreshold_putoption','breachedaboveorbelow'))
                 
                                             # ==========
                                             #print(df_cumprobsbystrikeranges)
                                             # ==========
                 print('================= ok got here dummy#2345245 ==================')
                 # ####################################################
                 # Here is where we find the value of the credit spread
                 #      still need to build the Put credit spread
                 candidaterows = []
                 
                 
                 '''@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
                                                 Candidate Header Row
                    @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@'''
                 candidateheader = ['openshortoptsym','openlongoptsym','ty','stkp','spdel','sstrk','bstrk','scump','bcump','sbid','bask','capt','myexdate','currdate','daysbackmid','pricecompare','obsv','siv','sel']
                 candidaterows.append(candidateheader)
                 
                 lastmycompareprice = df_comparestockpricehistory['Adj Close'][len(df_comparestockpricehistory)-1]
                 
                 #print df_cumprobsbystrikeranges
                 crossedthresholdf_call = 0
                 crossedthresholdf_put = 0
                 sum_of_iter_capt_at_cumprob_cross = float(0)
                 previousrow = None
                 for index,row in df_cumprobsbystrikeranges.iterrows():
                     sel = ''
                     
                     if float(row['iscandidate']) == 1:
                         openshortoptsym = row['optionsymbol']
                         myexdate = row['exdate']
                         myoptiontype = row['ty'] 
                         sellstockprice = row['st']
                         sellstrike = row['strike_at_sell_price']
                         sellcumprob = row['cumprob_to_sell_price']
                         buycumprob = row['cumprob_to_buy_price']
                         pdeltapct_to_sell_price = row['pdeltapct_to_sell_price']  
                         sellbidprice = row['bid']  
                         buyaskprice = float('NaN')
                         buystrike = float('NaN')
                         siv = row['iv']
                         row_inner_found = []
                         openlongoptsym = ''
                         iter_scump = float(row['cumprob_to_sell_price'].replace('%',''))
                         if row['ty'] == 'C' and crossedthresholdf_call == 0 and iter_scump >= mycumprobthreshold:
                             crossedthresholdf_call = 1
                             sel = 'x'
                             #print '&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&'
                         if row['ty'] == 'P' and crossedthresholdf_put == 0 and iter_scump <= mycumprobthreshold:
                             crossedthresholdf_put = 1
                             prevcandidaterow = candidaterows[len(candidaterows)-1]
                             a_indices = [i for i, x in enumerate(candidaterows[0]) if x == 'sel']
                             prevcandidaterow[a_indices[0]] = 'x'    
                 
                             #print 'tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt'
                             #print prevcandidaterow
                             
                         if row['ty'] == 'C':         
                             for index_inner,row_inner in df_cumprobsbystrikeranges.iterrows():
                                 if row_inner['ty'] == row['ty']:
                                     if float(row_inner['strike_at_sell_price']) == float(row['strike_at_sell_price'])+myspreadindollars:
                                         row_inner_found = row_inner
                                         buyaskprice = row_inner['ask']
                                         #buystrike = row_inner['strike_at_sell_price']
                                         buystrike = mytools.get_from_optionsymbol().strike(row_inner['optionsymbol'])
                                         openlongoptsym = row_inner['optionsymbol']
                                         #print('CALL >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
                                         
                         if row['ty'] == 'P':
                             for index_inner,row_inner in df_cumprobsbystrikeranges.iterrows():
                                 if row_inner['ty'] == row['ty']:
                                     if float(row_inner['strike_at_sell_price']) == float(row['strike_at_sell_price'])-myspreadindollars:
                                         row_inner_found = row_inner
                                         buyaskprice = row_inner['ask']
                                         #buystrike = row_inner['strike_at_sell_price']
                                         buystrike = mytools.get_from_optionsymbol().strike(row_inner['optionsymbol'])
                                         openlongoptsym = row_inner['optionsymbol']
                                         #print('PUT >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
                             previousrow = row
                         '''
                         @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
                                                 Candidate Value Rows
                         @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
                         '''
                 
                 #        row_sel = [x for i, x in enumerate(candidaterows) if x[a_columnidx[0]] == 'x']
                 #        print(row_sel)
                 #        row_sel[0][a_columnidx] = round(float(sellbidprice)-float(buyaskprice),3)
                 #        row_sel[1][a_columnidx] = round(float(sellbidprice)-float(buyaskprice),3)
                 #        sum_of_iter_capt_at_cumprob_cross = sum_of_iter_capt_at_cumprob_cross - float(prevcandidaterow[a_indices[0]])
                 
                         #candidaterows.append(candidateheader)
                         if len(openshortoptsym) > 0 and len(openlongoptsym) > 0:
                             candidaterows.append([openshortoptsym,openlongoptsym,myoptiontype,sellstockprice,pdeltapct_to_sell_price,sellstrike,buystrike,sellcumprob,buycumprob,sellbidprice,buyaskprice,round(float(sellbidprice)-float(buyaskprice),3),myexdate.strftime('%Y-%m-%d'),datestringforcsv,daysbackmid,compare_stock_price,number_of_observations,siv,sel])
                         #print(myexdate.strftime('%Y-%m-%d'),myoptiontype,sellstockprice,pdeltapct_to_sell_price,sellstrike,buystrike,sellcumprob,sellbidprice,buyaskprice,round(float(sellbidprice)-float(buyaskprice),3))
                         #print(row)
                         #print('row_inner_found ------------------------------------------------')
                         #print(row_inner_found)
                 
                 ## #############################
                 ## Example of List Comprehension
                 #a_columnidx = [i for i, x in enumerate(candidaterows[0]) if x == 'sel']
                 #print('x[a_columnidx[0]]',a_columnidx[0])
                 #for r in candidaterows:
                 #    if r[a_columnidx[0]] == 'x':                
                 #        print(r)
                 
                 print('================= ok got here dummy#42231 ==================')
                 headers = candidaterows.pop(0)
                 # This might not work - 9999999999999999
                 
                 print('len(candidaterows)',len(candidaterows))
                 if len(candidaterows) > 1:
                     if len(candidaterows[1]) > 0:
                         print('jkjkjkjkjkjkj candidaterows.count',len(candidaterows[1]))
                         df_candidates = pd.DataFrame(candidaterows,columns=headers)
                 
                         '''@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
                                                         Candidate Value CSV
                            @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@'''
                         df_candidates.to_csv(candidatesfolderwithsymbol + "\\ironcondor candidates (" + expirationdate_string + ') ' + symbol +  " " + datestringforfilename + ".csv",columns=('openshortoptsym','openlongoptsym','ty','stkp','spdel','sstrk','bstrk','scump','bcump','sbid','bask','capt','myexdate','currdate','daysbackmid','pricecompare','obsv','siv','sel'))
                         
                         # #################################################
                         # Use Dictionary Comprehension to get selected rows
                         selectedcandidaterows = []
                         list_selectedcandidaterows_1 = []
                         list_selectedcandidaterows_1.append(candidateheader)
                         
                         
                         #dict_selectedcandidaterows = { k:r for k,r in df_candidates.iterrows() if r['sel'] == 'x'}
                         #list_selectedcandidaterows_2 = [ r for k,r in df_candidates.items() if r['sel'] == 'x']
                         #print('777777777777777777777')
                         #print(list_selectedcandidaterows_2)
                         #selectedcandidaterows = list_selectedcandidaterows_2.extend(list_selectedcandidaterows_1)
                         
                         
                         
                         # ################################
                         # The above is equivalent to below
                         for k,candidaterow in df_candidates.iterrows():
                             if candidaterow['sel'] == 'x':
                                 list_selectedcandidaterows_1.append(candidaterow)
                                 print('!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
                                 print('!! Found a selected strike from candidate list')
                         headers = list_selectedcandidaterows_1.pop(0)
                         df_selectedcandidates = pd.DataFrame(list_selectedcandidaterows_1,columns=headers)
                         #print('2 ##########################################')
                         #print(df_selectedcandidates)
                         
                         
                         #print('1 ##########################################')
                         #print(list_selectedcandidaterows)
                         #print('Try dict',type(list_selectedcandidaterows) is dict)
                         #print(list(list_selectedcandidaterows.items()))
                         #print('2 ##########################################')
                         
                         filepath_to_selectedcandidates = os.path.join(myselectedcandidatesfolder,symbol,'selectedcandidates ' + expirationdate_string + ' ' + symbol + '.csv')
                         mygeneral.make_sure_path_exists(path_base(filepath_to_selectedcandidates)) #sourcedatafolderwithsymbol
                         #import os
                         
                         #Python 2-3 differences
                         #df_selectedcandidaterows = pd.DataFrame(df_selectedcandidates, columns=['ty','stkp','spdel','sstrk','bstrk','scump','bcump','sbid','bask','capt','myexdate','currdate','daysbackmid','pricecompare','obsv','sel'])
                         #print(df_selectedcandidaterows)
                         #df_selectedcandidaterows = pd.DataFrame(list_selectedcandidaterows.items(), columns=['ty','stkp','spdel','sstrk','bstrk','scump','bcump','sbid','bask','capt','myexdate','currdate','daysbackmid','pricecompare','obsv','sel'])
                         
                         #print('3 ##########################################')
                         '''@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
                                                         Selected Candidates CSV
                            @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@'''
                         print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
                         print('>> here are the current contents of the df_selectedcandidates dataframe')
                         print(df_selectedcandidates)
                         if os.path.isfile(filepath_to_selectedcandidates) != True:
                             df_selectedcandidates.to_csv(filepath_to_selectedcandidates,columns=('openshortoptsym','openlongoptsym','ty','stkp','spdel','sstrk','bstrk','scump','bcump','sbid','bask','capt','myexdate','currdate','daysbackmid','pricecompare','obsv','siv','sel'))
                             
                         elif len(df_selectedcandidates) > 0:
                             #df_fromcsv = pd.read_csv(filepath_to_selectedcandidates, index_col=0)
                             existingcsvfile = open(filepath_to_selectedcandidates, 'a') # Open file as append mode
                             df_selectedcandidates.to_csv(existingcsvfile, header = False)
                             existingcsvfile.close()
                         
                         
                         #print(df_candidates)
                         
                         crossedthresholdf_call = 0
                         crossedthresholdf_put = 0
                         capturedspreadf_at_call_thresholdf_cross = float('Nan')
                         capturedspreadf_at_put_thresholdf_cross = float('Nan')
                         sellstrike_at_call_thresholdf_cross = float('Nan')
                         sellstrike_at_put_thresholdf_cross = float('Nan')
                         
                         previousrow = None
                         for index,row in df_candidates.iterrows():
                             iter_scump = float(row['scump'].replace('%',''))
                             if row['ty'] == 'C' and crossedthresholdf_call == 0 and iter_scump >= mycumprobthreshold:
                                 crossedthresholdf_call = 1
                                 capturedspreadf_at_call_thresholdf_cross = float(row['capt'])
                                 sellstrike_at_call_thresholdf_cross = float(row['sstrk'])
                             if row['ty'] == 'P' and crossedthresholdf_put == 0 and iter_scump <= mycumprobthreshold:
                                 crossedthresholdf_put = 1
                                 if previousrow != None:
                                     capturedspreadf_at_put_thresholdf_cross = float(previousrow['capt'])
                                     sellstrike_at_put_thresholdf_cross = float(previousrow['sstrk'])
                                 # previousrow need only to be done for put side
                                 previousrow = row
                             #print('scump=',iter_scump)
                         
                         print('-- ------------------------------------------')
                         print('-- here are some result written out')
                         print('Symbol:',symbol)
                         print('  Current stock price:',round(stockprice,2))
                         print('  Today:',today_date)
                         print('  Expire Date:',expire_date )
                         print('  Number of Days to Expiration:',delta.days)
                         print('  Price changes start date:',startdatecalculatedf_string)
                         print('  Number Of Observations Found',number_of_observations)
                         print('  Price delta % for strike to meet',str(mycumprobthreshold)+'%','cumprob threshold (Call):','{percent:.2%}'.format(percent=pdeltapct_atthreshold_calloption))
                         print('  Price delta % for strike to meet',str(mycumprobthreshold)+'%','cumprob threshold (Put):','{percent:.2%}'.format(percent=pdeltapct_atthreshold_putoption))
                         
                         
                         print('DRAW UP & DOWN ---- What happened during')
                         print('    Breaches at',mycumprobthreshold,'% cumulative prob threshold (Far to Mid)')
                         
                         print('    Count of Draw UP breaches:  ',breachedmycumprobthresholdf_up)
                         print('    Count of Draw DOWN breaches:',breachedmycumprobthresholdf_down)
                         print('    Count of Draw TOTAL breaches:  ',breachedmycumprobthresholdf_total,'of',number_of_observations)
                         print('    Prcnt of Draw TOTAL breaches:  ','{percent:.2%}'.format(percent=(breachedmycumprobthresholdf_total)/float(number_of_observations)))
                         print('FINISH ---- What happened during')
                         print('    Count finishes.... Above price set by cumprob threshold=',icountfartomidbeyondf_above)
                         print('    Count finishes.... Below price set by cumprob threshold=',icountfartomidbeyondf_below)
                         print('    Count finishes.... Above or below price set by cumprob =',icountfartomidbeyondf_above+icountfartomidbeyondf_below,'of',number_of_observations)
                         print('    Prcnt finishes.... Above or below price set by cumprob =','{percent:.2%}'.format(percent=(icountfartomidbeyondf_above+icountfartomidbeyondf_below)/float(number_of_observations)))
                         
                         print('Condor Specs--------')
                         print('    sellstrike_at_call_thresholdf_cross=',round(sellstrike_at_call_thresholdf_cross,2))
                         print('    sellstrike_at_put_thresholdf_cross=',round(sellstrike_at_put_thresholdf_cross,2))
                         print('    Capture at Call threshold:',capturedspreadf_at_call_thresholdf_cross)
                         print('    Capture at Put threshold:',capturedspreadf_at_put_thresholdf_cross)
                         print('    Capture Total Amt=',round(capturedspreadf_at_call_thresholdf_cross+capturedspreadf_at_put_thresholdf_cross,2))
                         print('    sum_of_iter_capt_at_cumprob_cross',sum_of_iter_capt_at_cumprob_cross)
                         print('  Note: candidates occur where stock price does not close at levels set by cumprob sell price, meaning your condor was a success')
                         print('  ',mycomparesym+':',round(compare_stock_price,2))
                         print('-----------------')
                         #for k,v in df_candidates.items():
                         #    print(k)
                         #    print(v)
                         '''
Exemplo n.º 6
0
def stockhistorybackfilledtodatframeofstockhistoryinstancesusingcache(
        symbol, fromdate, todate):
    print(
        'initialized pullprices.stockhistorybackfilledtodatframeofstockhistoryinstances'
    )
    import pandas as pd
    #import numpy as np
    from pandas.io.data import DataReader
    from datetime import datetime, timedelta

    import config
    mycachefolder = config.mycachefolder
    import mytools
    mytools.general().make_sure_path_exists(mycachefolder)
    #dfnew.to_csv(mycachefolder + '\\stockhistorybackfilled '+ symbol '.csv',columns=(  'Open',   'High',    'Low',  'Close',    'Volume',  'Adj Close', 'Back Filled'))
    cachedfilepathname = mycachefolder + '\\stockhistorybackfilled ' + symbol + ' ' + fromdate + ' ' + todate + '.csv'
    import os
    if os.path.isfile(cachedfilepathname):
        print('--------------------------')
        #print('pullprices.stockhistorybackfilledtodatframeofstockhistoryinstancesusingcache')
        print('   Found cached file:  ' + cachedfilepathname)
        dfnew = pd.read_csv(cachedfilepathname, index_col=0)
    else:
        print('Getting new file:' + cachedfilepathname)
        date_format = "%Y-%m-%d"

        delta = timedelta(days=1)

        todate_date = datetime.strptime(todate, date_format)
        fromdate_date = datetime.strptime(fromdate, date_format)

        idates = 0
        d = datetime.strptime(fromdate, date_format)
        while d <= todate_date:
            idates = idates + 1
            d += delta

        # ##############
        # print(idates)

        #todays_date = datetime.datetime.now().date()
        index = pd.date_range(fromdate_date, periods=idates, freq='D')
        columns = [
            'Open',
            'High',
            'Low',
            'Close',
            'Volume',
            'Adj Close',
            'Back Filled',
        ]
        dfnew = pd.DataFrame(index=index, columns=columns)
        dfnew = dfnew.fillna('NaN')  # with 0s rather than NaNs

        # ############
        # print(dfnew)

        hist = DataReader(symbol, "yahoo", fromdate, todate)

        # #########
        # print(hist)

        last_open = 'NaN'
        last_high = 'NaN'
        last_low = 'NaN'
        last_close = 'NaN'
        last_adjclose = 'NaN'
        last_volume = 'NaN'
        backfilled = 'NaN'

        d = datetime.strptime(fromdate, date_format)
        while d <= todate_date:
            d_string = d.strftime(date_format)

            #print(d_string)
            if d_string in hist.index:
                last_open = hist.ix[d_string]['Open']
                last_high = hist.ix[d_string]['High']
                last_low = hist.ix[d_string]['Low']
                last_close = hist.ix[d_string]['Close']
                last_volume = hist.ix[d_string]['Volume']
                last_adjclose = hist.ix[d_string]['Adj Close']
                backfilled = 0
                #print(d_string,last_adjclose)
            else:
                backfilled = 1
                #print(d_string,'nothing',last_adjclose)

            dfnew.ix[d_string]['Open'] = last_open
            dfnew.ix[d_string]['High'] = last_high
            dfnew.ix[d_string]['Low'] = last_low
            dfnew.ix[d_string]['Close'] = last_close
            dfnew.ix[d_string]['Volume'] = last_volume
            dfnew.ix[d_string]['Adj Close'] = last_adjclose
            dfnew.ix[d_string]['Back Filled'] = backfilled

            d += delta

        dfnew.to_csv(cachedfilepathname,
                     columns=('Open', 'High', 'Low', 'Close', 'Volume',
                              'Adj Close', 'Back Filled'))

    #print(dfnew)
    return dfnew
    def execute(self, localunprocessedcsvpathname=''):

        import os, time
        import datetime
        import csv
        import re
        import mytools
        import config
        otools = mytools.general()
        localprocessedfolder = config.localunprocessedfolder + '\\barclays\\Ready for daily processing'
        mydict = {}
        #try:
        checkingon = 0
        mycolumn = -1
        #print 'checking:',localunprocessedcsvpathname
        if os.path.isfile(localunprocessedcsvpathname) == True:
            #print 'exists: ', localunprocessedcsvpathname

            (mode, ino, dev, nlink, uid, gid, size, atime, mtime,
             ctime) = os.stat(localunprocessedcsvpathname)
            #print "last modified: %s" % time.ctime(mtime)
            #datetime_object = datetime.strptime('Jun 1 2005  1:33PM', '%b %d %Y %I:%M%p')
            #Thu Mar 16 05:16:24 2017
            filedatetime_date = datetime.datetime.strptime(
                time.ctime(mtime), '%a %b %d %H:%M:%S %Y')
            filedatetime = filedatetime_date.strftime('%Y-%m-%d %H.%M.%S')
            #print filedatetime
            with open(localunprocessedcsvpathname, 'r') as f:

                reader = csv.reader(f, dialect='excel', delimiter='\t')
                filename = f.name
                #print 'got here 1', filename
                for row in reader:
                    #print len(row),row, 'xxxxxxxx'
                    if 'ToWorst' in str(row):
                        #print 'found ToWorst *******************'
                        if 'Value' in str(row):
                            #print 'found Value *******************'
                            mycharindex = [
                                (i.start(), i.end())
                                for i in re.finditer('    Value', str(row))
                            ]
                            mycharindex_string = str(mycharindex[0])
                            mycharindex_string = mycharindex_string.replace(
                                '(', '')
                            mycharindex_string = mycharindex_string.replace(
                                ')', '')
                            mycharindex_string = mycharindex_string.replace(
                                ' ', '')
                            mycharindex_start = int(
                                mycharindex_string.split(',')[0])
                            mycharindex_end = int(
                                mycharindex_string.split(',')[1])
                            #print mycharindex_start,mycharindex_end,'kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk'
                            checkingon = 1

                    elif checkingon == 1:
                        mymarketvalue = str(
                            row)[mycharindex_start:mycharindex_end]
                        onlydigits = mymarketvalue.replace(' ', '').replace(
                            '.', '').replace('-', '')

                        if checkingon == 1:
                            testsection1 = str(
                                row)[2:len('  Financial Institutio')][:2]
                            #print 'testsection1',testsection1,'11111111111111111111111111111111111111111111111'
                            if testsection1.strip() != '':
                                mysection1 = str(row)[
                                    2:len('  Financial Institutio')].strip()
                                mysection1 = mysection1.replace('[', '')
                                mysection1 = mysection1.replace(']', '')
                                mysection1 = mysection1.replace('\'', '')
                                #print mysection1, '@@@@@@@@@@@@@@ Section 1 @@@@@@@@@@@@@@@@'

                            testsection2 = str(
                                row)[2:len('  Financial Institutio')][:4]
                            #print 'testsection2',testsection2,'222222222222222222222222222222222222222222222222'
                            if testsection2.strip() != '':
                                mysection2 = str(row)[
                                    2:len('  Financial Institutio')].strip()
                                mysection2 = mysection2.replace('[', '')
                                mysection2 = mysection2.replace(']', '')
                                mysection2 = mysection2.replace('\'', '')
                                #print mysection2, '@@@@@@@@@@@@@@ Section 2 @@@@@@@@@@@@@@@@'

                        if 'INDEX RESULTS' in str(row):
                            break
                        if len(str(row)) > mycharindex_end:
                            #print str(row)[2:21],str(row)[22:31],str(row)[32:38],mymarketvalue,filename,'|||||',row[0], 'xxxxxxxx', str(row)[32:39]
                            issuename = str(
                                row)[2:len('  Financial Institutio')]
                            issuename = issuename.replace('[', '')
                            issuename = issuename.replace(']', '')
                            issuename = issuename.replace('\'', '')
                            #if not issuename[:2] == '  ':
                            if mymarketvalue.strip().replace('-', '').replace(
                                    '.', '').isdigit() == True:
                                if mysection1 == 'U.S. Aggregate' and mysection2 == 'U.S. Aggregate':
                                    #print mysection1,mysection2,mymarketvalue.strip(),filename
                                    mydict[len(mydict)] = {
                                        'source': 'barclays',
                                        'category': 'agg',
                                        'mysection1': mysection1,
                                        'mysection2': mysection2,
                                        'issue': issuename.strip(),
                                        'marketvalue': mymarketvalue.strip(),
                                        'filename': filename,
                                        'filedatetime': filedatetime
                                    }

        return mydict
@author: justin.malinchak
"""
# -*- coding: utf-8 -*-
"""
Created on Wed Jun 03 12:37:02 2015

@author: justin.malinchak
"""

# // Parameters
DataColumnTitle = 'MTD Total'  # can be [DayTot or MTDTot]
fileextension = '.agg'

import mytools
otools = mytools.general()

import config
config.localunprocessedfolder
config.localprocessedfolder
config.localuploadsreadyfolder
config.serveruploadsreadyfolder

localunprocessedfolder = config.localunprocessedfolder + '\\barclays'
localprocessedfolder = config.localunprocessedfolder + '\\barclays\\Ready for daily processing'
otools.make_sure_path_exists(localunprocessedfolder)
otools.make_sure_path_exists(localprocessedfolder)
#localunprocessedfolder = 'C:\Batches\AutomationProjects\Investment Strategy\ETL\zTest'

localoutputfolder = config.localuploadsreadyfolder
etluploadfolder = config.serveruploadsreadyfolder
    if dayofweek_word.lower() == 'monday':
        rv = int(0)
    if dayofweek_word.lower() == 'tuesday':
        rv = int(1)
    if dayofweek_word.lower() == 'wednesday':
        rv = int(2)
    if dayofweek_word.lower() == 'thursday':
        rv = int(3)
    return rv
    
#import csv
import os
candidatesfolderwithsymbol = os.path.join(mycandidatesfolder,symbol)
sourcedatafolderwithsymbol = os.path.join(mysourcedatafolder,symbol)
import mytools
mygeneral = mytools.general()
mygeneral.make_sure_path_exists(candidatesfolderwithsymbol) #candidatesfolderwithsymbol
mygeneral.make_sure_path_exists(sourcedatafolderwithsymbol) #sourcedatafolderwithsymbol
## ##########
## Date setup
#import datetime
#
#today_datetime = datetime.datetime.today()
#today_date = datetime.date.today()


# ##########
# Date setup
import datetime
today_datetime = datetime.datetime.today()
today_date = datetime.date.today()
class perform:
    import os
    import csv
    import re
    import mytools
    otools = mytools.general()
    import config

    def __init__(self,
                 # procname = 'xdeletethis_sylvan'
                 # , params = {}
                 ):
        print 'started perform.__init__'

    def readfile(self,
                 localunprocessedcsvpathname):  #filebasename, localfolder,
        try:
            checkingon = 0
            mycolumn = -1
            print 'got here'
            with open(localunprocessedcsvpathname, 'r') as f:
                reader = csv.reader(f, dialect='excel', delimiter='\t')
                for row in reader:
                    #print len(row),row, 'xxxxxxxx'
                    if 'ToWorst' in str(row):
                        if 'Value' in str(row):
                            print 'found ToWorst *******************'
                            mycharindex = [
                                (i.start(), i.end())
                                for i in re.finditer('    Value', str(row))
                            ]
                            mycharindex_string = str(mycharindex[0])
                            mycharindex_string = mycharindex_string.replace(
                                '(', '')
                            mycharindex_string = mycharindex_string.replace(
                                ')', '')
                            mycharindex_string = mycharindex_string.replace(
                                ' ', '')
                            mycharindex_start = int(
                                mycharindex_string.split(',')[0])
                            mycharindex_end = int(
                                mycharindex_string.split(',')[1])
                            print mycharindex_start, mycharindex_end, 'kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk'

                        checkingon = 1

                    elif checkingon == 1:
                        mymarketvalue = str(
                            row)[mycharindex_start:mycharindex_end]
                        onlydigits = mymarketvalue.replace(' ', '').replace(
                            '.', '').replace('-', '')

                        if checkingon == 1:
                            testsection1 = str(
                                row)[2:len('  Financial Institutio')][:2]
                            #print 'testsection1',testsection1,'11111111111111111111111111111111111111111111111'
                            if testsection1.strip() != '':
                                mysection1 = str(row)[
                                    2:len('  Financial Institutio')].strip()
                                mysection1 = mysection1.replace('[', '')
                                mysection1 = mysection1.replace(']', '')
                                mysection1 = mysection1.replace('\'', '')
                                #print mysection1, '@@@@@@@@@@@@@@ Section 1 @@@@@@@@@@@@@@@@'

                            testsection2 = str(
                                row)[2:len('  Financial Institutio')][:4]
                            #print 'testsection2',testsection2,'222222222222222222222222222222222222222222222222'
                            if testsection2.strip() != '':
                                mysection2 = str(row)[
                                    2:len('  Financial Institutio')].strip()
                                mysection2 = mysection2.replace('[', '')
                                mysection2 = mysection2.replace(']', '')
                                mysection2 = mysection2.replace('\'', '')
                                #print mysection2, '@@@@@@@@@@@@@@ Section 2 @@@@@@@@@@@@@@@@'

                        if 'INDEX RESULTS' in str(row):
                            break
                        if len(str(row)) > mycharindex_end:
                            #print str(row)[2:21],str(row)[22:31],str(row)[32:38],mymarketvalue,filename,'|||||',row[0], 'xxxxxxxx', str(row)[32:39]
                            issuename = str(
                                row)[2:len('  Financial Institutio')]
                            issuename = issuename.replace('[', '')
                            issuename = issuename.replace(']', '')
                            issuename = issuename.replace('\'', '')
                            #if not issuename[:2] == '  ':
                            if mymarketvalue.strip().replace('-', '').replace(
                                    '.', '').isdigit() == True:
                                if mysection1 == 'U.S. Aggregate' and mysection2 == 'U.S. Aggregate':
                                    #print mysection1,mysection2,mymarketvalue.strip(),filename
                                    mydict[len(mydict)] = {
                                        'source': 'barclays',
                                        'category': 'agg',
                                        'mysection1': mysection1,
                                        'mysection2': mysection2,
                                        'issue': issuename.strip(),
                                        'marketvalue': mymarketvalue.strip(),
                                        'filename': filename
                                    }

            ## #####################
            ## Moves processed files
            import os
            localprocessedcsvpathname = os.path.join(
                localprocessedfolder, filename
            )  #'E:\Batches\development\projects\Investment Strategy\ETL\Uploads\Ready'.encode('string_escape')
            #if os.path.exists(localprocessedcsvpathname):
            #    os.remove(localprocessedcsvpathname)
            #shutil.move(localunprocessedcsvpathname, localprocessedfolder)
            ## #####################

        except Exception as e:
            print '*** you better write an error log ***'
            print e.__doc__
import config
import mytools
import datetime
import os

basedir = '\\\\ipc-vfs01\\SEC$\\SEC Examination Information Request List 2017\\Independent Portfolio Consultants, Inc. Request'
mytools.general().make_sure_path_exists(basedir)
dirlist = [
    '01. Adviser organization chart ',
    '02. List of current employees',
    '03. List of any of the Advisers employees',
    '04. List of any former employees & complaints ',
    '05. Threatened, pending and settled litigation',
    '06. List of any sub-advisers.',
    '07. The Form ADV Part 2 ',
    '08. Client advisory contracts or agreements',
    '09. Current fee schedule',
    '10. A list of all committees ',
    '11. Names of any joint ventures',
    '12. Key service providers',
    '13. Compliance policies and procedures',
    '14. Annual review, recently completed',
    '15. Compliance risks, policies and changes',
    '16. Complaints & process used for monitoring',
    '17. Non-compliance Record',
    '18. Client Information',
    '18a. Data',
    '18b. Clients lost',
    '18c. Other advisory clients not named ',
    '19. Pitch books',
    '20. Advertisements ',
    def printoutput(self, ):

        import numpy
        import config
        import mytools
        import datetime
        import os

        d_returns = {}

        mycachefolder = config.mycachefolder
        mytools.general().make_sure_path_exists(mycachefolder)
        date14 = str(datetime.datetime.now().strftime("%Y%m%d%H%M%S"))

        o = self.EfficientFrontierObject
        print 'covariancematrix'
        cov = o.CovarianceMatrix
        cachedfilepathname = mycachefolder
        cachedfilepathname = os.path.join(cachedfilepathname,
                                          date14 + ' covariance.csv')
        cov.to_csv(cachedfilepathname, columns=(list(cov.columns.values)))
        d_returns['covariancematrix'] = cachedfilepathname

        print 'correlationmatrix'
        cor = o.CorrelationMatrix
        cachedfilepathname = mycachefolder
        cachedfilepathname = os.path.join(cachedfilepathname,
                                          date14 + ' correlation.csv')
        cor.to_csv(cachedfilepathname, columns=(list(cor.columns.values)))
        d_returns['correlationmatrix'] = cachedfilepathname

        print 'close prices'
        prc = o.AlignedClosePriceHistoryDataframe
        cachedfilepathname = mycachefolder
        cachedfilepathname = os.path.join(cachedfilepathname,
                                          date14 + ' closeprices.csv')
        prc.to_csv(cachedfilepathname, columns=(list(prc.columns.values)))
        d_returns['closeprices'] = cachedfilepathname

        print 'adjcloseprices'
        prc = o.AlignedAdjClosePriceHistoryDataframe
        cachedfilepathname = mycachefolder
        cachedfilepathname = os.path.join(cachedfilepathname,
                                          date14 + ' adjcloseprices.csv')
        prc.to_csv(cachedfilepathname, columns=(list(prc.columns.values)))
        d_returns['adjcloseprices'] = cachedfilepathname

        print 'aggregatedpricechangereturns'
        agret = o.ReturnsClass.AggregatedPriceChangeReturnsDataframe
        cachedfilepathname = mycachefolder
        cachedfilepathname = os.path.join(
            cachedfilepathname, date14 + ' aggregatedpricechangereturns.csv')
        agret.to_csv(cachedfilepathname, columns=(list(agret.columns.values)))
        d_returns['aggregatedpricechangereturns'] = cachedfilepathname

        print 'aggregatedtotalreturns'
        agret = o.ReturnsClass.AggregatedTotalReturnsDataframe
        cachedfilepathname = mycachefolder
        cachedfilepathname = os.path.join(
            cachedfilepathname, date14 + ' aggregatedtotalreturns.csv')
        agret.to_csv(cachedfilepathname, columns=(list(agret.columns.values)))
        d_returns['aggregatedtotalreturns'] = cachedfilepathname

        print 'totaldailyreturns'
        ret = o.ReturnsClass.TotalReturnsDataframe
        cachedfilepathname = mycachefolder
        cachedfilepathname = os.path.join(cachedfilepathname,
                                          date14 + ' totaldailyreturns.csv')
        ret.to_csv(cachedfilepathname, columns=(list(ret.columns.values)))
        d_returns['totaldailyreturns'] = cachedfilepathname

        print 'totalreturnsaligned'
        retalign = o.AlignedTotalReturnsDataframe
        cachedfilepathname = mycachefolder
        cachedfilepathname = os.path.join(cachedfilepathname,
                                          date14 + ' totalreturnsalign.csv')
        retalign.to_csv(cachedfilepathname,
                        columns=(list(retalign.columns.values)))
        d_returns['totalreturnsaligned'] = cachedfilepathname

        print 'pricechangereturns aligned'
        pcralign = o.AlignedPriceChangeReturnsDataframe
        cachedfilepathname = mycachefolder
        cachedfilepathname = os.path.join(
            cachedfilepathname, date14 + ' pricechangereturnsaligned.csv')
        pcralign.to_csv(cachedfilepathname,
                        columns=(list(pcralign.columns.values)))
        d_returns['pricechangereturnsaligned'] = cachedfilepathname

        print 'length of prc', len(prc)
        return d_returns
Exemplo n.º 13
0
def stockhistorybackfilledtodatframeofstockhistoryinstancesusingcache(symbol,fromdate,todate):
    print('initialized pullprices.stockhistorybackfilledtodatframeofstockhistoryinstances')
    import pandas as pd
    #import numpy as np
    from pandas.io.data import DataReader
    from datetime import datetime, timedelta
    
    import config
    mycachefolder = config.mycachefolder
    import mytools
    mytools.general().make_sure_path_exists(mycachefolder)
    #dfnew.to_csv(mycachefolder + '\\stockdatabackfilled '+ symbol '.csv',columns=(  'Open',   'High',    'Low',  'Close',    'Volume',  'Adj Close', 'Back Filled'))
    cachedfilepathname = mycachefolder + '\\stockdatabackfilled '+ symbol + ' ' + fromdate+ ' ' + todate + '.csv'
    import os
    if os.path.isfile(cachedfilepathname):
        print('--------------------------')
        print('pullprices.stockhistorybackfilledtodatframeofstockhistoryinstancesusingcache')
        print('   Found cached file:  '+cachedfilepathname)
        dfnew = pd.read_csv(cachedfilepathname,index_col=0)
    else:
        print('Getting new file:'+cachedfilepathname)
        date_format = "%Y-%m-%d"
    
        delta = timedelta(days=1)
    
        todate_date = datetime.strptime(todate, date_format)
        fromdate_date = datetime.strptime(fromdate, date_format)
        
        
    
        idates = 0
        d = datetime.strptime(fromdate, date_format)
        while d <= todate_date:
            idates = idates + 1
            d += delta
            
        # ##############
        # print(idates)
    
        #todays_date = datetime.datetime.now().date()
        index = pd.date_range(fromdate_date, periods=idates, freq='D')
        columns = [
         'Open',   'High',    'Low',  'Close',    'Volume',  'Adj Close', 'Back Filled',     
        ]
        dfnew = pd.DataFrame(index=index, columns=columns)
        dfnew = dfnew.fillna('NaN') # with 0s rather than NaNs
        
        # ############
        # print(dfnew)                 
        
        hist = DataReader(symbol,  "yahoo", fromdate,todate)
        
        # #########
        # print(hist)
        
        last_open = 'NaN'
        last_high = 'NaN'
        last_low = 'NaN'
        last_close = 'NaN'
        last_adjclose = 'NaN'
        last_volume = 'NaN'
        backfilled = 'NaN'
        
        d = datetime.strptime(fromdate, date_format)
        while d <= todate_date:
            d_string = d.strftime(date_format)
            
            #print(d_string)
            if d_string in hist.index:
                last_open = hist.ix[d_string]['Open']
                last_high = hist.ix[d_string]['High']
                last_low = hist.ix[d_string]['Low']
                last_close = hist.ix[d_string]['Close']
                last_volume = hist.ix[d_string]['Volume']
                last_adjclose = hist.ix[d_string]['Adj Close']
                backfilled = 0
                #print(d_string,last_adjclose)
            else:
                backfilled = 1
                #print(d_string,'nothing',last_adjclose)
                
            
            dfnew.ix[d_string]['Open'] = last_open 
            dfnew.ix[d_string]['High'] = last_high 
            dfnew.ix[d_string]['Low'] = last_low
            dfnew.ix[d_string]['Close'] = last_close        
            dfnew.ix[d_string]['Volume'] = last_volume
            dfnew.ix[d_string]['Adj Close'] = last_adjclose
            dfnew.ix[d_string]['Back Filled'] = backfilled
            
            d += delta
        
            
        dfnew.to_csv(cachedfilepathname,columns=('Open',   'High',    'Low',  'Close',    'Volume',  'Adj Close', 'Back Filled'))
        
    
    #print(dfnew)
    return dfnew

def copyFile(src, dest):
    try:
        shutil.copy(src, dest)
    # eg. src and dest are the same file
    except shutil.Error as e:
        print('Error: %s' % e)
    # eg. source or destination doesn't exist
    except IOError as e:
        print('Error: %s' % e.strerror)


# //  Main processing
import mytools
mytools.general().make_sure_path_exists(localprocessedfolder)
import datetime
filedatetime = datetime.datetime.today()
filedatetime_string = filedatetime.strftime('%Y%m%d%H%M%S%M')
outputfile = localoutputfolder + '\\barclays upload ' + fileextension[
    (-1) * (len(fileextension) -
            1):] + ' sinceinception ' + filedatetime_string + '.csv'
uploadfile = etluploadfolder + '\\barclays upload ' + fileextension[
    (-1) * (len(fileextension) -
            1):] + ' sinceinception ' + filedatetime_string + '.csv'

import os
import csv
import re
#localunprocessedfolder = 'C:\\Batches\\AutomationProjects\\Investment Strategy\\Downloads\\barclays\\'
Exemplo n.º 15
0
def stockhistorybackfilledtodatframeofstockhistoryinstancesusingcache(symbol, fromdate, todate):
    print("initialized pullprices.stockhistorybackfilledtodatframeofstockhistoryinstances")
    import pandas as pd

    # import numpy as np
    from pandas.io.data import DataReader
    from datetime import datetime, timedelta

    import config

    mycachefolder = config.mycachefolder
    import mytools

    mytools.general().make_sure_path_exists(mycachefolder)
    # dfnew.to_csv(mycachefolder + '\\stockdatabackfilled '+ symbol '.csv',columns=(  'Open',   'High',    'Low',  'Close',    'Volume',  'Adj Close', 'Back Filled'))
    cachedfilepathname = mycachefolder + "\\stockdatabackfilled " + symbol + " " + fromdate + " " + todate + ".csv"
    import os

    if os.path.isfile(cachedfilepathname):
        print("--------------------------")
        print("pullprices.stockhistorybackfilledtodatframeofstockhistoryinstancesusingcache")
        print("   Found cached file:  " + cachedfilepathname)
        dfnew = pd.read_csv(cachedfilepathname, index_col=0)
    else:
        print("Getting new file:" + cachedfilepathname)
        date_format = "%Y-%m-%d"

        delta = timedelta(days=1)

        todate_date = datetime.strptime(todate, date_format)
        fromdate_date = datetime.strptime(fromdate, date_format)

        idates = 0
        d = datetime.strptime(fromdate, date_format)
        while d <= todate_date:
            idates = idates + 1
            d += delta

        # ##############
        # print(idates)

        # todays_date = datetime.datetime.now().date()
        index = pd.date_range(fromdate_date, periods=idates, freq="D")
        columns = ["Open", "High", "Low", "Close", "Volume", "Adj Close", "Back Filled"]
        dfnew = pd.DataFrame(index=index, columns=columns)
        dfnew = dfnew.fillna("NaN")  # with 0s rather than NaNs

        # ############
        # print(dfnew)

        hist = DataReader(symbol, "yahoo", fromdate, todate)

        # #########
        # print(hist)

        last_open = "NaN"
        last_high = "NaN"
        last_low = "NaN"
        last_close = "NaN"
        last_adjclose = "NaN"
        last_volume = "NaN"
        backfilled = "NaN"

        d = datetime.strptime(fromdate, date_format)
        while d <= todate_date:
            d_string = d.strftime(date_format)

            # print(d_string)
            if d_string in hist.index:
                last_open = hist.ix[d_string]["Open"]
                last_high = hist.ix[d_string]["High"]
                last_low = hist.ix[d_string]["Low"]
                last_close = hist.ix[d_string]["Close"]
                last_volume = hist.ix[d_string]["Volume"]
                last_adjclose = hist.ix[d_string]["Adj Close"]
                backfilled = 0
                # print(d_string,last_adjclose)
            else:
                backfilled = 1
                # print(d_string,'nothing',last_adjclose)

            dfnew.ix[d_string]["Open"] = last_open
            dfnew.ix[d_string]["High"] = last_high
            dfnew.ix[d_string]["Low"] = last_low
            dfnew.ix[d_string]["Close"] = last_close
            dfnew.ix[d_string]["Volume"] = last_volume
            dfnew.ix[d_string]["Adj Close"] = last_adjclose
            dfnew.ix[d_string]["Back Filled"] = backfilled

            d += delta

        dfnew.to_csv(cachedfilepathname, columns=("Open", "High", "Low", "Close", "Volume", "Adj Close", "Back Filled"))

    # print(dfnew)
    return dfnew
Exemplo n.º 16
0
    def execute(self, localunprocessedcsvpathname=''):
        import os
        import csv
        import re
        import mytools
        import config
        otools = mytools.general()
        localprocessedfolder = config.localunprocessedfolder + '\\barclays\\Ready for daily processing'
        mydict = {}
        #try:
        checkingon = 0
        mycolumn = -1
        print 'checking:', localunprocessedcsvpathname
        if os.path.isfile(localunprocessedcsvpathname) == True:
            print 'not exists: ', localunprocessedcsvpathname

        with open(localunprocessedcsvpathname, 'r') as f:
            reader = csv.reader(f, dialect='excel', delimiter='\t')
            print 'got here 1'
            for row in reader:
                #print len(row),row, 'xxxxxxxx'
                if 'ToWorst' in str(row):
                    print 'found ToWorst *******************'
                    if 'Value' in str(row):
                        print 'found Value *******************'
                        mycharindex = [
                            (i.start(), i.end())
                            for i in re.finditer('    Value', str(row))
                        ]
                        mycharindex_string = str(mycharindex[0])
                        mycharindex_string = mycharindex_string.replace(
                            '(', '')
                        mycharindex_string = mycharindex_string.replace(
                            ')', '')
                        mycharindex_string = mycharindex_string.replace(
                            ' ', '')
                        mycharindex_start = int(
                            mycharindex_string.split(',')[0])
                        mycharindex_end = int(mycharindex_string.split(',')[1])
                        print mycharindex_start, mycharindex_end, 'kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk'
                        checkingon = 1

                elif checkingon == 1:
                    mymarketvalue = str(row)[mycharindex_start:mycharindex_end]
                    onlydigits = mymarketvalue.replace(' ', '').replace(
                        '.', '').replace('-', '')

                    if checkingon == 1:
                        testsection1 = str(
                            row)[2:len('  Financial Institutio')][:2]
                        #print 'testsection1',testsection1,'11111111111111111111111111111111111111111111111'
                        if testsection1.strip() != '':
                            mysection1 = str(
                                row)[2:len('  Financial Institutio')].strip()
                            mysection1 = mysection1.replace('[', '')
                            mysection1 = mysection1.replace(']', '')
                            mysection1 = mysection1.replace('\'', '')
                            #print mysection1, '@@@@@@@@@@@@@@ Section 1 @@@@@@@@@@@@@@@@'

                        testsection2 = str(
                            row)[2:len('  Financial Institutio')][:4]
                        #print 'testsection2',testsection2,'222222222222222222222222222222222222222222222222'
                        if testsection2.strip() != '':
                            mysection2 = str(
                                row)[2:len('  Financial Institutio')].strip()
                            mysection2 = mysection2.replace('[', '')
                            mysection2 = mysection2.replace(']', '')
                            mysection2 = mysection2.replace('\'', '')
                            #print mysection2, '@@@@@@@@@@@@@@ Section 2 @@@@@@@@@@@@@@@@'

                    if 'INDEX RESULTS' in str(row):
                        break
                    if len(str(row)) > mycharindex_end:
                        #print str(row)[2:21],str(row)[22:31],str(row)[32:38],mymarketvalue,filename,'|||||',row[0], 'xxxxxxxx', str(row)[32:39]
                        issuename = str(row)[2:len('  Financial Institutio')]
                        issuename = issuename.replace('[', '')
                        issuename = issuename.replace(']', '')
                        issuename = issuename.replace('\'', '')
                        #if not issuename[:2] == '  ':
                        if mymarketvalue.strip().replace('-', '').replace(
                                '.', '').isdigit() == True:
                            if mysection1 == 'U.S. Aggregate' and mysection2 == 'U.S. Aggregate':
                                #print mysection1,mysection2,mymarketvalue.strip(),filename
                                mydict[len(mydict)] = {
                                    'source': 'barclays',
                                    'category': 'agg',
                                    'mysection1': mysection1,
                                    'mysection2': mysection2,
                                    'issue': issuename.strip(),
                                    'marketvalue': mymarketvalue.strip(),
                                    'filename': filename
                                }

        return mydict