def create_data_json_pcr_79(): wb = xlrd.open_workbook( os.path.join(BASE_DIR, 'PCR_CheckList_Sprint_6.xlsx')) sheet = wb.sheet_by_name('Schedulers Create') payload = [] Expect = list() for rownum in range(1, 69): user = OrderedDict() row_value = sheet.row_values(rownum) des = row_value[0] expect = row_value[10] code = row_value[11] user['name'] = row_value[1] user['taskId'] = row_value[2] user['sellerId'] = row_value[3] user['active'] = row_value[4] user['settings'] = row_value[5] user['scheduleType'] = row_value[6] # user['startDate'] = row_value[7] date = int(row_value[7]) if isinstance( row_value[7], float) else datetime.now().strftime('%d-%m-%Y') if isinstance(date, str): user['startDate'] = date else: user['startDate'] = datetime.fromordinal( datetime(1900, 1, 1).toordinal() + date - 2).strftime('%d-%m-%Y') user['scheduleTime'] = row_value[8] user['variables'] = row_value[9] payload.append(user) Expect.append((des, user, int(expect), code)) return Expect
def convert_to_datetime(input): """ Converts the given object to a datetime object, if possible. If an actual datetime object is passed, it is returned unmodified. If the input is a string, it is parsed as a datetime. Date strings are accepted in three different forms: date only (Y-m-d), date with time (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro). :rtype: datetime """ from datetime import date, datetime, timedelta if isinstance(input, datetime): return input elif isinstance(input, date): return datetime.fromordinal(input.toordinal()) elif isinstance(input, basestring): m = _DATE_REGEX.match(input) if not m: raise ValueError('Invalid date string') values = [(k, int(v or 0)) for k, v in m.groupdict().items()] values = dict(values) return datetime(**values) raise TypeError('Unsupported input type: %s' % type(input))
def create_data_json_pcr_39_2(): wb = xlrd.open_workbook( os.path.join(BASE_DIR, 'PCR_CheckList_Sprint_5.xlsx')) sheet = wb.sheet_by_name('PCR-39 API Create schedule') payload = [] for rownum in range(21, 58): user = OrderedDict() row_value = sheet.row_values(rownum) user['crawlType'] = row_value[5] user['scheduleName'] = row_value[6] user['scheduleSettings'] = row_value[7] user['scheduleType'] = row_value[8] date = int(row_value[9]) if isinstance( row_value[9], float) else datetime.now().strftime('%Y-%m-%d') if isinstance(date, str): user['startDate'] = date else: user['startDate'] = datetime.fromordinal( datetime(1900, 1, 1).toordinal() + date - 2).strftime('%Y-%m-%d') user['scheduleTime'] = row_value[10] user['activate'] = random.choice([True, False]) payload.append(user) return payload
def basicCalculationTimeOnly(self,excel_date): if(excel_date==0): return "null" dt = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + int(excel_date) - 2) hour, minute, second = self.floatHourToTime(excel_date % 1) dt = dt.replace(hour=hour, minute=minute, second=second) Created_On = "' %s'" % (dt.strftime("%m/%d/%Y")) return Created_On
def __init__(self, split, data_dir=None): super(ExampleDataset, self).__init__() if split == 'train': time_start = 0 time_end = datetime(2013, 8, 31, tzinfo=self.TZ).toordinal() elif split == 'test': time_start = datetime(2013, 9, 1, tzinfo=self.TZ).toordinal() time_end = datetime(2014, 1, 1, tzinfo=self.TZ).toordinal() else: raise ValueError('invalid split', split) self.FIRST_DATE = datetime(2012, 12, 28, tzinfo=self.TZ) self.TEST_TIMESLOTS = [datetime(2013, 9, 1, tzinfo=self.TZ), datetime(2013, 9, 25, tzinfo=self.TZ), datetime(2013, 10, 20, tzinfo=self.TZ), datetime(2013, 11, 15, tzinfo=self.TZ), datetime(2013, 12, 10, tzinfo=self.TZ), datetime(2014, 1, 1, tzinfo=self.TZ)] self.N_nodes = 100 self.A_initial = np.random.randint(0, 2, size=(self.N_nodes, self.N_nodes)) self.A_last = np.random.randint(0, 2, size=(self.N_nodes, self.N_nodes)) print('\nA_initial', np.sum(self.A_initial)) print('A_last', np.sum(self.A_last), '\n') self.n_events = 10000 all_events = [] for i in range(self.n_events): user_id1 = np.random.randint(0, self.N_nodes) user_id2 = np.random.choice(np.delete(np.arange(self.N_nodes), user_id1)) ts = max((time_start, self.FIRST_DATE.toordinal())) event_time = datetime.fromordinal(ts + np.random.randint(0, time_end - ts) ) assert event_time.timestamp() >= self.FIRST_DATE.timestamp(), (event_time, self.FIRST_DATE) all_events.append((user_id1, user_id2, np.random.choice(['communication event', 'association event']), event_time)) self.event_types = ['communication event'] self.all_events = sorted(all_events, key=lambda t: t[3].timestamp()) print('\n%s' % split.upper()) print('%d events between %d users loaded' % (len(self.all_events), self.N_nodes)) print('%d communication events' % (len([t for t in self.all_events if t[2] == 1]))) print('%d assocition events' % (len([t for t in self.all_events if t[2] == 0]))) self.event_types_num = {'association event': 0} k = 1 # k >= 1 for communication events for t in self.event_types: self.event_types_num[t] = k k += 1 self.n_events = len(self.all_events)
def ordinal_to_date_1(date_data, base_time='2018-01-01'): from datetime import datetime d = datetime.strptime(base_time, '%Y-%m-%d').date() start_0 = d.toordinal() - 1 all_dates = date_data.copy() modified_all_dates = [] for ind in all_dates.index: dt = datetime.fromordinal(start_0 + all_dates.loc[ind]) new_date = '%s/%s/%s' % (dt.month, dt.day, dt.year) modified_all_dates.append(new_date) return modified_all_dates
def convertSerialDate(sh_cell_val): try: dt = datetime.fromordinal( datetime(1900, 1, 1).toordinal() + int(sh_cell_val) - 2) dt_str = str(dt) year = dt_str.split('-')[0][-2:] month = dt_str.split('-')[1] day = dt_str.split('-')[2].split(' ')[0] CSV_Date = month + '/' + day + '/' + year return sh_cell_val except: return False
def get_year(mat_date): """ Calc year from matlab's date format Parameters ---------- mat_date : matlab's datenum Date to be converted Return ---------- year : int Year from matlab datenum """ temp = int(mat_date) year = datetime.fromordinal(max(temp - 366, 1)).year return year
def datenum_to_datetime(datenum): """ Convert Matlab datenum into Python datetime. :param datenum: Date in datenum format :return: Datetime object corresponding to datenum. """ days = datenum % 1 hours = days % 1 * 24 minutes = hours % 1 * 60 seconds = minutes % 1 * 60 return datetime.fromordinal(int(datenum)) \ + timedelta(days=int(days)) \ + timedelta(hours=int(hours)) \ + timedelta(minutes=int(minutes)) \ + timedelta(seconds=round(seconds)) \ - timedelta(days=366)
def draw_graph(dates, counts): ########################################################### # Drawing takes place here. figure(1) ax = subplot(111) plot_date(dates, counts, color='r', linestyle='-', marker='o', markersize=3) ax.xaxis.set_major_formatter(DateFormatter('%Y')) ax.xaxis.set_major_locator(YearLocator()) ax.xaxis.set_minor_locator(MonthLocator()) ax.set_xlim((dates[0] - 92, dates[len(dates) - 1] + 92)) ax.yaxis.set_major_formatter(FormatStrFormatter('%d')) ylabel('Total # of Public DAV Servers') lastdate = datetime.fromordinal(dates[len(dates) - 1]).strftime("%B %Y") xlabel("Data as of " + lastdate) title('Security Space Survey of\nPublic Subversion DAV Servers') # End drawing ########################################################### png = open(OUTPUT_FILE, 'w') savefig(png) png.close() os.rename(OUTPUT_FILE, OUTPUT_FILE + ".tmp.png") try: im = Image.open(OUTPUT_FILE + ".tmp.png", 'r') (width, height) = im.size print "Original size: %d x %d pixels" % (width, height) scale = float(OUTPUT_IMAGE_WIDTH) / float(width) width = OUTPUT_IMAGE_WIDTH height = int(float(height) * scale) print "Final size: %d x %d pixels" % (width, height) im = im.resize((width, height), Image.ANTIALIAS) im.save(OUTPUT_FILE, im.format) os.unlink(OUTPUT_FILE + ".tmp.png") except Exception, e: sys.stderr.write("Error attempting to resize the graphic: %s\n" % (str(e))) os.rename(OUTPUT_FILE + ".tmp.png", OUTPUT_FILE) raise
def get_cpr_for_cusip(self, cusip, fix_lag=True): """BBG Mortgage data is specified a month forward of the GNM files' 'as_of_date'. Probably b/c it's really as-of the middle of the specified month. Calling fix_lag will put it on the same footing as the GNM data, i.e., it'll still need to be lagged another month for prediction.""" #Check if we already have it cusip = "_" + cusip address = self.p_root + cusip[3:5] toc = self.hdfstore.get_node(address) address = address + "/" + cusip if toc is not None and cusip in toc: data = self.hdfstore[address] if data.name == 'blank': logger.error("No {} data for {}".format( self.data_name, cusip[1:])) data = None elif data.index.dtype == 'int64': #legacy conversion logger.info("Legacy conversion of stored {} data index".format( self.data_name)) data.index = [ datetime.fromordinal(x).date() for x in data.index ] self.hdfstore[address] = data else: data = self.bbg.get_hist_pool_data(cusip[1:], field_code=self.field) if len(data) > 0: data.name = cusip self.hdfstore.append(address, data) else: #No data available logger.error("No {} data for {}".format( self.data_name, cusip[1:])) data = pd.Series({0: 0}) data.name = 'blank' self.hdfstore.append(address, data) data = None #put in a blank entry if there's no data. self.hdfstore.flush(fsync=True) if fix_lag and data is not None: data = data.shift(-1).dropna() return data
def draw_graph(dates, counts): ########################################################### # Drawing takes place here. ax = subplot(111) ax.xaxis.set_major_formatter(DateFormatter('%b,%y')) ax.yaxis.set_major_formatter(FormatStrFormatter('%d')) line = bar(dates, counts, color='r', width=24) ylabel('Total # of Public DAV Servers') lastdate = datetime.fromordinal(dates[len(dates) - 1]).strftime("%B %Y") xlabel("Data as of " + lastdate) title('Security Space Survey of Public Subversion DAV Servers') # End drawing ########################################################### png = open(OUTPUT_FILE, 'w') savefig(png) png.close() close()
def draw_graph(dates, counts): ########################################################### # Drawing takes place here. figure(1) ax = subplot(111) plot_date(dates, counts, color='r', linestyle='-', marker='o', markersize=3) ax.xaxis.set_major_formatter( DateFormatter('%Y') ) ax.xaxis.set_major_locator( YearLocator() ) ax.xaxis.set_minor_locator( MonthLocator() ) ax.set_xlim( (dates[0] - 92, dates[len(dates) - 1] + 92) ) ax.yaxis.set_major_formatter( FormatStrFormatter('%d') ) ylabel('Total # of Public DAV Servers') lastdate = datetime.fromordinal(dates[len(dates) - 1]).strftime("%B %Y") xlabel("Data as of " + lastdate) title('Security Space Survey of\nPublic Subversion DAV Servers') # End drawing ########################################################### png = open(OUTPUT_FILE, 'w') savefig(png) png.close() os.rename(OUTPUT_FILE, OUTPUT_FILE + ".tmp.png") try: im = Image.open(OUTPUT_FILE + ".tmp.png", 'r') (width, height) = im.size print "Original size: %d x %d pixels" % (width, height) scale = float(OUTPUT_IMAGE_WIDTH) / float(width) width = OUTPUT_IMAGE_WIDTH height = int(float(height) * scale) print "Final size: %d x %d pixels" % (width, height) im = im.resize((width, height), Image.ANTIALIAS) im.save(OUTPUT_FILE, im.format) os.unlink(OUTPUT_FILE + ".tmp.png") except Exception, e: sys.stderr.write("Error attempting to resize the graphic: %s\n" % (str(e))) os.rename(OUTPUT_FILE + ".tmp.png", OUTPUT_FILE) raise
def dia_util(): cal = Brazil() cal.holidays(2019) # dia de hj hoje = datetime.now() ano = hoje.year mes = hoje.month dia = hoje.day #verifica data = ('{0}/{1}/{2}').format(ano, mes, dia) data = datetime.strptime(data, '%Y/%m/%d').date() varind = (cal.is_working_day((data))) # é domingo # Retorna se o dia atual é util verifica_dia = varind #verifica e altera se não for um dia util. while varind == False: data = datetime.fromordinal(data.toordinal() + 1) varind = (cal.is_working_day((data))) # é domingo # coleta o ANO, MES e DIA da data final para inserir no código ano = data.year mes = int(data.month) dia = int(data.day) if mes < 10: mes = '0{0}'.format(mes) if dia < 10: dia = '0{0}'.format(dia) datafinal = '{0}/{1}/{2}'.format(dia, mes, ano) return datafinal, verifica_dia
def get_cal_menu(self, qpoint): table = self.calendar._table item = table.itemAt(qpoint) day = item.data(QtCore.Qt.UserRole) date2 = None date3 = None tzone = clnxcfg.observer.timezone date = datetime.fromordinal(day.toordinal()) date = date.replace(hour=12, minute=0, second=0, tzinfo=tzone) if self.calendar.lunarReturn: idx = self.calendar.fetchLunarReturn(day) if idx >= 0: date2 = self.calendar.lunarReturns[idx] if self.calendar.solarReturn and day == self.calendar.solarReturnTime.date(): date3 = self.calendar.solarReturnTime #self.calendar.setGridVisible(True) menu = QtGui.QMenu(self.calendar) if date2: lritem = menu.addAction("Lunar Return for %s" %(date.strftime("%m/%d/%Y"))) lritem.triggered.connect(lambda: self.get_info_for_date(date2)) lritem.setIcon(QtGui.QIcon.fromTheme("dialog-information")) if date3: sritem = menu.addAction("Solar Return for %s" %(date.strftime("%m/%d/%Y"))) sritem.triggered.connect(lambda: self.get_info_for_date(date3)) sritem.setIcon(QtGui.QIcon.fromTheme("dialog-information")) infoitem = menu.addAction("Info for %s" %(date.strftime("%m/%d/%Y"))) infoitem.triggered.connect(lambda: self.get_info_for_date(date)) infoitem.setIcon(QtGui.QIcon.fromTheme("dialog-information")) copymenu = menu.addMenu("Copy") copymenu.setIcon(QtGui.QIcon.fromTheme("edit-copy")) copyall = copymenu.addAction("All") copydate = copymenu.addAction("Date") copyplanetdata = copymenu.addAction("Planetary Hours") copymoonphasedata = copymenu.addAction("Moon Phases") copysignsdata = copymenu.addAction("Signs for this date") copyeventdata = copymenu.addAction("Events") copyall.triggered.connect(lambda: self.copy_to_clipboard("All", date)) copydate.triggered.connect(lambda: app.clipboard().setText(date.strftime("%m/%d/%Y"))) copyplanetdata.triggered.connect(lambda: self.copy_to_clipboard("Planetary Hours", date)) copymoonphasedata.triggered.connect(lambda: self.copy_to_clipboard("Moon Phase", date)) copysignsdata.triggered.connect(lambda: self.copy_to_clipboard("Planetary Signs", date)) copyeventdata.triggered.connect(lambda: self.copy_to_clipboard("Events", date)) savemenu = menu.addMenu("Save to File") savemenu.setIcon(QtGui.QIcon.fromTheme("document-save-as")) saveall = savemenu.addAction("All") saveplanetdata = savemenu.addAction("Planetary Hours") savemoonphasedata = savemenu.addAction("Moon Phases") savesignsdata = savemenu.addAction("Signs for this date") saveeventdata = savemenu.addAction("Events") saveall.triggered.connect(lambda: self.print_to_file("All", date)) saveplanetdata.triggered.connect(lambda: self.print_to_file("Planetary Hours", date)) savemoonphasedata.triggered.connect(lambda: self.print_to_file("Moon Phase", date)) savesignsdata.triggered.connect(lambda: self.print_to_file("Planetary Signs", date)) saveeventdata.triggered.connect(lambda: self.print_to_file("Events", date)) menu.exec_(self.calendar.mapToGlobal(qpoint))
def _plot_res(results=None, original_pixel=None, band=None, file_name=None): """Plots CCD residual for a given band. Accepts a 1x1xt xarray if a scatter-plot overlay of original acquisitions over the ccd results is needed.""" fig = plt.figure(1, figsize=(20, 5)) fig.suptitle(band.title() + " Residual ", fontsize=18, verticalalignment='bottom') lastdt = None dateLabels = [] for change_model in results["change_models"]: ax1 = fig.add_subplot(111) model = getattr(change_model, band) time = original_pixel.sel( time=slice(datetime.fromordinal(change_model.start_day), datetime.fromordinal(change_model.end_day))).time.values ordinal_time = list(map(n64_to_ordinal, time)) actual = original_pixel[band].values predicted = list( map( partial(ccd._lasso_eval, bias=model.intercept, weights=model.coefficients), ordinal_time)) residual = list( map(lambda x: euclidean(x[0], x[1]), zip(actual, predicted))) x = time y = residual ax1.plot(x, y, label=model.coefficients) # Curve Fit Code polycoeffs = np.polyfit(ordinal_time, residual, 5) f = np.poly1d(polycoeffs) x_new = np.linspace(ordinal_time[0], ordinal_time[-1], 50) y_new = f(x_new) ax1.plot(x_new, y_new, c="#333333") # Draw Vertical Lines dt = datetime.fromordinal(change_model.start_day) dateLabels.append(dt) if lastdt is not None: ax1.axvspan(lastdt, dt, color=(0, 0, 0, 0.1)) dt = datetime.fromordinal(change_model.end_day) dateLabels.append(dt) lastdt = dt ymin, ymax = ax1.get_ylim() for idx, dt in enumerate(dateLabels): plt.axvline(x=dt, linestyle='dotted', color=(0, 0, 0, 0.5)) # Top, inside plt.text( dt, ymax, "\n" + # HACK TO FIX SPACING dt.strftime('%b %d') + " \n" # HACK TO FIX SPACING , rotation=90, horizontalalignment='right' if (idx % 2) else 'left', verticalalignment='top') plt.tight_layout() if file_name is not None: _save_plot_to_file(plot=plt, file=filename, band_name=band) plt.show()
print(d.replace(year=2005)) # new object : datetime.date(2005, 3, 11)) print(d) # unchanged ######### class datetime is a subclass of datetime with timezone, utc # class datetime.datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) from time import time # import function time() from datetime import datetime # import function if not must write datetime.datetime() print(datetime.today()) # localtime print(datetime.utcnow()) # utc time print(datetime.fromtimestamp(time())) # time() -> timestamp as float print(datetime.utcfromtimestamp(time())) # utc print(datetime.fromordinal(100)) # arg int # classmethod datetime.combine(date, time, tzinfo=self.tzinfo)¶ print(d,datetime.combine(d, datetime.time(datetime(2014,12,1,8,15,20)))) # YYYY-MM-DD[*HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]]] print(datetime.fromisoformat('2014-12-21*12:05:45')) print(datetime.fromisoformat('2014-12-21+11:05:08')) # example print('Examples 2') d = datetime.fromisoformat('2011-11-04') print(d) d = datetime.fromisoformat('2011-11-04T00:05:23') print(d) d = datetime.fromisoformat('2011-11-04 00:05:23.283')
def get_total_days(self): now = datetime.utcnow() ordinal = datetime.fromordinal(1) totalNumDays = (now - ordinal).days return totalNumDays
def random_date(): start_date = datetime.now().replace(day=1, month=1).toordinal() end_date = datetime.now().toordinal() random_day = datetime.fromordinal(random.randint(start_date, end_date)) return random_day
Xtrain['Date'] = Xtrain['Date'] # In[370]: scaler = StandardScaler() xtrain = scaler.fit_transform(Xtrain.values) xtest = scaler.transform(Xtest.values) # In[376]: from datetime import datetime as dt traindateseries = pd.Series( Xtrain['Date'].apply(lambda x: dt.fromordinal(x)).dt.strftime('%Y-%m-%d')) testdateseries = pd.Series( Xtrain['Date'].apply(lambda x: dt.fromordinal(x)).dt.strftime('%Y-%m-%d')) traindateseries # In[ ]: # In[ ]: # In[ ]: # In[377]: Xtest.info() # In[ ]:
def datenum_to_datetime(datenum): datenum = float(datenum) days = datenum % 1 return datetime.fromordinal(int(datenum)) \ + timedelta(days=days) \ - timedelta(days=366)
def datenum_mat_2_python_datetime(datenum_vec): Pydater = datetime.fromordinal(int(datenum_vec)) + timedelta( days=datenum_vec % 1) - timedelta(days=366) return Pydater
def __init__(self, split, data_dir=None): super(ExampleDataset, self).__init__() if split == 'train': time_start = 0 # TODO: Training Data의 종료일 기입 time_end = datetime(2013, 8, 31, tzinfo=self.TZ).toordinal() elif split == 'test': # TODO: Test 데이터의 시작일/종료일 기입 time_start = datetime(2013, 9, 1, tzinfo=self.TZ).toordinal() time_end = datetime(2014, 1, 1, tzinfo=self.TZ).toordinal() else: raise ValueError('invalid split', split) #TODO: 데이터의 시작일 기입 self.FIRST_DATE = datetime(2012, 12, 28, tzinfo=self.TZ) #TODO: 시간별로 테스트데이터를 n개의 Slot으로 나눔 (n=6) self.TEST_TIMESLOTS = [ datetime(2013, 9, 1, tzinfo=self.TZ), datetime(2013, 9, 25, tzinfo=self.TZ), datetime(2013, 10, 20, tzinfo=self.TZ), datetime(2013, 11, 15, tzinfo=self.TZ), datetime(2013, 12, 10, tzinfo=self.TZ), datetime(2014, 1, 1, tzinfo=self.TZ) ] self.N_nodes = 100 self.A_initial = np.random.randint(0, 2, size=(self.N_nodes, self.N_nodes)) self.A_last = np.random.randint(0, 2, size=(self.N_nodes, self.N_nodes)) print('\nA_initial', np.sum(self.A_initial)) print('A_last', np.sum(self.A_last), '\n') #TODO: 이벤트 갯수 정함 self.n_events = 10000 all_events = [] for i in range(self.n_events): user_id1 = np.random.randint(0, self.N_nodes) user_id2 = np.random.choice( np.delete(np.arange(self.N_nodes), user_id1)) #u1이 아닌 사람들 중에 한명 ts = max((time_start, self.FIRST_DATE.toordinal())) #시작시간을 정함 # 시작시간 ~ 끝시간 사이값으로 이벤트 정함 event_time = datetime.fromordinal( ts + np.random.randint(0, time_end - ts)) #이벤트 시간이 시작일보다는 커야 한다! assert event_time.timestamp() >= self.FIRST_DATE.timestamp(), ( event_time, self.FIRST_DATE) all_events.append( (user_id1, user_id2, np.random.choice(['communication event', 'association event']), event_time)) #TODO: 모든 이벤트 타입에 대해 여기다가 추가함! self.event_types = ['communication event'] self.all_events = sorted(all_events, key=lambda t: t[3].timestamp()) print('\n%s' % split.upper()) print('%d events between %d users loaded' % (len(self.all_events), self.N_nodes)) print('%d communication events' % (len([t for t in self.all_events if t[2] == 1]))) print('%d assocition events' % (len([t for t in self.all_events if t[2] == 0]))) #'association event'는 0으로 지정해두고, 나머지 이벤트는 1부터 증가하는 값으로 인덱싱 self.event_types_num = {'association event': 0} k = 1 # k >= 1 for communication events for t in self.event_types: self.event_types_num[t] = k k += 1 self.n_events = len(self.all_events)
def main_page(request): if request.user.is_authenticated: incomes = FlowOfFunds.objects.filter(family_id=request.user) sum = 0 #sum = FlowOfFunds.objects.filter(family_id=request.user).aggregate(Sum('sum')) #как это работает? for inc in incomes: sum = sum + inc.sum if SavingMoney.objects.filter(user_id=request.user): today = datetime.today() ''' year = int(today[0:4]) month = int(today[5:7]) day = int(today[8:10]) ''' sal_day = SavingMoney.objects.filter( user_id=request.user).first().salary_day # start_salary_date = datetime(year, month, sal_day) today_date = datetime.fromordinal(today.toordinal()) if sal_day < today.day: if today.month < 12: finish_salary_date = datetime(today.year, today.month + 1, sal_day) else: finish_salary_date = datetime(today.year + 1, 1, sal_day) else: n = sal_day - today.day finish_salary_date = today_date + timedelta(days=n) delta = (finish_salary_date - today_date).days plans = ExpensesPlan.objects.filter(user_id=request.user) types = Categories.objects.filter(family_id=request.user, is_it_expense=True) i = 0 ''' names = [] while i < len(types): names.append(types.filter(type_name=types[i].type_name).first().type_name) i = i+1 ''' names = [type.type_name for type in types] all_plan_sum = [] all_money_spent = [] rest_of_money = [] n = 0 for plan in plans: all_plan_sum.append(plan.sum_plan) money_spent = FlowOfFunds.objects.filter( family_id=request.user, type_id=plan.type_id) mon_sum = 0 for mon in money_spent: mon_sum = mon_sum + mon.sum all_money_spent.append(abs(mon_sum)) rest_of_money.append(plan.sum_plan - abs(mon_sum)) final_list = [] fl = [] i = 0 for name in names: fl = [] fl.append(name) fl.append(all_plan_sum[i]) fl.append(all_money_spent[i]) fl.append(rest_of_money[i]) final_list.append(fl) i = i + 1 # days = SavingMoney.objects.filter(user_id=request.user).first().salary_day return render_to_response('index.html', { 'name': request.user.username, 'flag': True,\ 'flows': FlowOfFunds.objects.filter(family_id=request.user).order_by('date'),\ 'money': sum, 'delta': delta, 'plans': final_list,\ 'sum_euro': SavingMoney.objects.filter(user_id=request.user).first().euro,\ 'sum_dollars': SavingMoney.objects.filter(user_id=request.user).first().dollars,\ 'categories': Categories.objects.filter(family_id=request.user)} ) else: return render_to_response('index.html', { 'name': request.user.username, 'flag': True,\ 'flows': FlowOfFunds.objects.filter(family_id=request.user),\ 'money': sum,\ 'categories': Categories.objects.filter(family_id=request.user)} ) else: return render_to_response('index.html', {'flag': False})
def makeForecast(inputList): import urllib2# import urllib import datetime import time# import os import dateutil ###----------------------------------------------------------------------------- ### START part One "DATA COLLECTION" - getting fresch intraday data from YahooFinance API, to work with #### Get specified content for the instrument list, from file in Python root dir. yahoo_ticker_list = [] readThisFile = r'lista_' + inputList +'.txt' TickerFile = open(readThisFile) fleraTickers = TickerFile.read() yahoo_ticker_list = fleraTickers.split('\n') TickerFile.close() yahoo_RealNames_list = [] readThisFile = r'lista_' + inputList +'_RealNames.txt' TickerFile = open(readThisFile) fleraTickers = TickerFile.read() yahoo_RealNames_list = fleraTickers.split('\n') TickerFile.close() #### Get content for the FEATURE list, from file in Python root dir. FEATURES = [] readThisFile = r'FEATURES03.txt' featuresFile = open(readThisFile) fleraFeatures = featuresFile.read() FEATURES = fleraFeatures.split('\n') featuresFile.close() printToFile = ('DONE: reading instrument and features')#### LocationToSave = r"loooooooooooooooooooooogFile.txt" saveFile = open(LocationToSave,'a') saveFile.write(printToFile) #### Remove, possible old files in Python root dir. for eachTicker in yahoo_ticker_list: try: os.remove(r'for'+eachTicker+'_excel.xlsx') except Exception as e: print(str(e)) for eachTicker in yahoo_ticker_list: try: os.remove(r'EOD'+eachTicker+'.txt')### except Exception as e: printToFile = (str(e)) logFile = open('lo0gFile.txt','a') logFile.write("\n"+printToFile) logFile.close() print('DONE: deleting old files')#### time.sleep(4) #### Parse EOD data for every instrument from finance.yahoo and save it linebyline in a 1st .txt file for eachTicker in yahoo_ticker_list: try: urlToVisit = 'http://chartapi.finance.yahoo.com/instrument/1.0/'+eachTicker+'/chartdata;type=quote;range=2y/csv' sourceCode = urllib2.urlopen(urlToVisit).read() splitSource = sourceCode.split('\n') LocationToSave = r'EOD'+eachTicker+'.txt' for eachLine in splitSource: splitLine = eachLine.split(',') if len(splitLine)==6: if 'values' not in eachLine: saveFile = open(LocationToSave,'a') lineToWrite = eachLine+'\n' saveFile.write(lineToWrite) saveFile.close() time.sleep(1) ### in respect to yahoo.finance except Exception as e: print(str(e)) #pass print('DONE: parsing EOD data and save as EOD_ticker_txt')#### #### Parse 5min data for every instrument from yahoo AND save in 2nd .txt file for each ticker for eachTicker in yahoo_ticker_list: try: urlToVisit = 'http://chartapi.finance.yahoo.com/instrument/1.0/'+eachTicker+'/chartdata;type=quote;range=40d/csv' sourceCode = urllib2.urlopen(urlToVisit).read() splitSource = sourceCode.split('\n') LocationToSave = r'5min'+eachTicker+'.txt' for eachLine in splitSource: splitLine = eachLine.split(',') if len(splitLine)==6: if 'values' not in eachLine: saveFile = open(LocationToSave,'a') lineToWrite = eachLine+'\n' saveFile.write(lineToWrite) saveFile.close() time.sleep(1) ### in respect to yahoo.finance except Exception as e: print(str(e)) #pass print('DONE: parsing 5min data and save as 5min_ticker_txt')#### #### Sort out only todays 5min data from 2nd .txt file AND save todays data in a 3rd .txt file for each ticker for eachTicker in yahoo_ticker_list: try: FileLocation = r'5min'+eachTicker+'.txt' Unix,Open,High,Low,Close,Volume = np.loadtxt(FileLocation, unpack=True, delimiter=',') today = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d') UnixToday = datetime.datetime.fromtimestamp(Unix[-2]).strftime('%Y-%m-%d') if today != UnixToday: try: os.remove(r'EOD'+eachTicker+'.txt')### except Exception as e: print(str(e)) else: for x in range(1, 400):############# UnixToday = datetime.datetime.fromtimestamp(Unix[-x]).strftime('%Y-%m-%d') if today == UnixToday: forUnix = Unix[-x] forOpen = Open[-x] forHigh = High[-x] forLow = Low[-x] forClose = Close[-x] LocationToSave = r'todayEOD'+eachTicker+'.txt' saveFile = open(LocationToSave,'a') lineToWrite = str(forUnix)+','+str(forOpen)+','+str(forHigh)+','+str(forLow)+','+str(forClose)+'\n' saveFile.write(lineToWrite) saveFile.close() except Exception as e: #pass print(str(e)) e = str('DONE: sort out todays 5min data')#### printToFile = (str(e)) logFile = open('lo0gFile.txt','a') logFile.write(printToFile) logFile.close() #### Read the 3rd .txt file with only todays 5min data AND convert to EOD format to 4th .txt file for eachTicker in yahoo_ticker_list: try: FileLocation = r'todayEOD'+eachTicker+'.txt' Unix,Open,High,Low,Close = np.loadtxt(FileLocation, unpack=True, delimiter=',') NoLen = len(Unix) forUnix = datetime.datetime.fromtimestamp(Unix[-2]).strftime('%Y%m%d') forOpen = Open[-2] forHigh = np.amax(High[0:NoLen]) forLow = np.amin(Low[0:NoLen]) forClose = Close[0] # print(str(forUnix)+str(eachTicker)+str(UnixTodayInLoop)) LocationToSave = r'EODtoday'+eachTicker+'.txt' saveFile = open(LocationToSave,'w') lineToWrite = str(forUnix)+','+str(forOpen)+','+str(forHigh)+','+str(forLow)+','+str(forClose)+',1\n' saveFile.write(lineToWrite) saveFile.close() except Exception as e: #pass print(str(e)) print('DONE: convert 5min data tot EOD')#### printToFile = (str('DONE: convert 5min data tot EOD')) logFile = open('lo0gFile.txt','a') logFile.write("\n"+printToFile) logFile.close() #### append todays EOD from 4th .txt file to the list of all EOD data in 1st .txt file for eachTicker in yahoo_ticker_list: try: EODFile = open(r'EODtoday'+eachTicker+'.txt') EODline = EODFile.readlines() EODtoday = EODline[0] EODFile.close() LocationToSave = r'EOD'+eachTicker+'.txt' saveFile = open(LocationToSave,'a') lineToWrite = EODtoday saveFile.write(lineToWrite) saveFile.close() except Exception as e: print(str(e)) #pass ### END part One - getting fresch intraday data to work with ###----------------------------------------------------------------------------- ### START part Two "PREPROCESS" - Load fresh data and create all additional Features AND ### save all data to one .xlsx file for each ticker from datetime import datetime for eachTicker in yahoo_ticker_list: try: FileLocation = r'EOD'+eachTicker+'.txt' Date, Open, High, Low, Close, Volume = np.loadtxt(FileLocation, delimiter=',', unpack=True,converters={ 0: mdates.strpdate2num('%Y%m%d')}) Zeros = [1]*len(Date) date = int(Date[-1]) dt = datetime.fromordinal(date) ### create the individual Features ### START calculation of choosen list of FEATURES for the MACHINE LEARNING process ### _DayOfYear = float(dt.strftime('%j')) # part with calender based FEATURES _DayOfMonth = float(dt.strftime('%d')) _DayOfWeek = float(dt.strftime('%w')) # part with FEATURES based on % relative from last Close, _Diff_CtoH = round((Close[-1]-High[-1])/High[-1],3) _Diff_CtoH1 = round((Close[-1]-High[-2])/High[-2],3) _Diff_CtoH2 = round((Close[-1]-High[-3])/High[-3],3) _Diff_CtoH3 = round((Close[-1]-High[-4])/High[-4],3) _Diff_CtoH4 = round((Close[-1]-High[-5])/High[-5],3) _Diff_CtoH5 = round((Close[-1]-High[-6])/High[-6],3) _Diff_CtoH6 = round((Close[-1]-High[-7])/High[-7],3) _Diff_CtoH7 = round((Close[-1]-High[-8])/High[-8],3) _Diff_CtoH8 = round((Close[-1]-High[-9])/High[-9],3) _Diff_CtoH9 = round((Close[-1]-High[-10])/High[-10],3) _Diff_CtoH10 = round((Close[-1]-High[-11])/High[-11],3) _Diff_CtoH11 = round((Close[-1]-High[-12])/High[-12],3) _Diff_CtoH12 = round((Close[-1]-High[-13])/High[-13],3) _Diff_CtoH13 = round((Close[-1]-High[-14])/High[-14],3) _Diff_CtoH14 = round((Close[-1]-High[-15])/High[-15],3) _Diff_CtoH15 = round((Close[-1]-High[-16])/High[-16],3) _Diff_CtoH16 = round((Close[-1]-High[-17])/High[-17],3) _Diff_CtoH17 = round((Close[-1]-High[-18])/High[-18],3) _Diff_CtoH18 = round((Close[-1]-High[-19])/High[-19],3) _Diff_CtoH19 = round((Close[-1]-High[-20])/High[-20],3) _Diff_CtoH20 = round((Close[-1]-High[-21])/High[-21],3) _Diff_CtoH21 = round((Close[-1]-High[-22])/High[-22],3) _Diff_CtoH22 = round((Close[-1]-High[-23])/High[-23],3) _Diff_CtoH23 = round((Close[-1]-High[-24])/High[-24],3) _Diff_CtoH24 = round((Close[-1]-High[-25])/High[-25],3) _Diff_CtoH25 = round((Close[-1]-High[-26])/High[-26],3) _Diff_CtoL = round((Close[-1]-Low[-1])/Low[-1],3) _Diff_CtoL1 = round((Close[-1]-Low[-2])/Low[-2],3) _Diff_CtoL2 = round((Close[-1]-Low[-3])/Low[-3],3) _Diff_CtoL3 = round((Close[-1]-Low[-4])/Low[-4],3) _Diff_CtoL4 = round((Close[-1]-Low[-5])/Low[-5],3) _Diff_CtoL5 = round((Close[-1]-Low[-6])/Low[-6],3) _Diff_CtoL6 = round((Close[-1]-Low[-7])/Low[-7],3) _Diff_CtoL7 = round((Close[-1]-Low[-8])/Low[-8],3) _Diff_CtoL8 = round((Close[-1]-Low[-9])/Low[-9],3) _Diff_CtoL9 = round((Close[-1]-Low[-10])/Low[-10],3) _Diff_CtoL10 = round((Close[-1]-Low[-11])/Low[-11],3) _Diff_CtoL11 = round((Close[-1]-Low[-12])/Low[-12],3) _Diff_CtoL12 = round((Close[-1]-Low[-13])/Low[-13],3) _Diff_CtoL13 = round((Close[-1]-Low[-14])/Low[-14],3) _Diff_CtoL14 = round((Close[-1]-Low[-15])/Low[-15],3) _Diff_CtoL15 = round((Close[-1]-Low[-16])/Low[-16],3) _Diff_CtoL16 = round((Close[-1]-Low[-17])/Low[-17],3) _Diff_CtoL17 = round((Close[-1]-Low[-18])/Low[-18],3) _Diff_CtoL18 = round((Close[-1]-Low[-19])/Low[-19],3) _Diff_CtoL19 = round((Close[-1]-Low[-20])/Low[-20],3) _Diff_CtoL20 = round((Close[-1]-Low[-21])/Low[-21],3) _Diff_CtoL21 = round((Close[-1]-Low[-22])/Low[-22],3) _Diff_CtoL22 = round((Close[-1]-Low[-23])/Low[-23],3) _Diff_CtoL23 = round((Close[-1]-Low[-24])/Low[-24],3) _Diff_CtoL24 = round((Close[-1]-Low[-25])/Low[-25],3) _Diff_CtoL25 = round((Close[-1]-Low[-26])/Low[-26],3) _Diff_CtoO = round((Close[-1]-Open[-1])/Open[-1],3) _Diff_CtoO1 = round((Close[-1]-Open[-2])/Open[-2],3) _Diff_CtoO2 = round((Close[-1]-Open[-3])/Open[-3],3) _Diff_CtoO3 = round((Close[-1]-Open[-4])/Open[-4],3) _Diff_CtoO4 = round((Close[-1]-Open[-5])/Open[-5],3) _Diff_CtoO5 = round((Close[-1]-Open[-6])/Open[-6],3) _Diff_CtoO6 = round((Close[-1]-Open[-7])/Open[-7],3) _Diff_CtoO7 = round((Close[-1]-Open[-8])/Open[-8],3) _Diff_CtoO8 = round((Close[-1]-Open[-9])/Open[-9],3) _Diff_CtoO9 = round((Close[-1]-Open[-10])/Open[-10],3) _Diff_CtoC1 = round((Close[-1]-Close[-1])/Close[-1],3) _Diff_CtoC2 = round((Close[-1]-Close[-3])/Close[-3],3) _Diff_CtoC3 = round((Close[-1]-Close[-4])/Close[-4],3) _Diff_CtoC4 = round((Close[-1]-Close[-5])/Close[-5],3) _Diff_CtoC5 = round((Close[-1]-Close[-6])/Close[-6],3) _Diff_CtoC6 = round((Close[-1]-Close[-7])/Close[-7],3) _Diff_CtoC7 = round((Close[-1]-Close[-8])/Close[-8],3) _Diff_CtoC8 = round((Close[-1]-Close[-9])/Close[-9],3) _Diff_CtoC9 = round((Close[-1]-Close[-10])/Close[-10],3) ### PART with onle basic HLOC data _justOpen = Open[-1] _justHigh = High[-1] _justLow = Low[-1] _justClose = Close[-1] ### PART with FEATURES based on % relation Close or Sub-indicator to upper/lower BollingerBand _SMA_H3 = float(round(np.sum(High[:-4:-1])/5,2)) # short moving average based on H & L _SMA_L3 = float(round(np.sum(Low[:-4:-1])/5,2)) # this two are sub-indicators _BBU5 = round(np.sum(Close[:-4:-1])/5,3)+(round(np.std(Close[-4:-1])*2,3)) # Upper BollingerBand _BBD5 = round(np.sum(Close[:-4:-1])/5,3)-(round(np.std(Close[-4:-1])*2,3)) # Lower BollingerBand _DiffU5_C = round((Close[-1]-_BBU5)/_BBU5,3) _DiffD5_C = round((Close[-1]-_BBD5)/_BBD5,3) _BBU13 = round(np.sum(Close[:-12:-1])/13,3)+(round(np.std(Close[:-12:-1])*2,3)) _BBD13 = round(np.sum(Close[:-12:-1])/13,3)-(round(np.std(Close[:-12:-1])*2,3)) _DiffU13_L3 = round((_SMA_L3-_BBU13)/_BBU13,3) _BBU21 = round(np.sum(Close[:-20:-1])/21,3)+(round(np.std(Close[:-20:-1])*2,3)) _BBD21 = round(np.sum(Close[:-20:-1])/21,3)-(round(np.std(Close[:-20:-1])*2,3)) _DiffD21_C = round((Close[-1]-_BBD21)/_BBD21,3) _DiffD21_H3 = round((_SMA_H3-_BBD21)/_BBD21,3) _BBU34 = round(np.sum(Close[:-33:-1])/34,3)+(round(np.std(Close[:-33:-1])*2,3)) _BBD34 = round(np.sum(Close[:-33:-1])/34,3)-(round(np.std(Close[:-33:-1])*2,3)) _DiffU34_C = round((Close[-1]-_BBU34)/_BBU34,3) _DiffD34_H3 = round((_SMA_H3-_BBD34)/_BBD34,3) _BBU55 = round(np.sum(Close[:-54:-1])/55,3)+(round(np.std(Close[:-54:-1])*2,3)) _BBD55 = round(np.sum(Close[:-54:-1])/55,3)-(round(np.std(Close[:-54:-1])*2,3)) _DiffU55_L3 = round((_SMA_L3-_BBU55)/_BBU55,3) _BBU89 = round(np.sum(Close[:-88:-1])/89,3)+(round(np.std(Close[:-88:-1])*2,3)) _BBD89 = round(np.sum(Close[:-88:-1])/89,3)-(round(np.std(Close[:-88:-1])*2,3)) _DiffU89_C = round((Close[-1]-_BBU89)/_BBU89,3) _DiffU89_L3 = round((_SMA_L3-_BBU89)/_BBU89,3) _BBU144 = round(np.sum(Close[:-143:-1])/144,3)+(round(np.std(Close[:-143:-1])*2,3)) _BBD144 = round(np.sum(Close[:-143:-1])/144,3)-(round(np.std(Close[:-143:-1])*2,3)) _DiffU144_L3 = round((_SMA_L3-_BBU144)/_BBU144,3) _BBU233 = round(np.sum(Close[:-232:-1])/233,3)+(round(np.std(Close[:-232:-1])*2,3)) _BBD233 = round(np.sum(Close[:-232:-1])/233,3)-(round(np.std(Close[:-232:-1])*2,3)) _DiffU233_C = round((Close[-1]-_BBU233)/_BBU233,3) _BBU300 = round(np.sum(Close[:299:-1])/300,3)+(round(np.std(Close[:299:-1])*2,3)) _BBD300 = round(np.sum(Close[:299:-1])/300,3)-(round(np.std(Close[:299:-1])*2,3)) _DiffU300_C = round((Close[-1]-_BBU300)/_BBU300,3) _DiffD300_C = round((Close[-1]-_BBD300)/_BBD300,3) ### PART with % relation, Close to Maxium High or Low from varius days in history _High3_H = round((Close[-1]-np.amax(High[:-2:-1]))/np.amax(High[:-2:-1]),3) _High5_H = round((Close[-1]-np.amax(High[:-4:-1]))/np.amax(High[:-4:-1]),3) _High100_H = round((Close[-1]-np.amax(High[:-99:-1]))/np.amax(High[:-99:-1]),3) _High377_H = round((Close[-1]-np.amax(High[:-376:-1]))/np.amax(High[:-376:-1]),3) _Low3_L = round((Close[-1]-np.amin(Low[:-2:-1]))/np.amin(Low[:-2:-1]),3) _Low34_L = round((Close[-1]-np.amin(Low[:-33:-1]))/np.amin(Low[:-33:-1]),3) _Low55_L = round((Close[-1]-np.amin(Low[:-54:-1]))/np.amin(Low[:-54:-1]),3) _Hi3to5 = round((Close[-1]-np.amax(High[:-4:-2]))/np.amax(High[:-4:-2]),3) _Hi5to8 = round((Close[-1]-np.amax(High[:-7:-4]))/np.amax(High[:-7:-4]),3) _Hi8to13 = round((Close[-1]-np.amax(High[:-12:-7]))/np.amax(High[:-12:-7]),3) _Hi34to55 = round((Close[-1]-np.amax(High[:-54:-33]))/np.amax(High[:-54:-33]),3) _Hi233to377 = round((Close[-1]-np.amax(High[:-376:-232]))/np.amax(High[:-376:-232]),3) _Lo3to5 = round((Close[-1]-np.amin(Low[:-4:-2]))/np.amin(Low[:-4:-2]),3) _Lo233to377 = round((Close[-1]-np.amin(Low[:-376:-232]))/np.amin(Low[:-376:-232]),3) ### PART with simple aritmic Feature _EvNo5 = round((Close[-1]-5)/5,2) ### END calculation of choosen list of FEATURES for the MACHINE LEARNING process ### ### append the individual Features to Pandas Dataframe df = pd.DataFrame(columns = FEATURES) df = df.append({ '_DayOfYear':_DayOfYear, '_DayOfMonth':_DayOfMonth, '_DayOfWeek':_DayOfWeek, '_Diff_CtoH':_Diff_CtoH, '_Diff_CtoH1':_Diff_CtoH1, '_Diff_CtoH2':_Diff_CtoH2, '_Diff_CtoH3':_Diff_CtoH3, '_Diff_CtoH4':_Diff_CtoH4, '_Diff_CtoH5':_Diff_CtoH5, '_Diff_CtoH6':_Diff_CtoH6, '_Diff_CtoH7':_Diff_CtoH7, '_Diff_CtoH8':_Diff_CtoH8, '_Diff_CtoH9':_Diff_CtoH9, '_Diff_CtoH10':_Diff_CtoH10, '_Diff_CtoH11':_Diff_CtoH11, '_Diff_CtoH12':_Diff_CtoH12, '_Diff_CtoH13':_Diff_CtoH13, '_Diff_CtoH14':_Diff_CtoH14, '_Diff_CtoH15':_Diff_CtoH15, '_Diff_CtoH16':_Diff_CtoH16, '_Diff_CtoH17':_Diff_CtoH17, '_Diff_CtoH18':_Diff_CtoH18, '_Diff_CtoH19':_Diff_CtoH19, '_Diff_CtoH20':_Diff_CtoH20, '_Diff_CtoH21':_Diff_CtoH21, '_Diff_CtoH22':_Diff_CtoH22, '_Diff_CtoH23':_Diff_CtoH23, '_Diff_CtoH24':_Diff_CtoH24, '_Diff_CtoH25':_Diff_CtoH25, '_Diff_CtoL':_Diff_CtoL, '_Diff_CtoL1':_Diff_CtoL1, '_Diff_CtoL2':_Diff_CtoL2, '_Diff_CtoL3':_Diff_CtoL3, '_Diff_CtoL4':_Diff_CtoL4, '_Diff_CtoL5':_Diff_CtoL5, '_Diff_CtoL6':_Diff_CtoL6, '_Diff_CtoL7':_Diff_CtoL7, '_Diff_CtoL8':_Diff_CtoL8, '_Diff_CtoL9':_Diff_CtoL9, '_Diff_CtoL10':_Diff_CtoL10, '_Diff_CtoL11':_Diff_CtoL11, '_Diff_CtoL12':_Diff_CtoL12, '_Diff_CtoL13':_Diff_CtoL13, '_Diff_CtoL14':_Diff_CtoL14, '_Diff_CtoL15':_Diff_CtoL15, '_Diff_CtoL16':_Diff_CtoL16, '_Diff_CtoL17':_Diff_CtoL17, '_Diff_CtoL18':_Diff_CtoL18, '_Diff_CtoL19':_Diff_CtoL19, '_Diff_CtoL20':_Diff_CtoL20, '_Diff_CtoL21':_Diff_CtoL21, '_Diff_CtoL22':_Diff_CtoL22, '_Diff_CtoL23':_Diff_CtoL23, '_Diff_CtoL24':_Diff_CtoL24, '_Diff_CtoL25':_Diff_CtoL25, '_Diff_CtoO':_Diff_CtoO, '_Diff_CtoO1':_Diff_CtoO1, '_Diff_CtoO2':_Diff_CtoO2, '_Diff_CtoO3':_Diff_CtoO3, '_Diff_CtoO4':_Diff_CtoO4, '_Diff_CtoO5':_Diff_CtoO5, '_Diff_CtoO6':_Diff_CtoO6, '_Diff_CtoO7':_Diff_CtoO7, '_Diff_CtoO8':_Diff_CtoO8, '_Diff_CtoO9':_Diff_CtoO9, '_Diff_CtoC1':_Diff_CtoC1, '_Diff_CtoC2':_Diff_CtoC2, '_Diff_CtoC3':_Diff_CtoC3, '_Diff_CtoC4':_Diff_CtoC4, '_Diff_CtoC5':_Diff_CtoC5, '_Diff_CtoC6':_Diff_CtoC6, '_Diff_CtoC7':_Diff_CtoC7, '_Diff_CtoC8':_Diff_CtoC8, '_Diff_CtoC9':_Diff_CtoC9, '_justOpen':_justOpen, '_justHigh':_justHigh, '_justLow':_justLow, '_justClose':_justClose, '_DiffU5_C':_DiffU5_C, '_DiffD5_C':_DiffD5_C, '_DiffU13_L3':_DiffU13_L3, '_DiffD21_C':_DiffD21_C, '_DiffD21_H3':_DiffD21_H3, '_DiffU34_C':_DiffU34_C, '_DiffD34_H3':_DiffD34_H3, '_DiffU55_L3':_DiffU55_L3, '_DiffU89_C':_DiffU89_C, '_DiffU89_L3':_DiffU89_L3, '_DiffU144_L3':_DiffU144_L3, '_DiffU233_C':_DiffU233_C, '_DiffU300_C':_DiffU300_C, '_DiffD300_C':_DiffD300_C, '_High3_H':_High3_H, '_High5_H':_High5_H, '_High100_H':_High100_H, '_High377_H':_High377_H, '_Low3_L':_Low3_L, '_Low34_L':_Low34_L, '_Low55_L':_Low55_L, '_Hi3to5':_Hi3to5, '_Hi5to8':_Hi5to8, '_Hi8to13':_Hi8to13, '_Hi34to55':_Hi34to55, '_Hi233to377':_Hi233to377, '_Lo3to5':_Lo3to5, '_Lo233to377':_Lo233to377, '_EvNo5':_EvNo5, }, ignore_index = True) ### Write Pandas Datafram to .xlsx file FileLocation4excel = r'for'+eachTicker+'_excel.xlsx' df.to_excel(FileLocation4excel, index=False) except Exception as e: printToFile = (str(e)) logFile = open('lo0gFile.txt','a') logFile.write(printToFile) logFile.close() ### END part Two - Load fresh data and create all additional Features AND ### save all data to one .xlsx file for each ticker ###----------------------------------------------------------------------------- ### START part Three "PREDICTION PROCESS" - Load fresh data from .xlsc file and make Predictions from todays intraday data ### Cleaning up unnecessary files for eachTicker in yahoo_ticker_list: try: os.remove(r''+eachTicker+'.txt')### except Exception as e: print(str(e)) for eachTicker in yahoo_ticker_list: try: os.remove(r'EODtoday'+eachTicker+'.txt')### except Exception as e: print(str(e)) for eachTicker in yahoo_ticker_list: try: os.remove(r'todayEOD'+eachTicker+'.txt')### except Exception as e: print(str(e)) for eachTicker in yahoo_ticker_list: try: os.remove(r'5min'+eachTicker+'.txt')### except Exception as e: print(str(e)) ### END Cleaning up unnecessary files ### START part with letting the algo make prediction/Outlook from the fresh data from the .xlsx file printToFile = (str("Start makeing Forecasts")) logFile = open('lo0gFile.txt','a') logFile.write("\n" + printToFile) logFile.close() barChartName = [] #making some empty datalists barChartForecast = [] Ultimate_df2 = pd.DataFrame(columns = ['[Name]', '[Forecast]']) # creating empty Pandas dataframe for eachTicker, eachRealNames in zip(yahoo_ticker_list, yahoo_RealNames_list): try: global GLBeachName GLBeachName = eachRealNames Location = r'for'+eachTicker+'_excel.xlsx' data = pd.read_excel(Location) X = np.array(data[FEATURES].values) # making a Numpay array from the Pandas dataset y1 = data['_justClose'].values # saves the intraday close value ### START loading the saved .pkl files with trained algo information and creating final Prediction/Outlook ### Every joblib.load indicates one .pkl file ### The file name indicates for how many future days the algo is trained to predict the outcome for (1d = One day) ### The first 7 algos calculate on how high the Risk/Reward Ratio was for future move ### the second 7 algos calculate how Big the future move was, using a lot of averageing out to smother the result. ### Every trained algo give 5 predictions; StrogSell, Sell, Neutral, Buy and StrongBuy (5cat = five categories) ### Every predictions contains a percanage chanse for all of the 5 possible outcomes ### Each of the five predictions in % are stored in variable, for every trained algo logreg1 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat1d.pkl') Value1D = logreg1.predict_proba(X) Sell1D = round(Value1D[0][0],4) Under1D = round(Value1D[0][1],4) Hold1D = round(Value1D[0][2],4) Over1D = round(Value1D[0][3],4) Buy1D = round(Value1D[0][4],4) logreg2 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat2d.pkl') Value2D = logreg2.predict_proba(X) Sell2D = round(Value2D[0][0],4) Under2D = round(Value2D[0][1],4) Hold2D = round(Value2D[0][2],4) Over2D = round(Value2D[0][3],4) Buy2D = round(Value2D[0][4],4) logreg3 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat3d.pkl') Value3D = logreg3.predict_proba(X) Sell3D = round(Value3D[0][0],4) Under3D = round(Value3D[0][1],4) Hold3D = round(Value3D[0][2],4) Over3D = round(Value3D[0][3],4) Buy3D = round(Value3D[0][4],4) logreg5 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat5d.pkl') Value5D = logreg5.predict_proba(X) Sell5D = round(Value5D[0][0],4) Under5D = round(Value5D[0][1],4) Hold5D = round(Value5D[0][2],4) Over5D = round(Value5D[0][3],4) Buy5D = round(Value5D[0][4],4) logreg8 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat8d.pkl') Value8D = logreg8.predict_proba(X) Sell8D = round(Value8D[0][0],4) Under8D = round(Value8D[0][1],4) Hold8D = round(Value8D[0][2],4) Over8D = round(Value8D[0][3],4) Buy8D = round(Value8D[0][4],4) logreg13 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat13d.pkl') Value13D = logreg13.predict_proba(X) Sell13D = round(Value13D[0][0],4) Under13D = round(Value13D[0][1],4) Hold13D = round(Value13D[0][2],4) Over13D = round(Value13D[0][3],4) Buy13D = round(Value13D[0][4],4) logreg21 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat21d.pkl') Value21D = logreg21.predict_proba(X) Sell21D = round(Value21D[0][0],4) Under21D = round(Value21D[0][1],4) Hold21D = round(Value21D[0][2],4) Over21D = round(Value21D[0][3],4) Buy21D = round(Value21D[0][4],4) ### Part with algos based on calculating how Big the future move was roc_logreg1 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat1dROC.pkl') roc_Value1D = roc_logreg1.predict_proba(X) roc_Sell1D = round(roc_Value1D[0][0],4) roc_Under1D = round(roc_Value1D[0][1],4) roc_Hold1D = round(roc_Value1D[0][2],4) roc_Over1D = round(roc_Value1D[0][3],4) roc_Buy1D = round(roc_Value1D[0][4],4) roc_logreg2 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat2dROC.pkl') roc_Value2D = roc_logreg2.predict_proba(X) roc_Sell2D = round(roc_Value2D[0][0],4) roc_Under2D = round(roc_Value2D[0][1],4) roc_Hold2D = round(roc_Value2D[0][2],4) roc_Over2D = round(roc_Value2D[0][3],4) roc_Buy2D = round(roc_Value2D[0][4],4) roc_logreg3 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat3dROC.pkl') roc_Value3D = roc_logreg3.predict_proba(X) roc_Sell3D = round(roc_Value3D[0][0],4) roc_Under3D = round(roc_Value3D[0][1],4) roc_Hold3D = round(roc_Value3D[0][2],4) roc_Over3D = round(roc_Value3D[0][3],4) roc_Buy3D = round(roc_Value3D[0][4],4) roc_logreg5 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat5dROC.pkl') roc_Value5D = roc_logreg5.predict_proba(X) roc_Sell5D = round(roc_Value5D[0][0],4) roc_Under5D = round(roc_Value5D[0][1],4) roc_Hold5D = round(roc_Value5D[0][2],4) roc_Over5D = round(roc_Value5D[0][3],4) roc_Buy5D = round(roc_Value5D[0][4],4) roc_logreg8 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat8dROC.pkl') roc_Value8D = roc_logreg8.predict_proba(X) roc_Sell8D = round(roc_Value8D[0][0],4) roc_Under8D = round(roc_Value8D[0][1],4) roc_Hold8D = round(roc_Value8D[0][2],4) roc_Over8D = round(roc_Value8D[0][3],4) roc_Buy8D = round(roc_Value8D[0][4],4) roc_logreg13 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat13dROC.pkl') roc_Value13D = roc_logreg13.predict_proba(X) roc_Sell13D = round(roc_Value13D[0][0],4) roc_Under13D = round(roc_Value13D[0][1],4) roc_Hold13D = round(roc_Value13D[0][2],4) roc_Over13D = round(roc_Value13D[0][3],4) roc_Buy13D = round(roc_Value13D[0][4],4) roc_logreg21 = joblib.load(r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat21dROC.pkl') roc_Value21D = roc_logreg21.predict_proba(X) roc_Sell21D = round(roc_Value21D[0][0],4) roc_Under21D = round(roc_Value21D[0][1],4) roc_Hold21D = round(roc_Value21D[0][2],4) roc_Over21D = round(roc_Value21D[0][3],4) roc_Buy21D = round(roc_Value21D[0][4],4) ### END loading the saved .pkl files with trained algo information ### START building own indicators for final prediction values ### 1st stage, adding all StrongBuy percentage vs all StrongSell percantage ### continuing with comparing all Buy percantage vs all Sell percentage ### Doing this for the 7 algos based on future Pattern first Buy2stdav = Buy1D+Buy2D+Buy3D+Buy5D+Buy8D+Buy13D+Buy21D Sell2stdav = Sell1D+Sell2D+Sell3D+Sell5D+Sell8D+Sell13D+Sell21D Over1stdav = Over1D+Over2D+Over3D+Over5D+Over8D+Over13D+Over21D Under1stdav = Under1D+Under2D+Under3D+Under5D+Under8D+Under13D+Under21D ### 2nd stage, Comparing all positive percentage vs all negative percentage BuyVsSell = Buy2stdav-Sell2stdav OverVsUnder = (Over1stdav-Under1stdav)/27.0 ### using divide by 27 to normalise Strong vs Normal Buy/Sell preOutlook = BuyVsSell + OverVsUnder ### Will only use value if it is high enough, else = 0 pat_Outlook = float(0.0) if BuyVsSell > 0.010 and OverVsUnder > 0.010: pat_Outlook = round(preOutlook*100,2) elif BuyVsSell < -0.009 and OverVsUnder < -0.009: pat_Outlook = round(preOutlook*100,2) else: pass ### Doing this one mote time, for the 7 algos based on future Big move last roc_Buy2stdav = roc_Buy1D+roc_Buy2D+roc_Buy3D+roc_Buy5D+roc_Buy8D+roc_Buy13D+roc_Buy21D roc_Sell2stdav = roc_Sell1D+roc_Sell2D+roc_Sell3D+roc_Sell5D+roc_Sell8D+roc_Sell13D+roc_Sell21D roc_Over1stdav = roc_Over1D+roc_Over2D+roc_Over3D+roc_Over5D+roc_Over8D+roc_Over13D+roc_Over21D roc_Under1stdav = roc_Under1D+roc_Under2D+roc_Under3D+roc_Under5D+roc_Under8D+roc_Under13D+roc_Under21D roc_BuyVsSell = roc_Buy2stdav-roc_Sell2stdav roc_OverVsUnder = (roc_Over1stdav-roc_Under1stdav)/2.2 roc_preOutlook = (roc_BuyVsSell + roc_OverVsUnder)/6.1 roc_Outlook = float(0.0) if roc_BuyVsSell > 0.017 and roc_OverVsUnder > 0.017: roc_Outlook = round(roc_preOutlook*100,2) elif roc_BuyVsSell < -0.014 and roc_OverVsUnder < -0.014: roc_Outlook = round(roc_preOutlook*100,2) else: pass ### This value is the Final Outlook for the individual Instrument Outlook = round(pat_Outlook + (roc_Outlook),2) global GLBoutlook GLBoutlook = Outlook print(roc_Outlook) print(pat_Outlook) print(Outlook) ### In some circumstances I put Names to the Outlook values, if the value is not strong enough it gets set to 0 Trade = '--On Hold--' if roc_Outlook > 0.017 and pat_Outlook > 0.01 and Outlook > 0.04: Trade = 'BUY' elif roc_Outlook > 0.017 or pat_Outlook > 0.01 and Outlook > 0.04: Trade = 'Strong' elif roc_Outlook < -0.014 and pat_Outlook < -0.009 and Outlook < -0.009: Trade = 'SHORT' elif roc_Outlook < -0.014 or pat_Outlook < -0.009 and Outlook < -0.009: Trade = 'Weak' else: pass barChartName.append(GLBeachName) barChartForecast.append(GLBoutlook) Ultimate_df2 = Ultimate_df2.append({'[Name]':eachRealNames, '[Forecast]':Outlook, }, ignore_index = True) except Exception as e: printToFile = (str(e)) logFile = open('lo0gFile.txt','a') logFile.write("\n" + printToFile) logFile.close() printToFile = (str("Done makeing Forecasts")) logFile = open('lo0gFile.txt','a') logFile.write("\n" + printToFile) logFile.close() ### Saving the results to .xlsx file File2Location4excel = r'TEST_DB_excel.xlsx' import datetime import time# unixTimestamp = int(time.time()) timestamp = str(datetime.datetime.fromtimestamp(int(unixTimestamp)).strftime('%Y-%m-%d %H_%M')) print(timestamp) Ultimate_df = Ultimate_df2.sort('[Forecast]', ascending=False) print(Ultimate_df) global theList theList = Ultimate_df time.sleep(3) ### END part with letting the algo make prediction/Outlook from the fresh data from the .xlsx file ### START - Part that create barchart of all predictions in instrument list import matplotlib matplotlib.style.use('ggplot') import matplotlib.pyplot as plt import datetime import time# unixTimestamp = int(time.time()) timestamp = str(datetime.datetime.fromtimestamp(int(unixTimestamp)).strftime('%Y-%m-%d %H_%M')) pltTitle = str(timestamp + ' CET predictive algo IQ1.4: ' + inputList) Ultimate_df.set_index(["[Name]"],inplace=True) Ultimate_df.plot(kind='bar',alpha=0.75, rot=75, title="", legend=False) plt.xlabel("") fig1 = plt.gcf() fig1.set_size_inches(16, 9) plt.title(pltTitle,fontsize=26, fontweight='bold', color='#7f7f7f',family='Courier New') plt.show() plt.draw() fig1.savefig(r'C:\Users\UserTrader\Documents\_Image\\'+inputList+'.png', dpi=72)
import datetime print(datetime.datetime.today()) print(datetime.datetime.today().weekday()) # convert integer to a data from datetime import datetime dt = datetime.fromordinal(733828) print(dt) print(dt.weekday()) integer_date = datetime(2012, 10, 1, 0, 0) print(integer_date.toordinal()) print(integer_date.month) integer_date1 = datetime(2012, 10, 1, 0, 0) integer_date2 = datetime(2012, 12, 31, 0, 0) integer_date3 = datetime(2013, 12, 31, 0, 0) print(integer_date1.toordinal()) print(integer_date2.toordinal()) print(integer_date3.toordinal()) print(integer_date2.toordinal() - integer_date1.toordinal()) print(integer_date3.toordinal() - integer_date2.toordinal())
_Avg_CtoC21 = round( (np.average(Close[x + 1:x + 22]) - Close[x]) / Close[x], 4) _Men_CtoC21 = round( (np.median(Close[x + 1:x + 22]) - Close[x]) / Close[x], 4) _end21 = round( (_Avg_OtoO21 + _Men_OtoO21 + _Avg_HtoH21 + _Men_HtoH21 + _Avg_LtoL21 + _Men_LtoL21 + _Avg_CtoC21 + _Men_CtoC21) / 8, 4) except Exception as e: print(str('part14')) ### END Second part - calculate how Big the future move was, using a lot of averageing out to smother the result. ### START calculation of choosen list of FEATURES for the MACHINE LEARNING process ### #Get Date info from .txt file and convet it to string format date = int(Date[x]) dt = datetime.fromordinal(date) _DateStamp = str(dt.strftime('%Y-%m-%d')) #part with date related Features _DayOfYear = float(dt.strftime('%j')) _DayOfMonth = float(dt.strftime('%d')) _DayOfWeek = float(dt.strftime('%w')) _DayOfWeek = float(dt.strftime('%w')) #part with percentual relations with past price levels _Diff_CtoH = round((Close[x] - High[x]) / High[x], 3) _Diff_CtoH1 = round((Close[x] - High[x - 1]) / High[x - 1], 3) _Diff_CtoH2 = round((Close[x] - High[x - 2]) / High[x - 2], 3) _Diff_CtoH3 = round((Close[x] - High[x - 3]) / High[x - 3], 3) _Diff_CtoH4 = round((Close[x] - High[x - 4]) / High[x - 4], 3) _Diff_CtoH5 = round((Close[x] - High[x - 5]) / High[x - 5], 3) _Diff_CtoH6 = round((Close[x] - High[x - 6]) / High[x - 6], 3) _Diff_CtoH7 = round((Close[x] - High[x - 7]) / High[x - 7], 3)
def md2date(md, instr): return dt.fromordinal(md + dateOffset(instr).toordinal())
def makeForecast(inputList): import urllib2 # import urllib import datetime import time # import os import dateutil ###----------------------------------------------------------------------------- ### START part One "DATA COLLECTION" - getting fresch intraday data from YahooFinance API, to work with #### Get specified content for the instrument list, from file in Python root dir. yahoo_ticker_list = [] readThisFile = r'lista_' + inputList + '.txt' TickerFile = open(readThisFile) fleraTickers = TickerFile.read() yahoo_ticker_list = fleraTickers.split('\n') TickerFile.close() yahoo_RealNames_list = [] readThisFile = r'lista_' + inputList + '_RealNames.txt' TickerFile = open(readThisFile) fleraTickers = TickerFile.read() yahoo_RealNames_list = fleraTickers.split('\n') TickerFile.close() #### Get content for the FEATURE list, from file in Python root dir. FEATURES = [] readThisFile = r'FEATURES03.txt' featuresFile = open(readThisFile) fleraFeatures = featuresFile.read() FEATURES = fleraFeatures.split('\n') featuresFile.close() printToFile = ('DONE: reading instrument and features') #### LocationToSave = r"loooooooooooooooooooooogFile.txt" saveFile = open(LocationToSave, 'a') saveFile.write(printToFile) #### Remove, possible old files in Python root dir. for eachTicker in yahoo_ticker_list: try: os.remove(r'for' + eachTicker + '_excel.xlsx') except Exception as e: print(str(e)) for eachTicker in yahoo_ticker_list: try: os.remove(r'EOD' + eachTicker + '.txt') ### except Exception as e: printToFile = (str(e)) logFile = open('lo0gFile.txt', 'a') logFile.write("\n" + printToFile) logFile.close() print('DONE: deleting old files') #### time.sleep(4) #### Parse EOD data for every instrument from finance.yahoo and save it linebyline in a 1st .txt file for eachTicker in yahoo_ticker_list: try: urlToVisit = 'http://chartapi.finance.yahoo.com/instrument/1.0/' + eachTicker + '/chartdata;type=quote;range=2y/csv' sourceCode = urllib2.urlopen(urlToVisit).read() splitSource = sourceCode.split('\n') LocationToSave = r'EOD' + eachTicker + '.txt' for eachLine in splitSource: splitLine = eachLine.split(',') if len(splitLine) == 6: if 'values' not in eachLine: saveFile = open(LocationToSave, 'a') lineToWrite = eachLine + '\n' saveFile.write(lineToWrite) saveFile.close() time.sleep(1) ### in respect to yahoo.finance except Exception as e: print(str(e)) #pass print('DONE: parsing EOD data and save as EOD_ticker_txt') #### #### Parse 5min data for every instrument from yahoo AND save in 2nd .txt file for each ticker for eachTicker in yahoo_ticker_list: try: urlToVisit = 'http://chartapi.finance.yahoo.com/instrument/1.0/' + eachTicker + '/chartdata;type=quote;range=40d/csv' sourceCode = urllib2.urlopen(urlToVisit).read() splitSource = sourceCode.split('\n') LocationToSave = r'5min' + eachTicker + '.txt' for eachLine in splitSource: splitLine = eachLine.split(',') if len(splitLine) == 6: if 'values' not in eachLine: saveFile = open(LocationToSave, 'a') lineToWrite = eachLine + '\n' saveFile.write(lineToWrite) saveFile.close() time.sleep(1) ### in respect to yahoo.finance except Exception as e: print(str(e)) #pass print('DONE: parsing 5min data and save as 5min_ticker_txt') #### #### Sort out only todays 5min data from 2nd .txt file AND save todays data in a 3rd .txt file for each ticker for eachTicker in yahoo_ticker_list: try: FileLocation = r'5min' + eachTicker + '.txt' Unix, Open, High, Low, Close, Volume = np.loadtxt(FileLocation, unpack=True, delimiter=',') today = datetime.datetime.fromtimestamp( time.time()).strftime('%Y-%m-%d') UnixToday = datetime.datetime.fromtimestamp( Unix[-2]).strftime('%Y-%m-%d') if today != UnixToday: try: os.remove(r'EOD' + eachTicker + '.txt') ### except Exception as e: print(str(e)) else: for x in range(1, 400): ############# UnixToday = datetime.datetime.fromtimestamp( Unix[-x]).strftime('%Y-%m-%d') if today == UnixToday: forUnix = Unix[-x] forOpen = Open[-x] forHigh = High[-x] forLow = Low[-x] forClose = Close[-x] LocationToSave = r'todayEOD' + eachTicker + '.txt' saveFile = open(LocationToSave, 'a') lineToWrite = str(forUnix) + ',' + str( forOpen) + ',' + str(forHigh) + ',' + str( forLow) + ',' + str(forClose) + '\n' saveFile.write(lineToWrite) saveFile.close() except Exception as e: #pass print(str(e)) e = str('DONE: sort out todays 5min data') #### printToFile = (str(e)) logFile = open('lo0gFile.txt', 'a') logFile.write(printToFile) logFile.close() #### Read the 3rd .txt file with only todays 5min data AND convert to EOD format to 4th .txt file for eachTicker in yahoo_ticker_list: try: FileLocation = r'todayEOD' + eachTicker + '.txt' Unix, Open, High, Low, Close = np.loadtxt(FileLocation, unpack=True, delimiter=',') NoLen = len(Unix) forUnix = datetime.datetime.fromtimestamp( Unix[-2]).strftime('%Y%m%d') forOpen = Open[-2] forHigh = np.amax(High[0:NoLen]) forLow = np.amin(Low[0:NoLen]) forClose = Close[0] # print(str(forUnix)+str(eachTicker)+str(UnixTodayInLoop)) LocationToSave = r'EODtoday' + eachTicker + '.txt' saveFile = open(LocationToSave, 'w') lineToWrite = str(forUnix) + ',' + str(forOpen) + ',' + str( forHigh) + ',' + str(forLow) + ',' + str(forClose) + ',1\n' saveFile.write(lineToWrite) saveFile.close() except Exception as e: #pass print(str(e)) print('DONE: convert 5min data tot EOD') #### printToFile = (str('DONE: convert 5min data tot EOD')) logFile = open('lo0gFile.txt', 'a') logFile.write("\n" + printToFile) logFile.close() #### append todays EOD from 4th .txt file to the list of all EOD data in 1st .txt file for eachTicker in yahoo_ticker_list: try: EODFile = open(r'EODtoday' + eachTicker + '.txt') EODline = EODFile.readlines() EODtoday = EODline[0] EODFile.close() LocationToSave = r'EOD' + eachTicker + '.txt' saveFile = open(LocationToSave, 'a') lineToWrite = EODtoday saveFile.write(lineToWrite) saveFile.close() except Exception as e: print(str(e)) #pass ### END part One - getting fresch intraday data to work with ###----------------------------------------------------------------------------- ### START part Two "PREPROCESS" - Load fresh data and create all additional Features AND ### save all data to one .xlsx file for each ticker from datetime import datetime for eachTicker in yahoo_ticker_list: try: FileLocation = r'EOD' + eachTicker + '.txt' Date, Open, High, Low, Close, Volume = np.loadtxt( FileLocation, delimiter=',', unpack=True, converters={0: mdates.strpdate2num('%Y%m%d')}) Zeros = [1] * len(Date) date = int(Date[-1]) dt = datetime.fromordinal(date) ### create the individual Features ### START calculation of choosen list of FEATURES for the MACHINE LEARNING process ### _DayOfYear = float( dt.strftime('%j')) # part with calender based FEATURES _DayOfMonth = float(dt.strftime('%d')) _DayOfWeek = float(dt.strftime('%w')) # part with FEATURES based on % relative from last Close, _Diff_CtoH = round((Close[-1] - High[-1]) / High[-1], 3) _Diff_CtoH1 = round((Close[-1] - High[-2]) / High[-2], 3) _Diff_CtoH2 = round((Close[-1] - High[-3]) / High[-3], 3) _Diff_CtoH3 = round((Close[-1] - High[-4]) / High[-4], 3) _Diff_CtoH4 = round((Close[-1] - High[-5]) / High[-5], 3) _Diff_CtoH5 = round((Close[-1] - High[-6]) / High[-6], 3) _Diff_CtoH6 = round((Close[-1] - High[-7]) / High[-7], 3) _Diff_CtoH7 = round((Close[-1] - High[-8]) / High[-8], 3) _Diff_CtoH8 = round((Close[-1] - High[-9]) / High[-9], 3) _Diff_CtoH9 = round((Close[-1] - High[-10]) / High[-10], 3) _Diff_CtoH10 = round((Close[-1] - High[-11]) / High[-11], 3) _Diff_CtoH11 = round((Close[-1] - High[-12]) / High[-12], 3) _Diff_CtoH12 = round((Close[-1] - High[-13]) / High[-13], 3) _Diff_CtoH13 = round((Close[-1] - High[-14]) / High[-14], 3) _Diff_CtoH14 = round((Close[-1] - High[-15]) / High[-15], 3) _Diff_CtoH15 = round((Close[-1] - High[-16]) / High[-16], 3) _Diff_CtoH16 = round((Close[-1] - High[-17]) / High[-17], 3) _Diff_CtoH17 = round((Close[-1] - High[-18]) / High[-18], 3) _Diff_CtoH18 = round((Close[-1] - High[-19]) / High[-19], 3) _Diff_CtoH19 = round((Close[-1] - High[-20]) / High[-20], 3) _Diff_CtoH20 = round((Close[-1] - High[-21]) / High[-21], 3) _Diff_CtoH21 = round((Close[-1] - High[-22]) / High[-22], 3) _Diff_CtoH22 = round((Close[-1] - High[-23]) / High[-23], 3) _Diff_CtoH23 = round((Close[-1] - High[-24]) / High[-24], 3) _Diff_CtoH24 = round((Close[-1] - High[-25]) / High[-25], 3) _Diff_CtoH25 = round((Close[-1] - High[-26]) / High[-26], 3) _Diff_CtoL = round((Close[-1] - Low[-1]) / Low[-1], 3) _Diff_CtoL1 = round((Close[-1] - Low[-2]) / Low[-2], 3) _Diff_CtoL2 = round((Close[-1] - Low[-3]) / Low[-3], 3) _Diff_CtoL3 = round((Close[-1] - Low[-4]) / Low[-4], 3) _Diff_CtoL4 = round((Close[-1] - Low[-5]) / Low[-5], 3) _Diff_CtoL5 = round((Close[-1] - Low[-6]) / Low[-6], 3) _Diff_CtoL6 = round((Close[-1] - Low[-7]) / Low[-7], 3) _Diff_CtoL7 = round((Close[-1] - Low[-8]) / Low[-8], 3) _Diff_CtoL8 = round((Close[-1] - Low[-9]) / Low[-9], 3) _Diff_CtoL9 = round((Close[-1] - Low[-10]) / Low[-10], 3) _Diff_CtoL10 = round((Close[-1] - Low[-11]) / Low[-11], 3) _Diff_CtoL11 = round((Close[-1] - Low[-12]) / Low[-12], 3) _Diff_CtoL12 = round((Close[-1] - Low[-13]) / Low[-13], 3) _Diff_CtoL13 = round((Close[-1] - Low[-14]) / Low[-14], 3) _Diff_CtoL14 = round((Close[-1] - Low[-15]) / Low[-15], 3) _Diff_CtoL15 = round((Close[-1] - Low[-16]) / Low[-16], 3) _Diff_CtoL16 = round((Close[-1] - Low[-17]) / Low[-17], 3) _Diff_CtoL17 = round((Close[-1] - Low[-18]) / Low[-18], 3) _Diff_CtoL18 = round((Close[-1] - Low[-19]) / Low[-19], 3) _Diff_CtoL19 = round((Close[-1] - Low[-20]) / Low[-20], 3) _Diff_CtoL20 = round((Close[-1] - Low[-21]) / Low[-21], 3) _Diff_CtoL21 = round((Close[-1] - Low[-22]) / Low[-22], 3) _Diff_CtoL22 = round((Close[-1] - Low[-23]) / Low[-23], 3) _Diff_CtoL23 = round((Close[-1] - Low[-24]) / Low[-24], 3) _Diff_CtoL24 = round((Close[-1] - Low[-25]) / Low[-25], 3) _Diff_CtoL25 = round((Close[-1] - Low[-26]) / Low[-26], 3) _Diff_CtoO = round((Close[-1] - Open[-1]) / Open[-1], 3) _Diff_CtoO1 = round((Close[-1] - Open[-2]) / Open[-2], 3) _Diff_CtoO2 = round((Close[-1] - Open[-3]) / Open[-3], 3) _Diff_CtoO3 = round((Close[-1] - Open[-4]) / Open[-4], 3) _Diff_CtoO4 = round((Close[-1] - Open[-5]) / Open[-5], 3) _Diff_CtoO5 = round((Close[-1] - Open[-6]) / Open[-6], 3) _Diff_CtoO6 = round((Close[-1] - Open[-7]) / Open[-7], 3) _Diff_CtoO7 = round((Close[-1] - Open[-8]) / Open[-8], 3) _Diff_CtoO8 = round((Close[-1] - Open[-9]) / Open[-9], 3) _Diff_CtoO9 = round((Close[-1] - Open[-10]) / Open[-10], 3) _Diff_CtoC1 = round((Close[-1] - Close[-1]) / Close[-1], 3) _Diff_CtoC2 = round((Close[-1] - Close[-3]) / Close[-3], 3) _Diff_CtoC3 = round((Close[-1] - Close[-4]) / Close[-4], 3) _Diff_CtoC4 = round((Close[-1] - Close[-5]) / Close[-5], 3) _Diff_CtoC5 = round((Close[-1] - Close[-6]) / Close[-6], 3) _Diff_CtoC6 = round((Close[-1] - Close[-7]) / Close[-7], 3) _Diff_CtoC7 = round((Close[-1] - Close[-8]) / Close[-8], 3) _Diff_CtoC8 = round((Close[-1] - Close[-9]) / Close[-9], 3) _Diff_CtoC9 = round((Close[-1] - Close[-10]) / Close[-10], 3) ### PART with onle basic HLOC data _justOpen = Open[-1] _justHigh = High[-1] _justLow = Low[-1] _justClose = Close[-1] ### PART with FEATURES based on % relation Close or Sub-indicator to upper/lower BollingerBand _SMA_H3 = float(round(np.sum(High[:-4:-1]) / 5, 2)) # short moving average based on H & L _SMA_L3 = float(round(np.sum(Low[:-4:-1]) / 5, 2)) # this two are sub-indicators _BBU5 = round(np.sum(Close[:-4:-1]) / 5, 3) + (round( np.std(Close[-4:-1]) * 2, 3)) # Upper BollingerBand _BBD5 = round(np.sum(Close[:-4:-1]) / 5, 3) - (round( np.std(Close[-4:-1]) * 2, 3)) # Lower BollingerBand _DiffU5_C = round((Close[-1] - _BBU5) / _BBU5, 3) _DiffD5_C = round((Close[-1] - _BBD5) / _BBD5, 3) _BBU13 = round(np.sum(Close[:-12:-1]) / 13, 3) + (round( np.std(Close[:-12:-1]) * 2, 3)) _BBD13 = round(np.sum(Close[:-12:-1]) / 13, 3) - (round( np.std(Close[:-12:-1]) * 2, 3)) _DiffU13_L3 = round((_SMA_L3 - _BBU13) / _BBU13, 3) _BBU21 = round(np.sum(Close[:-20:-1]) / 21, 3) + (round( np.std(Close[:-20:-1]) * 2, 3)) _BBD21 = round(np.sum(Close[:-20:-1]) / 21, 3) - (round( np.std(Close[:-20:-1]) * 2, 3)) _DiffD21_C = round((Close[-1] - _BBD21) / _BBD21, 3) _DiffD21_H3 = round((_SMA_H3 - _BBD21) / _BBD21, 3) _BBU34 = round(np.sum(Close[:-33:-1]) / 34, 3) + (round( np.std(Close[:-33:-1]) * 2, 3)) _BBD34 = round(np.sum(Close[:-33:-1]) / 34, 3) - (round( np.std(Close[:-33:-1]) * 2, 3)) _DiffU34_C = round((Close[-1] - _BBU34) / _BBU34, 3) _DiffD34_H3 = round((_SMA_H3 - _BBD34) / _BBD34, 3) _BBU55 = round(np.sum(Close[:-54:-1]) / 55, 3) + (round( np.std(Close[:-54:-1]) * 2, 3)) _BBD55 = round(np.sum(Close[:-54:-1]) / 55, 3) - (round( np.std(Close[:-54:-1]) * 2, 3)) _DiffU55_L3 = round((_SMA_L3 - _BBU55) / _BBU55, 3) _BBU89 = round(np.sum(Close[:-88:-1]) / 89, 3) + (round( np.std(Close[:-88:-1]) * 2, 3)) _BBD89 = round(np.sum(Close[:-88:-1]) / 89, 3) - (round( np.std(Close[:-88:-1]) * 2, 3)) _DiffU89_C = round((Close[-1] - _BBU89) / _BBU89, 3) _DiffU89_L3 = round((_SMA_L3 - _BBU89) / _BBU89, 3) _BBU144 = round(np.sum(Close[:-143:-1]) / 144, 3) + (round( np.std(Close[:-143:-1]) * 2, 3)) _BBD144 = round(np.sum(Close[:-143:-1]) / 144, 3) - (round( np.std(Close[:-143:-1]) * 2, 3)) _DiffU144_L3 = round((_SMA_L3 - _BBU144) / _BBU144, 3) _BBU233 = round(np.sum(Close[:-232:-1]) / 233, 3) + (round( np.std(Close[:-232:-1]) * 2, 3)) _BBD233 = round(np.sum(Close[:-232:-1]) / 233, 3) - (round( np.std(Close[:-232:-1]) * 2, 3)) _DiffU233_C = round((Close[-1] - _BBU233) / _BBU233, 3) _BBU300 = round(np.sum(Close[:299:-1]) / 300, 3) + (round( np.std(Close[:299:-1]) * 2, 3)) _BBD300 = round(np.sum(Close[:299:-1]) / 300, 3) - (round( np.std(Close[:299:-1]) * 2, 3)) _DiffU300_C = round((Close[-1] - _BBU300) / _BBU300, 3) _DiffD300_C = round((Close[-1] - _BBD300) / _BBD300, 3) ### PART with % relation, Close to Maxium High or Low from varius days in history _High3_H = round( (Close[-1] - np.amax(High[:-2:-1])) / np.amax(High[:-2:-1]), 3) _High5_H = round( (Close[-1] - np.amax(High[:-4:-1])) / np.amax(High[:-4:-1]), 3) _High100_H = round( (Close[-1] - np.amax(High[:-99:-1])) / np.amax(High[:-99:-1]), 3) _High377_H = round((Close[-1] - np.amax(High[:-376:-1])) / np.amax(High[:-376:-1]), 3) _Low3_L = round( (Close[-1] - np.amin(Low[:-2:-1])) / np.amin(Low[:-2:-1]), 3) _Low34_L = round( (Close[-1] - np.amin(Low[:-33:-1])) / np.amin(Low[:-33:-1]), 3) _Low55_L = round( (Close[-1] - np.amin(Low[:-54:-1])) / np.amin(Low[:-54:-1]), 3) _Hi3to5 = round( (Close[-1] - np.amax(High[:-4:-2])) / np.amax(High[:-4:-2]), 3) _Hi5to8 = round( (Close[-1] - np.amax(High[:-7:-4])) / np.amax(High[:-7:-4]), 3) _Hi8to13 = round( (Close[-1] - np.amax(High[:-12:-7])) / np.amax(High[:-12:-7]), 3) _Hi34to55 = round((Close[-1] - np.amax(High[:-54:-33])) / np.amax(High[:-54:-33]), 3) _Hi233to377 = round((Close[-1] - np.amax(High[:-376:-232])) / np.amax(High[:-376:-232]), 3) _Lo3to5 = round( (Close[-1] - np.amin(Low[:-4:-2])) / np.amin(Low[:-4:-2]), 3) _Lo233to377 = round((Close[-1] - np.amin(Low[:-376:-232])) / np.amin(Low[:-376:-232]), 3) ### PART with simple aritmic Feature _EvNo5 = round((Close[-1] - 5) / 5, 2) ### END calculation of choosen list of FEATURES for the MACHINE LEARNING process ### ### append the individual Features to Pandas Dataframe df = pd.DataFrame(columns=FEATURES) df = df.append( { '_DayOfYear': _DayOfYear, '_DayOfMonth': _DayOfMonth, '_DayOfWeek': _DayOfWeek, '_Diff_CtoH': _Diff_CtoH, '_Diff_CtoH1': _Diff_CtoH1, '_Diff_CtoH2': _Diff_CtoH2, '_Diff_CtoH3': _Diff_CtoH3, '_Diff_CtoH4': _Diff_CtoH4, '_Diff_CtoH5': _Diff_CtoH5, '_Diff_CtoH6': _Diff_CtoH6, '_Diff_CtoH7': _Diff_CtoH7, '_Diff_CtoH8': _Diff_CtoH8, '_Diff_CtoH9': _Diff_CtoH9, '_Diff_CtoH10': _Diff_CtoH10, '_Diff_CtoH11': _Diff_CtoH11, '_Diff_CtoH12': _Diff_CtoH12, '_Diff_CtoH13': _Diff_CtoH13, '_Diff_CtoH14': _Diff_CtoH14, '_Diff_CtoH15': _Diff_CtoH15, '_Diff_CtoH16': _Diff_CtoH16, '_Diff_CtoH17': _Diff_CtoH17, '_Diff_CtoH18': _Diff_CtoH18, '_Diff_CtoH19': _Diff_CtoH19, '_Diff_CtoH20': _Diff_CtoH20, '_Diff_CtoH21': _Diff_CtoH21, '_Diff_CtoH22': _Diff_CtoH22, '_Diff_CtoH23': _Diff_CtoH23, '_Diff_CtoH24': _Diff_CtoH24, '_Diff_CtoH25': _Diff_CtoH25, '_Diff_CtoL': _Diff_CtoL, '_Diff_CtoL1': _Diff_CtoL1, '_Diff_CtoL2': _Diff_CtoL2, '_Diff_CtoL3': _Diff_CtoL3, '_Diff_CtoL4': _Diff_CtoL4, '_Diff_CtoL5': _Diff_CtoL5, '_Diff_CtoL6': _Diff_CtoL6, '_Diff_CtoL7': _Diff_CtoL7, '_Diff_CtoL8': _Diff_CtoL8, '_Diff_CtoL9': _Diff_CtoL9, '_Diff_CtoL10': _Diff_CtoL10, '_Diff_CtoL11': _Diff_CtoL11, '_Diff_CtoL12': _Diff_CtoL12, '_Diff_CtoL13': _Diff_CtoL13, '_Diff_CtoL14': _Diff_CtoL14, '_Diff_CtoL15': _Diff_CtoL15, '_Diff_CtoL16': _Diff_CtoL16, '_Diff_CtoL17': _Diff_CtoL17, '_Diff_CtoL18': _Diff_CtoL18, '_Diff_CtoL19': _Diff_CtoL19, '_Diff_CtoL20': _Diff_CtoL20, '_Diff_CtoL21': _Diff_CtoL21, '_Diff_CtoL22': _Diff_CtoL22, '_Diff_CtoL23': _Diff_CtoL23, '_Diff_CtoL24': _Diff_CtoL24, '_Diff_CtoL25': _Diff_CtoL25, '_Diff_CtoO': _Diff_CtoO, '_Diff_CtoO1': _Diff_CtoO1, '_Diff_CtoO2': _Diff_CtoO2, '_Diff_CtoO3': _Diff_CtoO3, '_Diff_CtoO4': _Diff_CtoO4, '_Diff_CtoO5': _Diff_CtoO5, '_Diff_CtoO6': _Diff_CtoO6, '_Diff_CtoO7': _Diff_CtoO7, '_Diff_CtoO8': _Diff_CtoO8, '_Diff_CtoO9': _Diff_CtoO9, '_Diff_CtoC1': _Diff_CtoC1, '_Diff_CtoC2': _Diff_CtoC2, '_Diff_CtoC3': _Diff_CtoC3, '_Diff_CtoC4': _Diff_CtoC4, '_Diff_CtoC5': _Diff_CtoC5, '_Diff_CtoC6': _Diff_CtoC6, '_Diff_CtoC7': _Diff_CtoC7, '_Diff_CtoC8': _Diff_CtoC8, '_Diff_CtoC9': _Diff_CtoC9, '_justOpen': _justOpen, '_justHigh': _justHigh, '_justLow': _justLow, '_justClose': _justClose, '_DiffU5_C': _DiffU5_C, '_DiffD5_C': _DiffD5_C, '_DiffU13_L3': _DiffU13_L3, '_DiffD21_C': _DiffD21_C, '_DiffD21_H3': _DiffD21_H3, '_DiffU34_C': _DiffU34_C, '_DiffD34_H3': _DiffD34_H3, '_DiffU55_L3': _DiffU55_L3, '_DiffU89_C': _DiffU89_C, '_DiffU89_L3': _DiffU89_L3, '_DiffU144_L3': _DiffU144_L3, '_DiffU233_C': _DiffU233_C, '_DiffU300_C': _DiffU300_C, '_DiffD300_C': _DiffD300_C, '_High3_H': _High3_H, '_High5_H': _High5_H, '_High100_H': _High100_H, '_High377_H': _High377_H, '_Low3_L': _Low3_L, '_Low34_L': _Low34_L, '_Low55_L': _Low55_L, '_Hi3to5': _Hi3to5, '_Hi5to8': _Hi5to8, '_Hi8to13': _Hi8to13, '_Hi34to55': _Hi34to55, '_Hi233to377': _Hi233to377, '_Lo3to5': _Lo3to5, '_Lo233to377': _Lo233to377, '_EvNo5': _EvNo5, }, ignore_index=True) ### Write Pandas Datafram to .xlsx file FileLocation4excel = r'for' + eachTicker + '_excel.xlsx' df.to_excel(FileLocation4excel, index=False) except Exception as e: printToFile = (str(e)) logFile = open('lo0gFile.txt', 'a') logFile.write(printToFile) logFile.close() ### END part Two - Load fresh data and create all additional Features AND ### save all data to one .xlsx file for each ticker ###----------------------------------------------------------------------------- ### START part Three "PREDICTION PROCESS" - Load fresh data from .xlsc file and make Predictions from todays intraday data ### Cleaning up unnecessary files for eachTicker in yahoo_ticker_list: try: os.remove(r'' + eachTicker + '.txt') ### except Exception as e: print(str(e)) for eachTicker in yahoo_ticker_list: try: os.remove(r'EODtoday' + eachTicker + '.txt') ### except Exception as e: print(str(e)) for eachTicker in yahoo_ticker_list: try: os.remove(r'todayEOD' + eachTicker + '.txt') ### except Exception as e: print(str(e)) for eachTicker in yahoo_ticker_list: try: os.remove(r'5min' + eachTicker + '.txt') ### except Exception as e: print(str(e)) ### END Cleaning up unnecessary files ### START part with letting the algo make prediction/Outlook from the fresh data from the .xlsx file printToFile = (str("Start makeing Forecasts")) logFile = open('lo0gFile.txt', 'a') logFile.write("\n" + printToFile) logFile.close() barChartName = [] #making some empty datalists barChartForecast = [] Ultimate_df2 = pd.DataFrame(columns=['[Name]', '[Forecast]' ]) # creating empty Pandas dataframe for eachTicker, eachRealNames in zip(yahoo_ticker_list, yahoo_RealNames_list): try: global GLBeachName GLBeachName = eachRealNames Location = r'for' + eachTicker + '_excel.xlsx' data = pd.read_excel(Location) X = np.array(data[FEATURES].values ) # making a Numpay array from the Pandas dataset y1 = data['_justClose'].values # saves the intraday close value ### START loading the saved .pkl files with trained algo information and creating final Prediction/Outlook ### Every joblib.load indicates one .pkl file ### The file name indicates for how many future days the algo is trained to predict the outcome for (1d = One day) ### The first 7 algos calculate on how high the Risk/Reward Ratio was for future move ### the second 7 algos calculate how Big the future move was, using a lot of averageing out to smother the result. ### Every trained algo give 5 predictions; StrogSell, Sell, Neutral, Buy and StrongBuy (5cat = five categories) ### Every predictions contains a percanage chanse for all of the 5 possible outcomes ### Each of the five predictions in % are stored in variable, for every trained algo logreg1 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat1d.pkl') Value1D = logreg1.predict_proba(X) Sell1D = round(Value1D[0][0], 4) Under1D = round(Value1D[0][1], 4) Hold1D = round(Value1D[0][2], 4) Over1D = round(Value1D[0][3], 4) Buy1D = round(Value1D[0][4], 4) logreg2 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat2d.pkl') Value2D = logreg2.predict_proba(X) Sell2D = round(Value2D[0][0], 4) Under2D = round(Value2D[0][1], 4) Hold2D = round(Value2D[0][2], 4) Over2D = round(Value2D[0][3], 4) Buy2D = round(Value2D[0][4], 4) logreg3 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat3d.pkl') Value3D = logreg3.predict_proba(X) Sell3D = round(Value3D[0][0], 4) Under3D = round(Value3D[0][1], 4) Hold3D = round(Value3D[0][2], 4) Over3D = round(Value3D[0][3], 4) Buy3D = round(Value3D[0][4], 4) logreg5 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat5d.pkl') Value5D = logreg5.predict_proba(X) Sell5D = round(Value5D[0][0], 4) Under5D = round(Value5D[0][1], 4) Hold5D = round(Value5D[0][2], 4) Over5D = round(Value5D[0][3], 4) Buy5D = round(Value5D[0][4], 4) logreg8 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat8d.pkl') Value8D = logreg8.predict_proba(X) Sell8D = round(Value8D[0][0], 4) Under8D = round(Value8D[0][1], 4) Hold8D = round(Value8D[0][2], 4) Over8D = round(Value8D[0][3], 4) Buy8D = round(Value8D[0][4], 4) logreg13 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat13d.pkl') Value13D = logreg13.predict_proba(X) Sell13D = round(Value13D[0][0], 4) Under13D = round(Value13D[0][1], 4) Hold13D = round(Value13D[0][2], 4) Over13D = round(Value13D[0][3], 4) Buy13D = round(Value13D[0][4], 4) logreg21 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat21d.pkl') Value21D = logreg21.predict_proba(X) Sell21D = round(Value21D[0][0], 4) Under21D = round(Value21D[0][1], 4) Hold21D = round(Value21D[0][2], 4) Over21D = round(Value21D[0][3], 4) Buy21D = round(Value21D[0][4], 4) ### Part with algos based on calculating how Big the future move was roc_logreg1 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat1dROC.pkl' ) roc_Value1D = roc_logreg1.predict_proba(X) roc_Sell1D = round(roc_Value1D[0][0], 4) roc_Under1D = round(roc_Value1D[0][1], 4) roc_Hold1D = round(roc_Value1D[0][2], 4) roc_Over1D = round(roc_Value1D[0][3], 4) roc_Buy1D = round(roc_Value1D[0][4], 4) roc_logreg2 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat2dROC.pkl' ) roc_Value2D = roc_logreg2.predict_proba(X) roc_Sell2D = round(roc_Value2D[0][0], 4) roc_Under2D = round(roc_Value2D[0][1], 4) roc_Hold2D = round(roc_Value2D[0][2], 4) roc_Over2D = round(roc_Value2D[0][3], 4) roc_Buy2D = round(roc_Value2D[0][4], 4) roc_logreg3 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat3dROC.pkl' ) roc_Value3D = roc_logreg3.predict_proba(X) roc_Sell3D = round(roc_Value3D[0][0], 4) roc_Under3D = round(roc_Value3D[0][1], 4) roc_Hold3D = round(roc_Value3D[0][2], 4) roc_Over3D = round(roc_Value3D[0][3], 4) roc_Buy3D = round(roc_Value3D[0][4], 4) roc_logreg5 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat5dROC.pkl' ) roc_Value5D = roc_logreg5.predict_proba(X) roc_Sell5D = round(roc_Value5D[0][0], 4) roc_Under5D = round(roc_Value5D[0][1], 4) roc_Hold5D = round(roc_Value5D[0][2], 4) roc_Over5D = round(roc_Value5D[0][3], 4) roc_Buy5D = round(roc_Value5D[0][4], 4) roc_logreg8 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat8dROC.pkl' ) roc_Value8D = roc_logreg8.predict_proba(X) roc_Sell8D = round(roc_Value8D[0][0], 4) roc_Under8D = round(roc_Value8D[0][1], 4) roc_Hold8D = round(roc_Value8D[0][2], 4) roc_Over8D = round(roc_Value8D[0][3], 4) roc_Buy8D = round(roc_Value8D[0][4], 4) roc_logreg13 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat13dROC.pkl' ) roc_Value13D = roc_logreg13.predict_proba(X) roc_Sell13D = round(roc_Value13D[0][0], 4) roc_Under13D = round(roc_Value13D[0][1], 4) roc_Hold13D = round(roc_Value13D[0][2], 4) roc_Over13D = round(roc_Value13D[0][3], 4) roc_Buy13D = round(roc_Value13D[0][4], 4) roc_logreg21 = joblib.load( r'C:\Users\UserTrader\Documents\_PKLfiles_02\IQ14_5cat21dROC.pkl' ) roc_Value21D = roc_logreg21.predict_proba(X) roc_Sell21D = round(roc_Value21D[0][0], 4) roc_Under21D = round(roc_Value21D[0][1], 4) roc_Hold21D = round(roc_Value21D[0][2], 4) roc_Over21D = round(roc_Value21D[0][3], 4) roc_Buy21D = round(roc_Value21D[0][4], 4) ### END loading the saved .pkl files with trained algo information ### START building own indicators for final prediction values ### 1st stage, adding all StrongBuy percentage vs all StrongSell percantage ### continuing with comparing all Buy percantage vs all Sell percentage ### Doing this for the 7 algos based on future Pattern first Buy2stdav = Buy1D + Buy2D + Buy3D + Buy5D + Buy8D + Buy13D + Buy21D Sell2stdav = Sell1D + Sell2D + Sell3D + Sell5D + Sell8D + Sell13D + Sell21D Over1stdav = Over1D + Over2D + Over3D + Over5D + Over8D + Over13D + Over21D Under1stdav = Under1D + Under2D + Under3D + Under5D + Under8D + Under13D + Under21D ### 2nd stage, Comparing all positive percentage vs all negative percentage BuyVsSell = Buy2stdav - Sell2stdav OverVsUnder = ( Over1stdav - Under1stdav ) / 27.0 ### using divide by 27 to normalise Strong vs Normal Buy/Sell preOutlook = BuyVsSell + OverVsUnder ### Will only use value if it is high enough, else = 0 pat_Outlook = float(0.0) if BuyVsSell > 0.010 and OverVsUnder > 0.010: pat_Outlook = round(preOutlook * 100, 2) elif BuyVsSell < -0.009 and OverVsUnder < -0.009: pat_Outlook = round(preOutlook * 100, 2) else: pass ### Doing this one mote time, for the 7 algos based on future Big move last roc_Buy2stdav = roc_Buy1D + roc_Buy2D + roc_Buy3D + roc_Buy5D + roc_Buy8D + roc_Buy13D + roc_Buy21D roc_Sell2stdav = roc_Sell1D + roc_Sell2D + roc_Sell3D + roc_Sell5D + roc_Sell8D + roc_Sell13D + roc_Sell21D roc_Over1stdav = roc_Over1D + roc_Over2D + roc_Over3D + roc_Over5D + roc_Over8D + roc_Over13D + roc_Over21D roc_Under1stdav = roc_Under1D + roc_Under2D + roc_Under3D + roc_Under5D + roc_Under8D + roc_Under13D + roc_Under21D roc_BuyVsSell = roc_Buy2stdav - roc_Sell2stdav roc_OverVsUnder = (roc_Over1stdav - roc_Under1stdav) / 2.2 roc_preOutlook = (roc_BuyVsSell + roc_OverVsUnder) / 6.1 roc_Outlook = float(0.0) if roc_BuyVsSell > 0.017 and roc_OverVsUnder > 0.017: roc_Outlook = round(roc_preOutlook * 100, 2) elif roc_BuyVsSell < -0.014 and roc_OverVsUnder < -0.014: roc_Outlook = round(roc_preOutlook * 100, 2) else: pass ### This value is the Final Outlook for the individual Instrument Outlook = round(pat_Outlook + (roc_Outlook), 2) global GLBoutlook GLBoutlook = Outlook print(roc_Outlook) print(pat_Outlook) print(Outlook) ### In some circumstances I put Names to the Outlook values, if the value is not strong enough it gets set to 0 Trade = '--On Hold--' if roc_Outlook > 0.017 and pat_Outlook > 0.01 and Outlook > 0.04: Trade = 'BUY' elif roc_Outlook > 0.017 or pat_Outlook > 0.01 and Outlook > 0.04: Trade = 'Strong' elif roc_Outlook < -0.014 and pat_Outlook < -0.009 and Outlook < -0.009: Trade = 'SHORT' elif roc_Outlook < -0.014 or pat_Outlook < -0.009 and Outlook < -0.009: Trade = 'Weak' else: pass barChartName.append(GLBeachName) barChartForecast.append(GLBoutlook) Ultimate_df2 = Ultimate_df2.append( { '[Name]': eachRealNames, '[Forecast]': Outlook, }, ignore_index=True) except Exception as e: printToFile = (str(e)) logFile = open('lo0gFile.txt', 'a') logFile.write("\n" + printToFile) logFile.close() printToFile = (str("Done makeing Forecasts")) logFile = open('lo0gFile.txt', 'a') logFile.write("\n" + printToFile) logFile.close() ### Saving the results to .xlsx file File2Location4excel = r'TEST_DB_excel.xlsx' import datetime import time # unixTimestamp = int(time.time()) timestamp = str( datetime.datetime.fromtimestamp( int(unixTimestamp)).strftime('%Y-%m-%d %H_%M')) print(timestamp) Ultimate_df = Ultimate_df2.sort('[Forecast]', ascending=False) print(Ultimate_df) global theList theList = Ultimate_df time.sleep(3) ### END part with letting the algo make prediction/Outlook from the fresh data from the .xlsx file ### START - Part that create barchart of all predictions in instrument list import matplotlib matplotlib.style.use('ggplot') import matplotlib.pyplot as plt import datetime import time # unixTimestamp = int(time.time()) timestamp = str( datetime.datetime.fromtimestamp( int(unixTimestamp)).strftime('%Y-%m-%d %H_%M')) pltTitle = str(timestamp + ' CET predictive algo IQ1.4: ' + inputList) Ultimate_df.set_index(["[Name]"], inplace=True) Ultimate_df.plot(kind='bar', alpha=0.75, rot=75, title="", legend=False) plt.xlabel("") fig1 = plt.gcf() fig1.set_size_inches(16, 9) plt.title(pltTitle, fontsize=26, fontweight='bold', color='#7f7f7f', family='Courier New') plt.show() plt.draw() fig1.savefig(r'C:\Users\UserTrader\Documents\_Image\\' + inputList + '.png', dpi=72)