def getAlldatabase(dbLoc):
dbLoc = dbLoc
files = utils.findFiles(dbLoc)
tableDict = {}
for file in files:
tableDict[file] = databaseCheck(file, dbLoc=dbLoc).tableColumnNames
return tableDict
def getAlldatabase(dbLoc):
'''
input : folder location
find all databse in the folder
'''
dbLoc = dbLoc
files = utils.findFiles(dbLoc)
tableDict = {}
for file in files :
tableDict[file] = databaseCheck(file, dbLoc = dbLoc).tableColumnNames
return tableDict
def strategyMix_summarySetter():
dbList = utils.findFiles(saveLoc + 'strategyMix/')
filt = pd.DataFrame()
for i in range(len(dbList)):
data = st.get_desc_ret(saveLoc + 'strategyMix/', dbList[i][:-3])
desc = st.descTest(data)
desc.make([st.annualized_ret, st.stdev, st.mdd, st.hit_ratio])
desc.ratio('annRet', 'stdev', 'riskAdj')
desc.rank('riskAdj')
filt = filt.append(desc.descriptor[0:20])
print(i)
df.insertDBLite(saveLoc + 'strategyMix/result/', 'strategyMix_summary', 'strategyMix_summary', filt)
def __init__(self, fn=None, p=None, persist=False, rstrs=None):
if self.init_flag: return # Found Specs in cache
if isinstance(fn, PhoneDB):
self.init(p, persist)
data = self.data
rec_order = self.rec_order
self.field_order = field_order = ['Item']
for i in fn.fieldOrder(): field_order.append(i)
count = 1
for rec in fn.getData():
k = str(count)
rec_order.append(k)
rec['Item'] = k
data[k] = rec
count += 1
self.init_flag = True
self.fn = fn.fn
return
self.init(p, persist)
p = self.path
self.lockfn = lockfn = os.path.join(p, '_lockfile_')
if fn is not None:
self.fn = fn = os.path.join(p, fn)
if os.path.exists(fn):
fl = FileLocker(lockfn)
if os.path.isdir(fn):
self.dir_f = True
for i in findFiles(path=fn):
if i == '_lockfile_': continue
self.readSpecs(os.path.join(fn,i), rstrs)
else:
self.readSpecs(fn, rstrs)
del fl
if rstrs is not None: self.fn = replaceMany(self.fn, rstrs)
else:
self.fn = fn
data = self.data
# Check for import files
if 'Import' in self.get(): self.importdata()
self.resolveRely()
self.init_flag = True
def parse(self, directory):
"""find CMakeLists.txt and parse it"""
if not os.path.exists(directory):
return False
fileNames = utils.findFiles(directory, "CMakelists.txt")
for fileName in fileNames:
with open(fileName, "r") as f:
for line in f.readlines():
if line.startswith("#"):
continue
if line.lower().find("find_package") > -1:
m = re.search(r"\(([a-zA-Z0-9]+)[ \)]", line.upper())
if m:
if not fileName in self.files1:
self.files1.append(fileName)
key = m.groups()
if key in self.packageIncludes:
if not fileName in self.packageIncludes[key]:
self.packageIncludes[key].append(fileName)
else:
self.packageIncludes[key] = [fileName]
# find package usage
for fileName in fileNames:
with open(fileName, "r") as f:
for _line in f.readlines():
line = _line.upper()
if line.startswith("#"):
continue
for key in self.packageIncludes:
keyInclude = "%s_INCLUDE" % key
keyLib = "%s_LIBRAR" % key
keyExec = "%s_EXEC" % key
if line.find(keyInclude) > -1 or line.find(
keyLib) > -1 or line.find(keyExec) > -1:
if not fileName in self.files2:
self.files2.append(fileName)
if key in self.packageUsage:
if not fileName in self.packageUsage[key]:
self.packageUsage[key].append(fileName)
else:
self.packageUsage[key] = [fileName]
return True
def parse(self, directory):
"""find CMakeLists.txt and parse it"""
if not os.path.exists(directory):
return False
fileNames = utils.findFiles(directory, "CMakelists.txt")
for fileName in fileNames:
with open(fileName, "r") as f:
for line in f.readlines():
if line.startswith("#"):
continue
if line.lower().find("find_package") > -1:
m = re.search(r"\(([a-zA-Z0-9]+)[ \)]", line.upper())
if m:
if not fileName in self.files1:
self.files1.append(fileName)
key = m.groups()
if key in self.packageIncludes:
if not fileName in self.packageIncludes[key]:
self.packageIncludes[key].append(fileName)
else:
self.packageIncludes[key] = [fileName]
# find package usage
for fileName in fileNames:
with open(fileName, "r") as f:
for _line in f.readlines():
line = _line.upper()
if line.startswith("#"):
continue
for key in self.packageIncludes:
keyInclude = "%s_INCLUDE" % key
keyLib = "%s_LIBRAR" % key
keyExec = "%s_EXEC" % key
if line.find(keyInclude) > -1 or line.find(keyLib) > -1 or line.find(keyExec) > -1:
if not fileName in self.files2:
self.files2.append(fileName)
if key in self.packageUsage:
if not fileName in self.packageUsage[key]:
self.packageUsage[key].append(fileName)
else:
self.packageUsage[key] = [fileName]
return True
def upload(self):
files = utils.findFiles(dbLoc)
if self.dbName + '.db' in files:
os.remove(dbLoc + self.dbName + '.db')
wb = xw.Book(self.excelLoc + self.excelName)
sht = wb.sheets(self.worksheetName)
if self.totalUpdate == True:
wb.macro('QntWS_RefresAll')()
data = pd.DataFrame(sht.range(self.dataLoc).value)
header = data.iloc[0]
data = pd.DataFrame(data.values[1:], columns=header)
data.replace('', np.nan, inplace=True)
data = data.dropna(thresh=1)
dateLists = list(data.columns)
dateLists = [dateList for dateList in dateLists if 'date' in dateList]
for dateList in dateLists:
data[dateList] = data[dateList].apply(lambda x: x if pd.isna(
x) else dt.datetime.strftime(x, '%Y-%m-%d'))
df.insertDBLite(self.dbLoc, self.dbName, self.tableName, data)
wb.save()
wb.close()
from setuptools import setup, Extension
from setuptools.config import read_configuration
import utils
from collections import namedtuple
from sys import exit
checker = utils.CheckCompiler('-std=c++14')
if not checker.run():
print("Cannot build pcm2mp3 unless compiler supports -std=c++14")
exit(1)
configuration = read_configuration('setup.cfg')
metadata = configuration['metadata']
src = utils.findFiles('/opt/git/OSGridConverter/cpp',
dirs=['src', 'python'],
pattern=r'^.*\.cpp$')
for f in src:
print(f)
Version = namedtuple('Version', ['major', 'minor', 'maintenance'])
def processVersion():
version = metadata['version']
parts = version.split('.')
if len(parts) < 3: parts.extend([0, 0, 0])
return Version(*(parts[:3]))
def makeExtension(module, src):
#print("Making {} with {}".format(module,src))
import pandas as pd
import os
import numpy as np
from utils import findFiles, readLines, letterToTensor, lineToTensor
from tqdm import tqdm
data_dir = '../../../data/data'
csv_file_name = 'train.csv'
if __name__ == '__main__':
category_lines = {}
all_categories = []
df = pd.DataFrame()
#file_names = os.listdir(os.path.join(data_dir, 'names'))
for filename in tqdm(findFiles(os.path.join(data_dir, 'names/*.txt')),
desc='Creating csv file'):
category = os.path.splitext(os.path.basename(filename))[0]
#all_categories.append(category)
lines = readLines(filename)
#category_lines[category] = lines
categories = [category for _ in range(len(lines))]
for (line, cat) in zip(lines, categories):
df = df.append({'name': line, 'target': cat}, ignore_index=True)
# shuffle datasets
df = df.sample(frac=1).reset_index(drop=True)
# save dataframe as csv file
df.to_csv(os.path.join(data_dir, csv_file_name), index=False)
print(f"[INFO] {csv_file_name} saved to {data_dir}")
def upload(self, thresh=4):
files = utils.findFiles(dbLoc)
if self.dbName + '.db' in files:
# 부분업데이트
db = df.dbLite(dbLoc, self.dbName)
lastUpdatedDate = db.getList(self.tableName, 'date')[-1]
updateStart = self.tradingDate.index[
self.tradingDate['date'] == lastUpdatedDate].tolist()[0] + 1
start = updateStart
end = len(self.tradingDate)
if start < end:
wb = xw.Book(self.excelLoc + self.excelName)
sht = wb.sheets(self.worksheetName)
sht.cells(7, 2).value = self.tradingDate.date[start]
# if crss is false then update before looping
if self.crss == False:
if self.totalUpdate == True:
wb.macro('QntWS_RefresAll')()
for i in range(start, end):
sht.cells(7, 2).value = self.tradingDate.date[i]
if (self.econ == True) and (i > 0):
sht.cells(10, 2).value = self.tradingDate.date[i - 1]
if self.crss == True:
wb.macro('QntWS_RefresAll')()
data = pd.DataFrame(sht.range(self.dataLoc).value)
header = data.iloc[0]
data = pd.DataFrame(data.values[1:], columns=header)
data.replace('', np.nan, inplace=True)
data = data.dropna(thresh=thresh)
data['date'] = data['date'].apply(
lambda x: dt.datetime.strftime(x, '%Y-%m-%d'))
df.insertDBLite(self.dbLoc, self.dbName, self.tableName,
data)
print('----------', self.dbName, str(i),
len(self.tradingDate), '---------')
wb.save()
wb.close()
else:
print(self.dbName, 'do not need to update')
else:
#전체 업데이트
wb = xw.Book(self.excelLoc + self.excelName)
sht = wb.sheets(self.worksheetName)
sht.cells(7, 2).value = self.tradingDate.date[0]
if (self.totalUpdate == True) and (self.crss == False):
wb.macro('QntWS_RefreshAll')()
for i in range(len(self.tradingDate)):
sht.cells(7, 2).value = self.tradingDate.date[i]
if (self.econ == True) and (i > 0):
sht.cells(10, 2).value = self.tradingDate.date[i - 1]
if self.crss == True:
wb.macro('QntWS_RefreshAll')()
data = pd.DataFrame(sht.range(self.dataLoc).value)
header = data.iloc[0]
data = pd.DataFrame(data.values[1:], columns=header)
data.replace('', np.nan, inplace=True)
data = data.dropna(thresh=thresh)
data['date'] = data['date'].apply(
lambda x: dt.datetime.strftime(x, '%Y-%m-%d'))
df.insertDBLite(self.dbLoc, self.dbName, self.tableName, data)
print('----------', self.dbName, str(i), len(self.tradingDate),
'---------')
# wb.macro('QntWS_RefreshAll')()
wb.save()
wb.close()
data['date'] = data['date'].apply(
lambda x: dt.datetime.strftime(x, '%Y-%m-%d'))
df.insertDBLite(self.dbLoc, self.dbName, self.tableName, data)
print('----------', self.dbName, str(i), len(self.tradingDate),
'---------')
# wb.macro('QntWS_RefreshAll')()
wb.save()
wb.close()
#%% database deletion
# print('hi')
if __name__ == '__main__':
db = ['stock_tech_ohlc', 'previousAdjPrice']
db = utils.findFiles('C:/Users/nine/dbLocKgh/db_ml_v1')
db = [d[:-3] for d in db]
dateList = ['2021-05-21']
for d in db:
for date in dateList:
print(str(d) + '_' + str(date))
dbM = df.dbLite(dbLoc, d)
dbM.deleteDB2(d, date)
# dbM = df.dbLite(dbLoc,'stock_con_fwd_fcf_roe_pe_pb_ps_pc')
# date = dbM.getList('stock_con_fwd_fcf_roe_pe_pb_ps_pc','date')
# data = dbM.getDateBulk('stock_con_fwd_fcf_roe_pe_pb_ps_pc','2021-03-30')
# dbM.deleteDB2('stock_con_fwd_fcf_roe_pe_pb_ps_pc','2021-03-30')
#%%