def __init__(self, WINDOW_LENGTH, EPOCHS, BATCH_SIZE, symbol, train=True):
"""
Constants used for things like
- WINDOW_LENGTH : how many elements of data are in a state
- EPOCHS : number of episodes the model is trained for
- BATCH_SIZE : how many cycles we go until we fit the agent's model
"""
self.WINDOW_LENGTH = WINDOW_LENGTH
self.EPOCHS = EPOCHS
self.BATCH_SIZE = BATCH_SIZE
self.AMPLIFIER = 1000
"""
Variables for the environment data (prices of the stock)
Train variable to determine if were being run in training mode or trading mode
"""
self.stock = GetData(symbol, train)
self.data = self.stock.format_data()
self.symbol = symbol
self.data_len = len(self.data)
self.train = train
self.p = Portfolio('lucas')
"""
Parameters that are subject to reset after every training episode
"""
self.buy_count = 0
self.sell_count = 0
self.active_positions = 0
self.history = []
def run_method(self): server = Server() server.main() data = GetData() action_method = ActionMethod() lines = data.get_case_lines() for i in range(1,lines): handle_step = data.get_handle_step(i) element_key = data.get_element_key(i) handle_value = data.get_handle_value(i) expect_key = data.get_expect_element(i) expect_step = data.get_expect_handle(i) #input() login_button #input str #None excute_method = getattr(action_method,handle_step) if element_key != None: excute_method(element_key,handle_value) else: excute_method(handle_value) if expect_step != None: expect_result = getattr(action_method,expect_step) result = expect_result(expect_key) if result: data.write_value(i,"pass") else: data.write_value(i,"fail")
def create_dataframe(index, year):
"""
Creates pandas dataframe that will be used in visualization
:param index: str
:param year: str
:return: dataframe
"""
# Define variables
data = {'Country Name': [], 'Country Code': [], year: []}
# Create datafreme
for country in ISO:
if index == '1':
value = cpi_info(country, year)
else:
value = GetData(country, int(index), year).export_data()
if value is None:
value = np.nan
data['Country Name'].append(country)
data['Country Code'].append(ISO[country])
data[year].append(value)
df = pd.DataFrame(data, columns=['Country Name', 'Country Code', year])
return df
def weatherTask(self):
data = GetData(top, mdl, leaf, service_key)
# 기상청으로부터 지역의 X, Y좌표를 획득
data.setRegion()
# 기상청에서 요구하는 형태의 날짜와 시간 문자열 설정
date, time = self.timeString()
data.setTime(date, time)
# 정보수집
weather = data.getData()
# 딕셔너리에 정보 저장
for i in weather['response']['body']['items']['item']:
self.weather_dictionary[i['category']] = i['obsrValue']
if time != self.time:
# 템플릿에 정보 대입
post = self.template.format(
str(settings["region"]),
str(self.weather_dictionary['T1H']) + "°C",
pty[self.weather_dictionary['PTY']],
str(self.weather_dictionary['WSD']) + "m/s", str(date),
str(time))
# 투고
api.PostUpdate(post)
#시간 갱신
self.time = time
threading.Timer(self.refresh_time, self.weatherTask).start()
def test_0_qlib_data(self):
GetData().qlib_data_cn(QLIB_DIR)
df = D.features(D.instruments("csi300"), self.FIELDS)
self.assertListEqual(list(df.columns), self.FIELDS,
"get qlib data failed")
self.assertFalse(df.dropna().empty, "get qlib data failed")
def run_method(self):
server = Server()
server.main_server()
data = GetData()
action_method = ActionMethod()
lines = data.get_case_lines()
for i in range(1, lines):
#步骤
handle_step = data.get_handle_step(i)
#元素
element_key = data.get_element_key(i)
#操作值
handle_value = data.get_handle_value(i)
print('this is handle_value -------', handle_value)
#预期元素
expect_key = data.get_expect_element(i)
#预期步骤
expect_step = data.get_expect_handle(i)
#input() login_button
#input str
excute_method = getattr(action_method, handle_step)
if element_key != None:
excute_method(element_key, handle_value)
else:
excute_method(handle_value)
if expect_step != None:
expect_result = getattr(action_method, expect_step)
expect_result(expect_key)
def __init__(self):
self.getdata = GetData()
self.run_method = RunMethod()
self.oper_excel = OperationExcel()
self.common_util = CommonUtil()
self.oper_mysql = OperationMysql()
self.sen_mail = SendEmail()
def getData():
# Get input text
start = e1.get()
end = e2.get()
name = e3.get()
csv = e4.get()
file = e5.get()
# Validate inputs
if (not os.path.isfile(csv) and not csv.lower().endswith('.csv')):
tk.messagebox.showerror("Error", "Invalid csv!", icon="error")
return
if (not validDate(start)):
tk.messagebox.showerror("Error", "Invalid start date!", icon="error")
return
if (not validDate(end)):
tk.messagebox.showerror("Error", "Invalid end date!", icon="error")
return
if (len(name) == 0):
tk.messagebox.showerror("Error",
"You need to name your file!",
icon="error")
return
# Check if overwriting
name = name + ".xlsx"
if (not file.endswith("/")):
file = file + "/"
file = file + name
if (os.path.exists(file)):
question = name + " already exists. This will overwrite the file. Do you wish to continue?"
msg = tk.messagebox.askquestion("Overwrite", question, icon="warning")
if (msg == "no"):
return
# Get data
errors = []
try:
myData = GetData(start, end, csv, file)
errors = myData.getErrors()
except:
tk.messagebox.showerror("Error", "Something went wrong", icon="error")
return
else:
# Displays any errors getting tickers
if (len(errors) != 0):
msg = "The following ticker data could not be retrieved: " + " ".join(
errors)
tk.messagebox.showwarning('Error', msg, icon="warning")
# Closes window if finished
msg = tk.messagebox.askquestion('Finished',
'Do you want to get more data?')
if (msg == "yes"):
return
else:
root.destroy()
def fourth_choose(country, index):
"""
Lets the user choose what year he wants to get information about
:param country: (str, list)
:param index: (str)
:return: str
"""
# Define variables
user_choice = None
error = 'Sorry, there is no available data for this country'
# Define the available choices
if index == '1':
available_choices = [str(x) for x in range(1995, 2019)]
text = '\nChoose a year between 1995 and 2018: '
else:
if type(country) == str:
value = get_year(country, int(index))
available_choices = [x for x in value]
text = f'\nChoose a year. Available years: {" ".join(available_choices)} \n'
else:
if country == DEVELOPING_COUNTRIES:
available_choices = QUANDL_YEARS[int(index)]
if available_choices:
text = f'\nChoose a year. Available years: {" ".join(available_choices)} \n' \
'(The first available year has least data, the last one has the most): '
else:
# Get available years to show data
years_dict = {}
for c in country:
value = GetData().get_quandl_year(c, int(index))
for year in value:
if year in years_dict:
years_dict[year] += 1
else:
years_dict[year] = 1
available_choices = [x for x in years_dict]
if available_choices:
available_choices = sorted(available_choices,
key=lambda x: years_dict[x])
text = f'\nChoose a year. Available years: {" ".join(available_choices)} \n' \
'(The first available year has least data, the last one has the most): '
# Get user`s choice
if available_choices:
if type(country) == list:
print(
'\nNOTE: Information about some countries may not be shown.\n'
'It means that data is missing in the source.')
while user_choice not in available_choices:
user_choice = input(text)
else:
print(error)
return user_choice
def get_year(country, index):
"""
Gets information about available years to get data
for specific country
:param country: str
:param index: int
:return: list
"""
return GetData().get_quandl_year(country, index)
def setUpClass(cls) -> None:
# use default data
provider_uri = "~/.qlib/qlib_data/cn_data_simple" # target_dir
if not exists_qlib_data(provider_uri):
print(f"Qlib data is not found in {provider_uri}")
sys.path.append(str(Path(__file__).resolve().parent.parent.parent.joinpath("scripts")))
from get_data import GetData
GetData().qlib_data(name="qlib_data_simple", target_dir=provider_uri)
qlib.init(provider_uri=provider_uri, region=REG_CN)
def setUpClass(cls) -> None:
GetData().csv_data_cn(SOURCE_DIR)
TestDumpData.DUMP_DATA = DumpData(csv_path=SOURCE_DIR, qlib_dir=QLIB_DIR)
TestDumpData.STOCK_NAMES = list(map(lambda x: x.name[:-4].upper(), SOURCE_DIR.iterdir()))
provider_uri = str(QLIB_DIR.resolve())
qlib.init(
provider_uri=provider_uri,
expression_cache=None,
dataset_cache=None,
)
def test_0_qlib_data(self):
GetData().qlib_data(name="qlib_data_simple",
target_dir=QLIB_DIR,
region="cn",
interval="1d",
version="latest")
df = D.features(D.instruments("csi300"), self.FIELDS)
self.assertListEqual(list(df.columns), self.FIELDS,
"get qlib1 data failed")
self.assertFalse(df.dropna().empty, "get qlib1 data failed")
def visualize_bar(countries, index, year):
"""
Visualizes Cdata about several countries as a bar plot
:param countries: list
:param index: str
:param year: str
:return: None
"""
# define variables
if index == '1':
description = 'The Corruption Perceptions Index (CPI) is an index published annually by ' \
'Transparency International since 1995, which ranks countries \n"by their ' \
'perceived levels of public sector corruption, as determined by expert ' \
'assessments and opinion surveys" • Author: Oleksandr Dubas'
title = f'CPI in {year}'
else:
description = 'The information is provided by Quandl • https://www.quandl.com/ • Author: Oleksandr Dubas'
title = f'{QUANDL[int(index)]} in {year}'
# Make list of available data
if countries:
values = []
objects = []
for country in countries:
if index == '1':
value = cpi_info(country, year)
else:
value = GetData(country, int(index), year).export_data()
if value is not None:
objects.append(country.upper())
values.append(value)
# Create bar plot if data exists
if objects:
y_pos = np.arange(len(objects))
plt.barh(y_pos, values, align='center', alpha=1)
plt.yticks(y_pos, objects)
plt.xlabel(description)
plt.title(title)
for i, v in enumerate(values):
plt.text(v, i, str(v))
# Display bar plot
plt.show()
# Print massage if there is no available data
else:
print("There is no information about these countries this year")
else:
print("There is no information about these countries this year")
def get_news():
data = request.values.to_dict()
page = int(data.get('page', 0))
# with open('/Users/shogakusha/Workspaces/github/pythonDev/python/day10/static/data.json') as f:
# lines = f.readlines()
# print(len(lines))
# data = list(map(json.loads, lines))
try:
news = GetData()
results = news.get(page)
return jsonify({'status': 0, 'message': 'success', 'data': results})
except Exception as error:
return jsonify({'status': 500, 'message': str(error), 'data': results})
def _getPredictionData(self):
data = None
try:
logger.debug('Trying to load prediction data')
data_save = open('prediction_data.save', 'rb')
data = pickle.load(data_save)
data_save.close
except Exception:
logger.debug('Getting data from prediction database')
cnx = mysql.connector.connect(**self._config)
cursor = cnx.cursor()
# data column |Date|DTree|SVM|SGDLinear|SGDRegression|LASSORegression|
# first try to get all the tables we have in our database
allTables = GetData().getAllSymbols()
# get learn date
data = {}
for table in allTables:
data[table] = []
query = """SELECT * FROM `%s` ORDER BY Date DESC LIMIT %s;""" % (
table, self._dataCount)
cursor.execute(query)
try:
# just printing something so we know it's working
print(' ', end='\r', file=sys.stderr)
print(table, end='\r', file=sys.stderr)
# save result
result = cursor.fetchall()
for row in result:
temp = []
temp.append(row[0].strftime("%Y%m%d"))
temp.extend(row[1:])
data[table].append(temp)
except Exception:
logger.error('Unable to fectch row', exc_info=True)
# save all the data we get to local storage
data_save = open('prediction_data.save', 'wb')
pickle.dump(data, data_save)
data_save.close
logger.debug(' ', end='\r')
logger.debug('\n[DONE] get data: ' + str(len(data)))
# since we get all the data, we can close the cursor now
cursor.close()
finally:
logger.info('DONE')
return data
def main():
data_release = '20180901'
field = 'DDF'
model = '%'
save_dir = os.path.join(ROOT_DIR, 'plasticc', 'Tables', 'features',
'hdf_features_{}_{}'.format(field, data_release))
if not os.path.exists(save_dir):
os.makedirs(save_dir)
getter = GetData(data_release)
#nobjects = next(getter.get_lcs_headers(field=field, model=model, get_num_lightcurves=True, big=False))
#print("{} objects for model {} in field {}".format(nobjects, model, field))
batch_size = 1000
sort = True
redo = True
# offset = 6
# i = 2
# while offset < nobjects:
# fname = os.path.join(save_dir, 'features_{}.hdf5'.format(i))
# save_antares_features(data_release=data_release, fname=fname, field_in=field, model_in=model,
# batch_size=batch_size, offset=offset, sort=sort, redo=redo)
# offset += batch_size
# i += 1
offset = int(sys.argv[1])
offset_next = int(sys.argv[2])
print(offset, offset_next)
# Multiprocessing
i_list = np.arange(offset, offset_next)
args_list = []
file_list = os.listdir(save_dir)
for i in i_list:
if 'features_{}.hdf5'.format(i) not in file_list:
print(os.path.join(save_dir, 'features_{}.hdf5'.format(i)))
args_list.append((data_release, i, save_dir, field, model,
batch_size, sort, redo))
pool = mp.Pool(processes=20)
pool.map_async(create_all_hdf_files, args_list)
pool.close()
pool.join()
def main():
# initialize class to handle user input
user = GetInput()
# get data from file
dataset = GetData(PATH)
# scale features
dataset.feature_scale(user.get_feature_scale_technique())
# divide data into training and test data
dataset.split_train_test(user.get_test_size())
# create neural network
n = NeuralNetwork(user.get_activation())
# train neural network with training data
n.train(dataset.traindata, dataset.trainlabels)
# test the neural network with test data
n.test(dataset.testdata, dataset.testlabels)
def run_method(self):
data = GetData()
lines = data.get_case_line()
for i in range(1, lines):
handle_step = data.get_handle_step(i)
element_key = data.get_element_key(i)
handle_value = data.get_hadle_value(i)
expect_element = data.get_expect_element_key(i)
excute_method = getattr(login_action_method, handle_step)
element_value = self.get_element(element_key)
element_list = element_value.split(',')
element_value_tu = self.tuple_make(element_list[0],
element_list[1])
if handle_value != None:
excute_method(element_key, handle_value, 'android')
else:
excute_method(element_key, 'android')
time.sleep(10)
def main():
"""
Main function to interact with user
:return: (str, None)
"""
# Define variables
intro = "\nThis program is made to help in researching about \n" \
"Corruption in Developing countries in comparison with Ukraine \n"
note = "The list of all available countries is in config.py \n" \
"Some information about specific countries or specific years \n" \
"May be missing"
# Print start information
print(intro)
print('NOTE \n')
print(note)
# Create CPI table
global cpi_table
cpi_table = GetData().get_cpi_table()
# Make screenplay
def screenplay():
repeat = None
choice = '\nWould you like to know something more (y/n)? '
mode = first_choose()
sort = second_choose()
country = third_choose(int(mode))
year = fourth_choose(country, sort)
if year:
get_data(country, sort, year)
while repeat != 'y' and repeat != 'n':
repeat = input(choice)
if repeat == 'y':
screenplay()
# Start
screenplay()
def get_data(country, index, year):
"""
Gets required data about a specific country/countries in a specific year
:param index: str
:param country: (str, list)
:param year: str
:return: (str, list, None)
"""
# Define variables
error = "Sorry, there is not data about this country this year"
data = None
# Define what type of data should be displayed
# Single data
if type(country) == str:
if index == '1':
data = cpi_info(country, year)
else:
data = GetData(country, int(index), year).export_data()
# Display data
if data is None:
print('\n')
print(error)
else:
print('\n')
print('\\' * (len(str(data)) + 10))
print('\\' * 4, str(data), '\\' * 4)
print('\\' * (len(str(data)) + 10))
# Visualize data about all countries
elif country == DEVELOPING_COUNTRIES:
print('\nPlease wait up to two minutes...')
visualize_map(create_dataframe(index, year), index, year)
# Visualize data about specific countries
else:
print('\nPlease wait few seconds...')
visualize_bar(country, index, year)
def main():
data_release = '20180901'
field = 'WFD'
model = '%'
save_dir = '.' # os.path.join(ROOT_DIR, 'plasticc', 'Tables', 'features', 'hdf_features_{}_{}'.format(field, data_release))
if not os.path.exists(save_dir):
os.makedirs(save_dir)
getter = GetData(data_release)
nobjects = next(
getter.get_lcs_headers(field=field,
model=model,
get_num_lightcurves=True,
big=False))
print("{} objects for model {} in field {}".format(nobjects, model, field))
batch_size = 100
sort = True
redo = True
offset = 100
i = 0
while offset < nobjects:
fname = os.path.join(save_dir, 'features_{}.hdf5'.format(i))
save_antares_features(data_release=data_release,
fname=fname,
field_in=field,
model_in=model,
batch_size=batch_size,
offset=offset,
sort=sort,
redo=redo)
offset += batch_size
i += 1
offset = int(sys.argv[1])
offset_next = int(sys.argv[2])
print(offset, offset_next)
def main():
image_size = 28 # Pixel width and height.
pixel_depth = 255.0 # Number of levels per pixel.
data = GetData()
train_filename = data.maybe_download('notMNIST_large.tar.gz', 247336696)
test_filename = data.maybe_download('notMNIST_small.tar.gz', 8458043)
train_folders = data.maybe_extract(train_filename)
test_folders = data.maybe_extract(test_filename)
train_datasets = data.maybe_pickle(train_folders, image_size, pixel_depth,
45000)
test_datasets = data.maybe_pickle(test_folders, image_size, pixel_depth,
1800)
train_size = 200000
valid_size = 10000
test_size = 10000
valid_dataset, valid_labels, train_dataset, train_labels = data.merge_datasets(
train_datasets, train_size, image_size, valid_size)
_, _, test_dataset, test_labels = data.merge_datasets(
test_datasets, test_size, image_size)
print('Training:', train_dataset.shape, train_labels.shape)
print('Validation:', valid_dataset.shape, valid_labels.shape)
print('Testing:', test_dataset.shape, test_labels.shape)
train_dataset, train_labels = data.randomize(train_dataset, train_labels)
test_dataset, test_labels = data.randomize(test_dataset, test_labels)
valid_dataset, valid_labels = data.randomize(valid_dataset, valid_labels)
print(
'Compressed pickle size:',
data.save_data(train_dataset, train_labels, valid_dataset,
valid_labels, test_dataset, test_labels))
# Run a logistic classifier on the training data
run_sklearn_classifier(train_dataset, train_labels, test_dataset,
test_labels, valid_dataset, valid_labels)
def __init__(self):
self.run_method = RunMethod()
self.data = GetData()
self.com_util = CommonUtil()
for cell_line in ['MCF7', 'A549', 'HT29']:
for target in ['up', 'dn']:
for descriptor in ['ecfp', 'ecfp']:
mlflow.set_tracking_uri('http://193.140.108.166:5000')
mlflow.set_experiment(cell_line + '_experiments_min_100')
mlflow.start_run()
mlflow.set_tag('Author', 'RIZA')
if descriptor == 'ecfp':
if ecfp_control == 0:
mlflow.log_param('useChirality', 'True')
obj = GetData(L=L,
cell_line=cell_line,
descriptor=descriptor,
n_fold=5,
random_state=42,
random_genes=False,
useChirality=True)
ecfp_control = 1
else:
mlflow.log_param('useChirality', 'False')
obj = GetData(L=L,
cell_line=cell_line,
descriptor=descriptor,
n_fold=5,
random_state=42,
random_genes=False,
useChirality=False)
ecfp_control = 0
elif descriptor == 'jtvae':
from sklearn import tree
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
import os
import sys
# save accuracy score
results = open(os.path.basename(__file__) + '.csv', 'w')
# get_data_block_start
from get_data import GetData
getData = GetData()
fields = ['Open', 'High', 'Low', 'Close', 'Adj_Close']
accuracy = {}
features = getData.getAllFeatures()
symbols = getData.getAllSymbols()
# get_data_block_end
for symbol in symbols:
accuracy[symbol] = []
for field in range(1, 5):
labels = getData.getSymbolCLFLabels(symbol, field)
########################
# now the real MA work #
########################
exp_path = str(Path(os.getcwd()).resolve() / exp_folder_name)
exp_manager = {
"class": "MLflowExpManager",
"module_path": "qlib1.workflow.expm",
"kwargs": {
"uri": "file:" + exp_path,
"default_exp_name": "Experiment",
},
}
if not exists_qlib_data(provider_uri):
print(f"Qlib data is not found in {provider_uri}")
sys.path.append(
str(Path(__file__).resolve().parent.parent.joinpath("scripts")))
from get_data import GetData
GetData().qlib_data(target_dir=provider_uri, region=REG_CN)
qlib1.init(provider_uri=provider_uri, region=REG_CN, exp_manager=exp_manager)
# decorator to check the arguments
def only_allow_defined_args(function_to_decorate):
@functools.wraps(function_to_decorate)
def _return_wrapped(*args, **kwargs):
"""Internal wrapper function."""
argspec = inspect.getfullargspec(function_to_decorate)
valid_names = set(argspec.args + argspec.kwonlyargs)
if "self" in valid_names:
valid_names.remove("self")
for arg_name in kwargs:
if arg_name not in valid_names:
raise ValueError("Unknown argument seen '%s', expected: [%s]" %
def endo():
command = input()
command = command.lower()
if (command[:14]) == 'create project':
website_name = command[17:]
upload.repository_name = website_name
data = GetData(web_data)
if hasattr(data, 'is_loaded') and data.is_loaded == True:
installer.run_installer(data.json_data, website_name)
if installer.success == True:
print(Fore.YELLOW + "Type 'start' to deploy the server")
print(Style.RESET_ALL)
else:
pass
else:
data.retrieve_failed()
elif (command[:10]) == 'create app':
app_name = command[13:]
upload.repository_name = app_name
data = GetData(app_data)
if hasattr(data, 'is_loaded') and data.is_loaded == True:
installer.run_installer(data.json_data, app_name)
if installer.success == True:
print(Fore.YELLOW + "Type 'run' to start up the app")
print(Style.RESET_ALL)
else:
pass
else:
data.retrieve_failed()
elif command == 'info':
info = Info()
elif command == 'commands':
command = Commands()
elif command == 'start':
installer.start_server()
elif command == 'run':
installer.start_app()
elif (command[:4]) == 'open':
folder = command[7:]
folder = '"'+folder+'"'
try:
open = OpenFile(folder)
repo_name = os.path.basename(os.path.normpath(folder))
repo_name = repo_name.replace(" ", "")
repo_name = repo_name.replace('"', "")
upload.repository_name = repo_name
except:
print(Fore.RED + "ERROR_10: Invalid directory be sure to type 'open - directory name' and please try again")
print(Style.RESET_ALL)
elif command == 'upload':
upload.start_upload()
elif (command[:3]) == 'new':
new_file = command[6:]
create_file = CreateFile(new_file)
elif command == 'quit':
print("Endo is shutting down...")
sleep(1)
sys.exit()
elif command == 'endo':
webbrowser.open('https://hugopage17.github.io/endo/')
else:
print(Fore.RED + "ERROR_05: "+command+" is not a valid command, type 'commands' to get a list of all valid commands")
print(Style.RESET_ALL)
def test_1_csv_data(self):
GetData().csv_data_cn(SOURCE_DIR)
stock_name = set(
map(lambda x: x.name[:-4].upper(), SOURCE_DIR.glob("*.csv")))
self.assertEqual(len(stock_name), 96, "get csv data failed")
def test_getAllFeatures1(self):
getData = GetData()
features = getData.getAllFeatures()
self.assertIsNotNone(features)
