def __init__(self, WINDOW_LENGTH, EPOCHS, BATCH_SIZE, symbol, train=True):
"""
Constants used for things like
- WINDOW_LENGTH : how many elements of data are in a state
- EPOCHS : number of episodes the model is trained for
- BATCH_SIZE : how many cycles we go until we fit the agent's model
"""
self.WINDOW_LENGTH = WINDOW_LENGTH
self.EPOCHS = EPOCHS
self.BATCH_SIZE = BATCH_SIZE
self.AMPLIFIER = 1000
"""
Variables for the environment data (prices of the stock)
Train variable to determine if were being run in training mode or trading mode
"""
self.stock = GetData(symbol, train)
self.data = self.stock.format_data()
self.symbol = symbol
self.data_len = len(self.data)
self.train = train
self.p = Portfolio('lucas')
"""
Parameters that are subject to reset after every training episode
"""
self.buy_count = 0
self.sell_count = 0
self.active_positions = 0
self.history = []
def __init__(self):
self.getdata = GetData()
self.run_method = RunMethod()
self.oper_excel = OperationExcel()
self.common_util = CommonUtil()
self.oper_mysql = OperationMysql()
self.sen_mail = SendEmail()
class DependentData:
def __init__(self, case_id):
self.opera_excel = OperationExcel()
self.case_id = case_id
self.data = GetData()
# 通过case_id获取case_id的整行数据
def get_case_line_data(self, case_id):
rows_data = self.opera_excel.get_rows_data(case_id)
return rows_data
# 执行依赖测试，获取结果
def run_dependent(self):
run_method = RunMethod()
row_num = self.opera_excel.get_row_num(self.case_id)
url = self.data.get_url(row_num)
data = self.data.get_data_for_json(row_num)
header = self.data.get_is_header(row_num)
method = self.data.get_request_method(row_num)
res = run_method.run_main(method, url, data, header)
return json.loads(res)
# 根据依赖的key去获取执行依赖测试case的响应，然后返回
def get_data_for_key(self, row):
depend_data = self.data.get_depend_key(row)
response_data = self.run_dependent()
json_exe = parse(depend_data)
madle = json_exe.find(response_data)
return [math.value for math in madle][0]
def getData():
# Get input text
start = e1.get()
end = e2.get()
name = e3.get()
csv = e4.get()
file = e5.get()
# Validate inputs
if (not os.path.isfile(csv) and not csv.lower().endswith('.csv')):
tk.messagebox.showerror("Error", "Invalid csv!", icon="error")
return
if (not validDate(start)):
tk.messagebox.showerror("Error", "Invalid start date!", icon="error")
return
if (not validDate(end)):
tk.messagebox.showerror("Error", "Invalid end date!", icon="error")
return
if (len(name) == 0):
tk.messagebox.showerror("Error",
"You need to name your file!",
icon="error")
return
# Check if overwriting
name = name + ".xlsx"
if (not file.endswith("/")):
file = file + "/"
file = file + name
if (os.path.exists(file)):
question = name + " already exists. This will overwrite the file. Do you wish to continue?"
msg = tk.messagebox.askquestion("Overwrite", question, icon="warning")
if (msg == "no"):
return
# Get data
errors = []
try:
myData = GetData(start, end, csv, file)
errors = myData.getErrors()
except:
tk.messagebox.showerror("Error", "Something went wrong", icon="error")
return
else:
# Displays any errors getting tickers
if (len(errors) != 0):
msg = "The following ticker data could not be retrieved: " + " ".join(
errors)
tk.messagebox.showwarning('Error', msg, icon="warning")
# Closes window if finished
msg = tk.messagebox.askquestion('Finished',
'Do you want to get more data?')
if (msg == "yes"):
return
else:
root.destroy()
class RunTest:
def __init__(self):
self.run_method = RunMethod()
self.data = GetData()
self.com = CommonUtil()
#程序执行的入口
def go_on_run(self):
res = None
rows_count = int(self.data.get_case_lines())
for i in range(1, rows_count):
is_run = self.data.get_is_run(i)
if is_run:
url = self.data.get_request_url(i)
method = self.data.get_request_way(i)
request_data = self.data.get_data_for_json(i)
expect = self.data.get_expect_data(i)
header = self.data.is_header(i)
depend_case = self.data.is_depend(i)
if depend_case != None:
self.depend_data = DependdentData(depend_case)
#响应数据
depend_response_data = self.depend_data.get_data_for_key(i)
#获取依赖的key
depend_key = self.data.get_depend_field(i)
request_data[depend_key] = depend_response_data
res = self.run_method.run_main(method, url, request_data,
header)
#print res
if self.com.is_contain(expect, res):
# self.data.write_result(i,'pass')
print 'pass'
else:
# self.data.write_result(i, 'fail')
print 'fail'
def get_news():
data = request.values.to_dict()
page = int(data.get('page', 0))
# with open('/Users/shogakusha/Workspaces/github/pythonDev/python/day10/static/data.json') as f:
# lines = f.readlines()
# print(len(lines))
# data = list(map(json.loads, lines))
try:
news = GetData()
results = news.get(page)
return jsonify({'status': 0, 'message': 'success', 'data': results})
except Exception as error:
return jsonify({'status': 500, 'message': str(error), 'data': results})
def get_team_power(group_members):
# group_members: list of user id
dbcon, dbcur = get_db.get_db()
dbcur.execute('select * from users')
user_list = dbcur.fetchall()
dbcur.close()
dbcon.close()
gd = GetData(user_list)
user_skils = []
for member in group_members:
user_skils.append(gd.get_user_skill_type(member))
return user_skils
def run_method(self):
server = Server()
server.main_server()
data = GetData()
action_method = ActionMethod()
lines = data.get_case_lines()
for i in range(1, lines):
#步骤
handle_step = data.get_handle_step(i)
#元素
element_key = data.get_element_key(i)
#操作值
handle_value = data.get_handle_value(i)
print('this is handle_value -------', handle_value)
#预期元素
expect_key = data.get_expect_element(i)
#预期步骤
expect_step = data.get_expect_handle(i)
#input() login_button
#input str
excute_method = getattr(action_method, handle_step)
if element_key != None:
excute_method(element_key, handle_value)
else:
excute_method(handle_value)
if expect_step != None:
expect_result = getattr(action_method, expect_step)
expect_result(expect_key)
def setup_three(self):
# lets use a position in a bananarchy file where we have already eaten at least one banana
# and using a trial later in the file.
print('test three, bananarchy file')
self.time_stamp = 1370018957436
self.trial_num = 4
self.avatar_pos = (3.8853, 4.31563, 1)
self.avatar_head = -1546.99902344
self.gone_bananas = ['banana9', 'banana6', 'banana3']
self.banana_pos = {'banana00': (-4.63066, -1.58329, 1),
'banana01': (3.01044, 0.545034, 1),
'banana02': (0.955586, 1.0967, 1),
'banana03': (4.38659, 4.133, 1),
'banana04': (4.15384, 2.57773, 1),
'banana05': (-4.58963, -1.25701, 1),
'banana06': (2.01608, 4.68744, 1),
'banana07': (3.0936, 3.26741, 1),
'banana08': (4.42608, -4.20852, 1),
'banana09': (0.980457, 4.95074, 1),
}
# initiate test_data
self.test_data = GetData()
# now can change some parameters
self.test_data.data_filename = '../play_data/giz_bananarchy.txt'
self.test_data.time_stamp = self.time_stamp
self.test_data.start_time = 0
# and get data
self.test_data.get_data_from_file()
def setup_two(self):
# lets use a position in a gobananas file where we use a trial later in the file.
print('test_two, gobananas, trial 2')
self.time_stamp = 1389990322667
self.trial_num = 2
self.avatar_pos = (-5.53812, 1.49036, 1)
self.avatar_head = -743.258793833
self.gone_bananas = ['banana03', 'banana05', 'banana08']
self.banana_pos = {'banana00': (-0.853292, 4.7575, 1),
'banana01': (-4.34033, 3.84758, 1),
'banana02': (3.75634, -3.56932, 1),
'banana03': (-5.61975, 1.61125, 1),
'banana04': (2.52662, 6.00765, 1),
'banana05': (-3.67816, -5.89663, 1),
'banana06': (4.81536, 3.20421, 1),
'banana07': (2.61812, 4.21512, 1),
'banana08': (-3.33551, -6.21121, 1),
'banana09': (-0.293181, -2.1774, 1),
}
# initiate test_data
self.test_data = GetData()
# now can change some parameters
self.test_data.data_filename = '../play_data/giz_short.txt'
self.test_data.time_stamp = self.time_stamp
self.test_data.start_time = 1389990322200
# and get data
self.test_data.get_data_from_file()
def setup_one(self):
# lets use a position in a gobananas file where we have already eaten at least one banana
print('gobananas, first trial')
self.time_stamp = 1389990270636
self.trial_num = 1
self.avatar_pos = (6.39612, 2.1649, 1)
self.avatar_head = -94.6614046942
self.gone_bananas = ['banana00', 'banana09']
self.banana_pos = {'banana00': (6.5513, 2.14656, 1),
'banana01': (-0.558629, -6.15313, 1),
'banana02': (-5.30907, 0.710984, 1),
'banana03': (6.9584, -3.37868, 1),
'banana04': (-1.05225, -1.14995, 1),
'banana05': (0.0321884, -3.29353, 1),
'banana06': (4.72547, -1.949, 1),
'banana07': (-6.63089, -2.5445, 1),
'banana08': (-2.79261, -6.84966, 1),
'banana09': (-0.461612, 4.1383, 1),
}
# initiate test_data
self.test_data = GetData()
# now can change some parameters
self.test_data.data_filename = '../play_data/giz_short.txt'
self.test_data.time_stamp = self.time_stamp
self.test_data.start_time = 0
# and get data
self.test_data.get_data_from_file()
def setup_four(self):
# lets use a position in a bananarchy file where we have already eaten at least one banana
# and using a trial later in the file.
print('test four, bananarchy file, first trial, full')
self.time_stamp = 1370018859357
self.trial_num = 1
self.avatar_pos = (-1.01359, 2.54856, 1)
self.avatar_head = -387.116973877
self.gone_bananas = ['banana0', 'banana1', 'banana2', 'banana3', 'banana4', 'banana5', 'banana6', 'banana7', 'banana8', 'banana9']
self.banana_pos = {'banana00': (-1.85695, -3.5224, 1),
'banana01': (-0.706922, 2.96965, 1),
'banana02': (-0.121917, 0.0990294, 1),
'banana03': (-3.49998, -1.30716, 1),
'banana04': (-2.88193, 4.38051, 1),
'banana05': (-1.27652, 4.36617, 1),
'banana06': (0.268909, 3.23278, 1),
'banana07': (-4.04485, -2.37258, 1),
'banana08': (0.350244, -2.79667, 1),
'banana09': (-1.74603, 4.66188, 1),
}
# initiate test_data
self.test_data = GetData()
# now can change some parameters
self.test_data.data_filename = '../play_data/giz_bananarchy.txt'
self.test_data.time_stamp = self.time_stamp
self.test_data.start_time = 1370018819737
# and get data
self.test_data.get_data_from_file()
def test_0_qlib_data(self):
GetData().qlib_data_cn(QLIB_DIR)
df = D.features(D.instruments("csi300"), self.FIELDS)
self.assertListEqual(list(df.columns), self.FIELDS,
"get qlib data failed")
self.assertFalse(df.dropna().empty, "get qlib data failed")
def create_dataframe(index, year):
"""
Creates pandas dataframe that will be used in visualization
:param index: str
:param year: str
:return: dataframe
"""
# Define variables
data = {'Country Name': [], 'Country Code': [], year: []}
# Create datafreme
for country in ISO:
if index == '1':
value = cpi_info(country, year)
else:
value = GetData(country, int(index), year).export_data()
if value is None:
value = np.nan
data['Country Name'].append(country)
data['Country Code'].append(ISO[country])
data[year].append(value)
df = pd.DataFrame(data, columns=['Country Name', 'Country Code', year])
return df
def weatherTask(self):
data = GetData(top, mdl, leaf, service_key)
# 기상청으로부터 지역의 X, Y좌표를 획득
data.setRegion()
# 기상청에서 요구하는 형태의 날짜와 시간 문자열 설정
date, time = self.timeString()
data.setTime(date, time)
# 정보수집
weather = data.getData()
# 딕셔너리에 정보 저장
for i in weather['response']['body']['items']['item']:
self.weather_dictionary[i['category']] = i['obsrValue']
if time != self.time:
# 템플릿에 정보 대입
post = self.template.format(
str(settings["region"]),
str(self.weather_dictionary['T1H']) + "°C",
pty[self.weather_dictionary['PTY']],
str(self.weather_dictionary['WSD']) + "m/s", str(date),
str(time))
# 투고
api.PostUpdate(post)
#시간 갱신
self.time = time
threading.Timer(self.refresh_time, self.weatherTask).start()
def fourth_choose(country, index):
"""
Lets the user choose what year he wants to get information about
:param country: (str, list)
:param index: (str)
:return: str
"""
# Define variables
user_choice = None
error = 'Sorry, there is no available data for this country'
# Define the available choices
if index == '1':
available_choices = [str(x) for x in range(1995, 2019)]
text = '\nChoose a year between 1995 and 2018: '
else:
if type(country) == str:
value = get_year(country, int(index))
available_choices = [x for x in value]
text = f'\nChoose a year. Available years: {" ".join(available_choices)} \n'
else:
if country == DEVELOPING_COUNTRIES:
available_choices = QUANDL_YEARS[int(index)]
if available_choices:
text = f'\nChoose a year. Available years: {" ".join(available_choices)} \n' \
'(The first available year has least data, the last one has the most): '
else:
# Get available years to show data
years_dict = {}
for c in country:
value = GetData().get_quandl_year(c, int(index))
for year in value:
if year in years_dict:
years_dict[year] += 1
else:
years_dict[year] = 1
available_choices = [x for x in years_dict]
if available_choices:
available_choices = sorted(available_choices,
key=lambda x: years_dict[x])
text = f'\nChoose a year. Available years: {" ".join(available_choices)} \n' \
'(The first available year has least data, the last one has the most): '
# Get user`s choice
if available_choices:
if type(country) == list:
print(
'\nNOTE: Information about some countries may not be shown.\n'
'It means that data is missing in the source.')
while user_choice not in available_choices:
user_choice = input(text)
else:
print(error)
return user_choice
def get_year(country, index):
"""
Gets information about available years to get data
for specific country
:param country: str
:param index: int
:return: list
"""
return GetData().get_quandl_year(country, index)
def setUpClass(cls) -> None:
GetData().csv_data_cn(SOURCE_DIR)
TestDumpData.DUMP_DATA = DumpData(csv_path=SOURCE_DIR, qlib_dir=QLIB_DIR)
TestDumpData.STOCK_NAMES = list(map(lambda x: x.name[:-4].upper(), SOURCE_DIR.iterdir()))
provider_uri = str(QLIB_DIR.resolve())
qlib.init(
provider_uri=provider_uri,
expression_cache=None,
dataset_cache=None,
)
def setUpClass(cls) -> None:
# use default data
provider_uri = "~/.qlib/qlib_data/cn_data_simple" # target_dir
if not exists_qlib_data(provider_uri):
print(f"Qlib data is not found in {provider_uri}")
sys.path.append(str(Path(__file__).resolve().parent.parent.parent.joinpath("scripts")))
from get_data import GetData
GetData().qlib_data(name="qlib_data_simple", target_dir=provider_uri)
qlib.init(provider_uri=provider_uri, region=REG_CN)
def test_0_qlib_data(self):
GetData().qlib_data(name="qlib_data_simple",
target_dir=QLIB_DIR,
region="cn",
interval="1d",
version="latest")
df = D.features(D.instruments("csi300"), self.FIELDS)
self.assertListEqual(list(df.columns), self.FIELDS,
"get qlib1 data failed")
self.assertFalse(df.dropna().empty, "get qlib1 data failed")
def visualize_bar(countries, index, year):
"""
Visualizes Cdata about several countries as a bar plot
:param countries: list
:param index: str
:param year: str
:return: None
"""
# define variables
if index == '1':
description = 'The Corruption Perceptions Index (CPI) is an index published annually by ' \
'Transparency International since 1995, which ranks countries \n"by their ' \
'perceived levels of public sector corruption, as determined by expert ' \
'assessments and opinion surveys" • Author: Oleksandr Dubas'
title = f'CPI in {year}'
else:
description = 'The information is provided by Quandl • https://www.quandl.com/ • Author: Oleksandr Dubas'
title = f'{QUANDL[int(index)]} in {year}'
# Make list of available data
if countries:
values = []
objects = []
for country in countries:
if index == '1':
value = cpi_info(country, year)
else:
value = GetData(country, int(index), year).export_data()
if value is not None:
objects.append(country.upper())
values.append(value)
# Create bar plot if data exists
if objects:
y_pos = np.arange(len(objects))
plt.barh(y_pos, values, align='center', alpha=1)
plt.yticks(y_pos, objects)
plt.xlabel(description)
plt.title(title)
for i, v in enumerate(values):
plt.text(v, i, str(v))
# Display bar plot
plt.show()
# Print massage if there is no available data
else:
print("There is no information about these countries this year")
else:
print("There is no information about these countries this year")
class DependData(object):
'''有依赖的case处理模块，case2依赖case1点数据
'''
def __init__(self, case_id):
self.case_id = case_id
self.opera_excel = OperationExcel()
self.get_data = GetData()
# 获取目标行的数据
def get_case_line_data(self):
return self.opera_excel.get_rows_data(self.case_id)
# 执行case1用例返回结果
def run_denpend(self):
run_method = RunMain()
# 执行请求
row_num = self.opera_excel.get_rows_num(self.case_id) + 1
# print(row_num)
request_json = self.get_data.get_data_json(row_num)
url = self.get_data.get_request_url(row_num)
# print(url)
request_method = self.get_data.get_request_method(row_num)
is_header = self.get_data.get_is_header(row_num)
res = run_method.run_main(url, request_method, request_json, is_header)
# 返回请求结果
return res
# 在case1用例返回的数据中，拿到case2所需要的字段的数据，返回
def get_data_key(self, row):
depend_data_key = self.get_data.get_depend_key(row)
# print(depend_data_key)
# 将str数据类型转换为json数据类型（dict字典）
response_data = json.loads(self.run_denpend())
# print(type(response_data))
# 需要在json中匹配的字段规则
jsonpath_expr = parse(depend_data_key)
# 在json数据中查找规则字段 返回一个list
male = jsonpath_expr.find(response_data)
# 返回需要的对应字段中的内容
return [match.value for match in male][0]
def main():
data_release = '20180901'
field = 'WFD'
model = '%'
save_dir = '.' # os.path.join(ROOT_DIR, 'plasticc', 'Tables', 'features', 'hdf_features_{}_{}'.format(field, data_release))
if not os.path.exists(save_dir):
os.makedirs(save_dir)
getter = GetData(data_release)
nobjects = next(
getter.get_lcs_headers(field=field,
model=model,
get_num_lightcurves=True,
big=False))
print("{} objects for model {} in field {}".format(nobjects, model, field))
batch_size = 100
sort = True
redo = True
offset = 100
i = 0
while offset < nobjects:
fname = os.path.join(save_dir, 'features_{}.hdf5'.format(i))
save_antares_features(data_release=data_release,
fname=fname,
field_in=field,
model_in=model,
batch_size=batch_size,
offset=offset,
sort=sort,
redo=redo)
offset += batch_size
i += 1
offset = int(sys.argv[1])
offset_next = int(sys.argv[2])
print(offset, offset_next)
def _getPredictionData(self):
data = None
try:
logger.debug('Trying to load prediction data')
data_save = open('prediction_data.save', 'rb')
data = pickle.load(data_save)
data_save.close
except Exception:
logger.debug('Getting data from prediction database')
cnx = mysql.connector.connect(**self._config)
cursor = cnx.cursor()
# data column |Date|DTree|SVM|SGDLinear|SGDRegression|LASSORegression|
# first try to get all the tables we have in our database
allTables = GetData().getAllSymbols()
# get learn date
data = {}
for table in allTables:
data[table] = []
query = """SELECT * FROM `%s` ORDER BY Date DESC LIMIT %s;""" % (
table, self._dataCount)
cursor.execute(query)
try:
# just printing something so we know it's working
print(' ', end='\r', file=sys.stderr)
print(table, end='\r', file=sys.stderr)
# save result
result = cursor.fetchall()
for row in result:
temp = []
temp.append(row[0].strftime("%Y%m%d"))
temp.extend(row[1:])
data[table].append(temp)
except Exception:
logger.error('Unable to fectch row', exc_info=True)
# save all the data we get to local storage
data_save = open('prediction_data.save', 'wb')
pickle.dump(data, data_save)
data_save.close
logger.debug(' ', end='\r')
logger.debug('\n[DONE] get data: ' + str(len(data)))
# since we get all the data, we can close the cursor now
cursor.close()
finally:
logger.info('DONE')
return data
def get_ranking(user_list, user_id):
"""
ユーザidだけ渡して，おすすめユーザのidランキングを返す
"""
gd = GetData(user_list)
cl = Calc()
rank1 = cl.make_skill_level_ranking(gd.get_user_skill_level(user_id), gd.get_others_skill_level(user_id))
rank2 = cl.make_skill_type_ranking(gd.get_user_skill_type(user_id), gd.get_others_skill_type(user_id))
rank3 = cl.make_design_favorite_ranking(gd.get_user_design_favor(user_id), gd.get_others_design_favor(user_id))
rank4 = cl.make_character_ranking(gd.get_user_character(user_id), gd.get_others_character(user_id))
final_ranking = cl.final_ranking(rank1, rank2, rank3, rank4)
return final_ranking
class Dependent:
def __init__(self,case_id):
self.case_id = case_id
self.oper_excel = OperationExcel()
self.data = GetData()
self.run = RunMethod()
def dependent_run(self): #运行依赖的用例
case_row = self.oper_excel.get_case_row(self.case_id)
method = self.data.get_request_way_value(case_row)
url = self.data.get_url_value(case_row)
data = self.data.get_data_for_json(case_row)
header = self.data.get_is_header_value(case_row)
res = self.run.run_method(method,url,data,header)
return res
def dependent_key_values(self,row): #获取需要依赖键的值
response_data = self.dependent_run() #运行依赖用例，获取其返回值
requests_data = self.data.get_response_dependent_value(row) #获取返回值里的某个键的值
response_data = json.loads(response_data) #将json数据转换成python可识别的对象
json_exe = parse(requests_data) #定义一个规则，以json格式去格式化接口返回的数据
mable = json_exe.find(response_data) #按照规则来查找整个结果集
return [math.value for math in mable] #官方写法，从结果集查找某部分数据
class main():
def __init__(self):
self.run_method = RunMethod()
self.data = GetData()
self.json = UseJson()
def run(self):
res = header = method = None
pass_count = []
fail_count = []
rows_count = self.data.get_case_lines()
for i in range(1, rows_count):
is_run = self.data.get_is_run(i)
if is_run:
url = self.data.get_request_url(i)
method = self.data.get_request_method(i)
request_data = self.data.get_data_for_json(i)
# expect = self.data.get_expcet_data_for_mysql(i)
header = self.data.is_header(i)
request_name = self.data.get_request_name(i)
# depend_case = self.data.is_depend(i)
# if depend_case != None:
# self.depend_data = DependdentData(depend_case)
# #获取的依赖响应数据
# depend_response_data = self.depend_data.get_data_for_key(i)
# #获取依赖的key
# depend_key = self.data.get_depend_field(i)
#request_data[depend_key] = depend_response_data
if header == 'yes':
token = self.json.get_data("token")
head = {"token": token}
res = self.run_method.main(method)(url, request_data, head)
else:
res = self.run_method.main(method)(url, request_data)
if res['error_code'] & res['error_code'] == 0:
print("测试：" + request_name + ",", "结果：success")
#存token到文件
if 'token' in res['data']:
self.json.write_data("token", res['data']['token'])
elif res['error_code'] & res['error_code'] != 0:
print("测试：" + request_name + ",", "结果：fail" + ",",
"原因：" + res["msg"])
else:
print("测试：" + request_name + ",", "结果：fail" + ",",
"原因：" + res)
def create_newcastle_map(self, date):
"""
Build the map
"""
date = dt.datetime.strptime(date, "%Y-%m-%d")
start = dt.datetime(date.year, date.month, date.day, 0, 0)
end = dt.datetime(date.year, date.month, date.day, 23, 59)
px.set_mapbox_access_token(self.access_token)
self.nc_dframe = GetData.newcastle(start, end)
figure = px.scatter_mapbox(self.nc_dframe , lat="latitude", lon="longitude", color="Value",
color_continuous_scale=px.colors.cyclical.IceFire, size_max=15, zoom=10)
return figure
def main():
data_release = '20180901'
field = 'DDF'
model = '%'
save_dir = os.path.join(ROOT_DIR, 'plasticc', 'Tables', 'features',
'hdf_features_{}_{}'.format(field, data_release))
if not os.path.exists(save_dir):
os.makedirs(save_dir)
getter = GetData(data_release)
#nobjects = next(getter.get_lcs_headers(field=field, model=model, get_num_lightcurves=True, big=False))
#print("{} objects for model {} in field {}".format(nobjects, model, field))
batch_size = 1000
sort = True
redo = True
# offset = 6
# i = 2
# while offset < nobjects:
# fname = os.path.join(save_dir, 'features_{}.hdf5'.format(i))
# save_antares_features(data_release=data_release, fname=fname, field_in=field, model_in=model,
# batch_size=batch_size, offset=offset, sort=sort, redo=redo)
# offset += batch_size
# i += 1
offset = int(sys.argv[1])
offset_next = int(sys.argv[2])
print(offset, offset_next)
# Multiprocessing
i_list = np.arange(offset, offset_next)
args_list = []
file_list = os.listdir(save_dir)
for i in i_list:
if 'features_{}.hdf5'.format(i) not in file_list:
print(os.path.join(save_dir, 'features_{}.hdf5'.format(i)))
args_list.append((data_release, i, save_dir, field, model,
batch_size, sort, redo))
pool = mp.Pool(processes=20)
pool.map_async(create_all_hdf_files, args_list)
pool.close()
pool.join()
def run_method(self):
data = GetData()
lines = data.get_case_line()
for i in range(1, lines):
handle_step = data.get_handle_step(i)
element_key = data.get_element_key(i)
handle_value = data.get_hadle_value(i)
expect_element = data.get_expect_element_key(i)
excute_method = getattr(login_action_method, handle_step)
element_value = self.get_element(element_key)
element_list = element_value.split(',')
element_value_tu = self.tuple_make(element_list[0],
element_list[1])
if handle_value != None:
excute_method(element_key, handle_value, 'android')
else:
excute_method(element_key, 'android')
time.sleep(10)
def main():
"""
Main function to interact with user
:return: (str, None)
"""
# Define variables
intro = "\nThis program is made to help in researching about \n" \
"Corruption in Developing countries in comparison with Ukraine \n"
note = "The list of all available countries is in config.py \n" \
"Some information about specific countries or specific years \n" \
"May be missing"
# Print start information
print(intro)
print('NOTE \n')
print(note)
# Create CPI table
global cpi_table
cpi_table = GetData().get_cpi_table()
# Make screenplay
def screenplay():
repeat = None
choice = '\nWould you like to know something more (y/n)? '
mode = first_choose()
sort = second_choose()
country = third_choose(int(mode))
year = fourth_choose(country, sort)
if year:
get_data(country, sort, year)
while repeat != 'y' and repeat != 'n':
repeat = input(choice)
if repeat == 'y':
screenplay()
# Start
screenplay()
def get_data(country, index, year):
"""
Gets required data about a specific country/countries in a specific year
:param index: str
:param country: (str, list)
:param year: str
:return: (str, list, None)
"""
# Define variables
error = "Sorry, there is not data about this country this year"
data = None
# Define what type of data should be displayed
# Single data
if type(country) == str:
if index == '1':
data = cpi_info(country, year)
else:
data = GetData(country, int(index), year).export_data()
# Display data
if data is None:
print('\n')
print(error)
else:
print('\n')
print('\\' * (len(str(data)) + 10))
print('\\' * 4, str(data), '\\' * 4)
print('\\' * (len(str(data)) + 10))
# Visualize data about all countries
elif country == DEVELOPING_COUNTRIES:
print('\nPlease wait up to two minutes...')
visualize_map(create_dataframe(index, year), index, year)
# Visualize data about specific countries
else:
print('\nPlease wait few seconds...')
visualize_bar(country, index, year)
def main():
# initialize class to handle user input
user = GetInput()
# get data from file
dataset = GetData(PATH)
# scale features
dataset.feature_scale(user.get_feature_scale_technique())
# divide data into training and test data
dataset.split_train_test(user.get_test_size())
# create neural network
n = NeuralNetwork(user.get_activation())
# train neural network with training data
n.train(dataset.traindata, dataset.trainlabels)
# test the neural network with test data
n.test(dataset.testdata, dataset.testlabels)
class RunTest:
def __init__(self):
self.run_method = RunMethod()
self.data = GetData()
def go_on_run(self):
rows_count = self.data.get_case_lines()
for i in range(1, rows_count):
url = self.data.get_request_url(i)
method = self.data.get_request_method(i)
is_run = self.data.get_is_run(i)
data = self.data.get_data_for_json(i)
header = self.data.is_header(i)
if is_run:
res = self.run_method.run_main(method, url, data, header)
return res
class DependentData:
def __init__(self, case_id):
self.case_id = case_id
self.opera_excel = OperationExcel
self.data = GetData()
# 执行依赖测试获取结果
def run_dependent(self):
run_method = RunMethod()
row_num1 = self.opera_excel()
row_num = row_num1.get_rows_num(self.case_id)
request_data1 = self.data.get_data_for_json(row_num)
request_data = json.dumps(request_data1)
header1 = self.data.is_header(row_num)
header = eval(header1)
method = self.data.get_request_method(row_num)
url = self.data.get_request_url(row_num)
token = self.data.get_token(row_num)
if token == 'yes':
# 获取当前文件路径
current_path = os.path.abspath(__file__)
# 获取当前文件的父目录
father_path = os.path.dirname(current_path)
path = os.path.join(father_path, "wenjian", "cookie.json")
op_json = OperationJson(path)
token = op_json.get_data("access_token")
tokens = {'Authorization': "Bearer " + token}
# 将header与token合并
header = dict(header, **tokens)
res = run_method.run_main(method, url, request_data, header)
return json.loads(res)
# 根据依赖的key去获取执行依赖测试case的响应，然后返回
def get_data_for_key(self, row):
depend_data = self.data.get_depend_key(row)
response_data = self.run_dependent()
json_exe = parse(depend_data)
madle = json_exe.find(response_data)
return [math.value for math in madle][0]
from avatar_movie import AvatarWorld
import os.path
config_file = 'GR_BR_15_06_18_11_55'
# config_file = 'GR_BR_15_06_18_11_55'
save_data = True # normally saves if there is no data file yet, but this ensures writing over any existing data file.
config = {}
execfile('configs/' + config_file + '.py', config)
distance_goal = config.get('distance_goal', False)
config['data_filename'] = '../raw_data/' + config_file + '/log.txt'
#execfile('config.py', config)
print config['movie_data_filename']
if os.path.isfile(config['movie_data_filename']) and not save_data:
save_data = False
if save_data:
newData = GetData(config)
newData.get_data_from_file()
newData.get_data_for_end_time(newData.end_time)
if config['lfp_data_file']:
for i, filename in enumerate(newData.lfp_data_file):
newData.lfp_data.append(newData.get_lfp_data(filename))
newData.pickle_info()
# data_file_name, record, use_eye_data=False, use_lfp_data=False):
if config['watch_movie'] and config['lfp_data_file']:
BW = BananaWorld(config['movie_data_filename'], config['save_movie'], config['use_eye_data'],
config['lfp_data_file'])
BW.base.run()
elif config['watch_movie']:
BW = BananaWorld(config['movie_data_filename'], config['save_movie'], config['use_eye_data'])
BW.base.run()
def __init__(self):
self.run_method = RunMethod()
self.data = GetData()
self.com_util = CommonUtil()
class TestGetData(unittest.TestCase):
i = 0
@classmethod
def setUpClass(cls):
if cls.i == 0:
print('class one')
cls.n_test = cls.setup_one
elif cls.i == 1:
print('class two')
cls.n_test = cls.setup_two
elif cls.i == 2:
print('class three')
cls.n_test = cls.setup_three
elif cls.i == 3:
print('class four')
cls.n_test = cls.setup_four
#cls.n_test = cls.setup_four
cls.i += 1
def setUp(self):
print('test', self.n_test)
self.n_test()
def setup_one(self):
# lets use a position in a gobananas file where we have already eaten at least one banana
print('gobananas, first trial')
self.time_stamp = 1389990270636
self.trial_num = 1
self.avatar_pos = (6.39612, 2.1649, 1)
self.avatar_head = -94.6614046942
self.gone_bananas = ['banana00', 'banana09']
self.banana_pos = {'banana00': (6.5513, 2.14656, 1),
'banana01': (-0.558629, -6.15313, 1),
'banana02': (-5.30907, 0.710984, 1),
'banana03': (6.9584, -3.37868, 1),
'banana04': (-1.05225, -1.14995, 1),
'banana05': (0.0321884, -3.29353, 1),
'banana06': (4.72547, -1.949, 1),
'banana07': (-6.63089, -2.5445, 1),
'banana08': (-2.79261, -6.84966, 1),
'banana09': (-0.461612, 4.1383, 1),
}
# initiate test_data
self.test_data = GetData()
# now can change some parameters
self.test_data.data_filename = '../play_data/giz_short.txt'
self.test_data.time_stamp = self.time_stamp
self.test_data.start_time = 0
# and get data
self.test_data.get_data_from_file()
def setup_two(self):
# lets use a position in a gobananas file where we use a trial later in the file.
print('test_two, gobananas, trial 2')
self.time_stamp = 1389990322667
self.trial_num = 2
self.avatar_pos = (-5.53812, 1.49036, 1)
self.avatar_head = -743.258793833
self.gone_bananas = ['banana03', 'banana05', 'banana08']
self.banana_pos = {'banana00': (-0.853292, 4.7575, 1),
'banana01': (-4.34033, 3.84758, 1),
'banana02': (3.75634, -3.56932, 1),
'banana03': (-5.61975, 1.61125, 1),
'banana04': (2.52662, 6.00765, 1),
'banana05': (-3.67816, -5.89663, 1),
'banana06': (4.81536, 3.20421, 1),
'banana07': (2.61812, 4.21512, 1),
'banana08': (-3.33551, -6.21121, 1),
'banana09': (-0.293181, -2.1774, 1),
}
# initiate test_data
self.test_data = GetData()
# now can change some parameters
self.test_data.data_filename = '../play_data/giz_short.txt'
self.test_data.time_stamp = self.time_stamp
self.test_data.start_time = 1389990322200
# and get data
self.test_data.get_data_from_file()
def setup_three(self):
# lets use a position in a bananarchy file where we have already eaten at least one banana
# and using a trial later in the file.
print('test three, bananarchy file')
self.time_stamp = 1370018957436
self.trial_num = 4
self.avatar_pos = (3.8853, 4.31563, 1)
self.avatar_head = -1546.99902344
self.gone_bananas = ['banana9', 'banana6', 'banana3']
self.banana_pos = {'banana00': (-4.63066, -1.58329, 1),
'banana01': (3.01044, 0.545034, 1),
'banana02': (0.955586, 1.0967, 1),
'banana03': (4.38659, 4.133, 1),
'banana04': (4.15384, 2.57773, 1),
'banana05': (-4.58963, -1.25701, 1),
'banana06': (2.01608, 4.68744, 1),
'banana07': (3.0936, 3.26741, 1),
'banana08': (4.42608, -4.20852, 1),
'banana09': (0.980457, 4.95074, 1),
}
# initiate test_data
self.test_data = GetData()
# now can change some parameters
self.test_data.data_filename = '../play_data/giz_bananarchy.txt'
self.test_data.time_stamp = self.time_stamp
self.test_data.start_time = 0
# and get data
self.test_data.get_data_from_file()
def setup_four(self):
# lets use a position in a bananarchy file where we have already eaten at least one banana
# and using a trial later in the file.
print('test four, bananarchy file, first trial, full')
self.time_stamp = 1370018859357
self.trial_num = 1
self.avatar_pos = (-1.01359, 2.54856, 1)
self.avatar_head = -387.116973877
self.gone_bananas = ['banana0', 'banana1', 'banana2', 'banana3', 'banana4', 'banana5', 'banana6', 'banana7', 'banana8', 'banana9']
self.banana_pos = {'banana00': (-1.85695, -3.5224, 1),
'banana01': (-0.706922, 2.96965, 1),
'banana02': (-0.121917, 0.0990294, 1),
'banana03': (-3.49998, -1.30716, 1),
'banana04': (-2.88193, 4.38051, 1),
'banana05': (-1.27652, 4.36617, 1),
'banana06': (0.268909, 3.23278, 1),
'banana07': (-4.04485, -2.37258, 1),
'banana08': (0.350244, -2.79667, 1),
'banana09': (-1.74603, 4.66188, 1),
}
# initiate test_data
self.test_data = GetData()
# now can change some parameters
self.test_data.data_filename = '../play_data/giz_bananarchy.txt'
self.test_data.time_stamp = self.time_stamp
self.test_data.start_time = 1370018819737
# and get data
self.test_data.get_data_from_file()
def test_get_data_pos(self):
# Ensure that the trial start is before the timestamp.
self.test_data.get_data_for_time_stamp(self.test_data.time_stamp)
self.assertGreater(self.time_stamp, int(self.test_data.now_trial))
def test_first_banana_position(self):
# Make sure we have the correct bananas.
self.test_data.get_data_for_time_stamp(self.test_data.time_stamp)
#print(self.test_data.now_banana_pos)
#print(self.banana_pos['banana00'])
self.assertEqual(float(self.test_data.now_banana_pos[0][0]), self.banana_pos['banana00'][0])
self.assertEqual(float(self.test_data.now_banana_pos[0][1]), self.banana_pos['banana00'][1])
def test_last_banana_position(self):
# Make sure we have the correct bananas.
# last banana is going to be (number of bananas - 1) * 3 (each banana has 3 positions,
# but we only care about 2, since the last is always the same)
#print('before')
self.test_data.get_data_for_time_stamp(self.test_data.time_stamp)
#print('after')
#print(len(self.test_data.now_banana_pos))
last_b = (self.test_data.num_bananas - 1)
#print(last_b)
#print(self.test_data.banana_pos[0])
self.assertEqual(float(self.test_data.now_banana_pos[last_b][0]), self.banana_pos['banana09'][0])
self.assertEqual(float(self.test_data.now_banana_pos[last_b][1]), self.banana_pos['banana09'][1])
def test_bananas_eaten(self):
# Make sure the correct bananas have been identified as eaten
self.test_data.get_data_for_time_stamp(self.test_data.time_stamp)
self.assertItemsEqual(self.test_data.now_gone_bananas, self.gone_bananas)
def test_avatar_position(self):
# Make sure we have the correct avatar position. Always last position
# (very last number is time stamp)
#print('test', self.avatar_pos)
#print('test', self.avatar_pos[0])
#print('data', self.test_data.avatar_pos[0])
#print('data', self.test_data.avatar_pos[-1])
self.assertEqual(float(self.test_data.avatar_pos[-1][0]), self.avatar_pos[0])
self.assertEqual(float(self.test_data.avatar_pos[-1][1]), self.avatar_pos[1])
def test_avatar_heading(self):
# Make sure we have the correct avatar heading.
#print(self.test_data.avatar_head)
self.assertEqual(float(self.test_data.avatar_head[-1]), self.avatar_head)
def tearDown(self):
del self.test_data
