def second_insert():
data = query(test_factor, (start_time, second_end_time))
HDF5Engine.insert(
data,
ParamsParser.from_dict(
db_path, {
'rel_path':
rel_path,
'store_fmt':
(DataClassification.STRUCTURED, DataValueCategory.NUMERIC,
DataFormatCategory.PANEL),
'dtype':
np_dtype('float64')
}))
def query_compare():
old_db_data = query(test_factor, (start_time, second_end_time))
threshold_loc = old_db_data.index.get_loc(first_end_time) + 1
new_db_data = HDF5Engine.query(
ParamsParser.from_dict(
db_path, {
"rel_path":
'test',
'start_time':
start_time,
'end_time':
second_end_time,
"store_fmt":
(DataClassification.STRUCTURED, DataValueCategory.NUMERIC,
DataFormatCategory.PANEL)
}))
old_db_data = old_db_data.fillna(0)
new_db_data = new_db_data.fillna(0)
columns1 = new_db_data.columns[:initial_size]
columns2 = new_db_data.columns
is_close1 = np.isclose(old_db_data.ix[:threshold_loc, columns1],
new_db_data.ix[:threshold_loc, columns1])
is_close2 = np.isclose(old_db_data.ix[threshold_loc:, columns2],
new_db_data.iloc[threshold_loc:])
print(np.all(is_close1))
print(np.all(is_close2))
def first_insert():
data = query(test_factor,
(start_time, first_end_time)).iloc[:, :initial_size]
columns = list(data.columns)
shuffle(columns)
data = data.loc[:, columns]
HDF5Engine.insert(
data,
ParamsParser.from_dict(
db_path, {
'rel_path':
rel_path,
'store_fmt':
(DataClassification.STRUCTURED, DataValueCategory.NUMERIC,
DataFormatCategory.PANEL),
'dtype':
np_dtype('float64')
}))
from numpy import dtype as np_dtype
from database.hdf5Engine.dbcore import HDF5Engine
from database.const import DataFormatCategory, DataValueCategory, DataClassification
from database.db import ParamsParser
from fmanager import query
TEST_FACTOR = 'CLOSE'
start_time = '2017-01-01'
end_time = '2018-01-15'
new_end = '2018-02-01'
sample_df = query(TEST_FACTOR, (start_time, end_time))
new_data = query(TEST_FACTOR, (end_time, new_end))
db_path = r'C:\Users\c\Desktop\test'
# file_path = join(db_path, 'test.h5')
# if exists(file_path):
# remove(file_path)
HDF5Engine.insert(
new_data,
ParamsParser.from_dict(
db_path, {
"rel_path":
'test',
"store_fmt": (DataClassification.STRUCTURED,
DataValueCategory.NUMERIC, DataFormatCategory.PANEL),
"dtype":
np_dtype('float64')
}))
# -*- encoding: utf-8
from database.hdf5Engine.dbcore import HDF5Engine
from database.db import ParamsParser
from database.const import DataClassification, DataValueCategory, DataFormatCategory
db_path = r'C:\Users\c\Desktop\test'
rel_path = 'test_remove'
result = HDF5Engine.remove_data(ParamsParser.from_dict(db_path, {'rel_path': rel_path,
'store_fmt': (DataClassification.STRUCTURED,
DataValueCategory.NUMERIC,
DataFormatCategory.PANEL)}))
print(result)
# @Version : $Id$
import numpy as np
from database.hdf5Engine.dbcore import HDF5Engine
from database.db import ParamsParser
from database.const import DataClassification, DataValueCategory, DataFormatCategory
from fmanager import query
start_time = '2017-01-01'
end_time = '2018-01-01'
db_path = r'C:\Users\c\Desktop\test'
data = HDF5Engine.query(
ParamsParser.from_dict(
db_path, {
"rel_path":
'test_series',
'start_time':
start_time,
'end_time':
end_time,
"store_fmt":
(DataClassification.STRUCTURED, DataValueCategory.NUMERIC,
DataFormatCategory.TIME_SERIES)
}))
fm_data = query('CLOSE', (start_time, end_time)).iloc[:, 0]
data = data.fillna(-10000)
fm_data = fm_data.fillna(-10000)
print(np.all(data == fm_data))
second_insert_test = True
sample_series = query(TEST_FACTOR, (start_time, end_time)).iloc[:, 0]
new_sample_series = query(TEST_FACTOR, (start_time, new_end)).iloc[:, 0]
db_path = r'C:\Users\c\Desktop\test'
if not second_insert_test:
file_path = join(db_path, 'test_series.h5')
if exists(file_path):
remove(file_path)
HDF5Engine.insert(
sample_series,
ParamsParser.from_dict(
db_path, {
"rel_path":
'test_series',
"store_fmt":
(DataClassification.STRUCTURED, DataValueCategory.NUMERIC,
DataFormatCategory.TIME_SERIES),
"dtype":
np_dtype('float64')
}))
else:
HDF5Engine.insert(
new_sample_series,
ParamsParser.from_dict(
db_path, {
"rel_path":
'test_series',
"store_fmt":
(DataClassification.STRUCTURED, DataValueCategory.NUMERIC,
DataFormatCategory.TIME_SERIES),
# -*- encoding: utf-8
from database.hdf5Engine.dbcore import HDF5Engine
from database.const import DataClassification, DataFormatCategory, DataValueCategory
from database.db import ParamsParser
db_path = r'C:\Users\c\Desktop\test'
src_path = 'test_move_src'
dest_path = 'test_move_dest'
src_params = ParamsParser.from_dict(
db_path, {
'rel_path':
src_path,
'store_fmt': (DataClassification.STRUCTURED, DataValueCategory.NUMERIC,
DataFormatCategory.PANEL)
})
dest_params = ParamsParser.from_dict(
db_path, {
'rel_path':
dest_path,
'store_fmt': (DataClassification.STRUCTURED, DataValueCategory.NUMERIC,
DataFormatCategory.PANEL)
})
result = HDF5Engine.move_to(src_params, dest_params)
print(result)