def train(train_fn,
test_fn,
reporthook=None,
final_reporthook=None,
**cfg_kwargs): # pragma: no cover
from longling.ML.toolkit.hyper_search import prepare_hyper_search
cfg_kwargs, reporthook, final_reporthook, tag = prepare_hyper_search(
cfg_kwargs,
Configuration,
reporthook,
final_reporthook,
primary_key="macro_avg:f1")
_cfg = Configuration(**cfg_kwargs)
print(_cfg)
_net = get_net(**_cfg.hyper_params)
train_data = etl(_cfg.var2val(train_fn), params=_cfg)
test_data = etl(_cfg.var2val(test_fn), params=_cfg)
numerical_check(_net,
_cfg,
train_data,
test_data,
dump_result=not tag,
reporthook=reporthook,
final_reporthook=final_reporthook)
def train(train_fn, test_fn, vec_files, reporthook=None, final_reporthook=None, **cfg_kwargs): # pragma: no cover
from longling import print_time
from longling.ML.toolkit.hyper_search import prepare_hyper_search
cfg_kwargs, reporthook, final_reporthook, tag = prepare_hyper_search(
cfg_kwargs, Configuration, reporthook, final_reporthook, final_key="prf:avg:f1"
)
_cfg = Configuration(**cfg_kwargs)
_cfg.logger.info(_cfg)
vec_files = parse_vec_files(vec_files)
with print_time(tips='loading embedding', logger=_cfg.logger):
embeddings = load_embedding({k: _cfg.var2val(v) for k, v in vec_files.items()}, _cfg.logger)
train_data = etl(_cfg.var2val(train_fn), embeddings, params=_cfg)
test_data = etl(_cfg.var2val(test_fn), embeddings, params=_cfg)
embedding_size = get_embedding_size(embeddings)
_net = get_net(embedding_size=embedding_size, **_cfg.hyper_params)
_net.initialize(ctx=_cfg.ctx)
_net.embedding.set_weight(get_embedding_array(embeddings))
numerical_check(_net, _cfg, train_data, test_data, dump_result=not tag, reporthook=reporthook,
final_reporthook=final_reporthook)
def train(train_fn, test_fn, **cfg_kwargs):
_cfg = Configuration(**cfg_kwargs)
_net = get_net(**_cfg.hyper_params)
train_data = etl(_cfg.var2val(train_fn), params=_cfg)
test_data = etl(_cfg.var2val(test_fn), params=_cfg)
numerical_check(_net, _cfg, train_data, test_data, dump_result=True)
def test_etl_wwc(cls):
extract_date = datetime.strptime('2019-03-02',
transform.EXTRACT_DATE_FORMAT).date()
data_dir = os.path.join(TEST_DIR, etl.DATA_DIR)
etl.etl('wwc', extract_date, data_dir=data_dir, db=TEST_DB)
conn = sqlite3.connect(TEST_DB)
c = conn.cursor()
c.execute('SELECT count(*) FROM wwc_accounts')
assert c.fetchone()[0] == 5
def main():
# delete_rds_instance()
if DB_ON_CLOUD:
setup_rds_instance()
create_db()
create_schema()
etl()
def train(train_fn,
test_fn,
reporthook=None,
final_reporthook=None,
primary_key="macro_auc",
params_save=False,
**cfg_kwargs): # pragma: no cover
from longling.ML.toolkit.hyper_search import prepare_hyper_search
cfg_kwargs, reporthook, final_reporthook, tag = prepare_hyper_search(
cfg_kwargs,
Configuration,
reporthook,
final_reporthook,
primary_key=primary_key)
_cfg = Configuration(**cfg_kwargs)
print(_cfg)
train_data, train_df = etl(_cfg.var2val(train_fn), params=_cfg)
test_data, _ = etl(_cfg.var2val(test_fn), params=_cfg)
_net = get_net(**_cfg.hyper_params)
net_init(_net,
cfg=_cfg,
**_cfg.init_params,
int_df=train_df,
user_num=_cfg.hyper_params["user_num"],
item_num=_cfg.hyper_params["item_num"],
logger=_cfg.logger)
numerical_check(_net,
_cfg,
train_data,
test_data,
dump_result=not tag,
reporthook=reporthook,
final_reporthook=final_reporthook,
params_save=params_save)
def main():
# Create new DB and connect to it
conn = sqlite3.connect("Transcation_DB.db")
select_cursor = conn.cursor()
insert_cursor = conn.cursor()
# Load transactions from csv file to SQLite DB
transaction_table_name = 'transaction_raw_data'
transaction_column_types = [
'text', 'date', 'text', 'int', 'text', 'text', 'text', 'text'
]
transaction_path = 'anon_transactions_sample.csv'
load_csv_sqlite(transaction_path, transaction_table_name,
transaction_column_types, insert_cursor)
# Load category csv file to SQLite DB
category_table_name = 'category_raw_data'
category_column_types = ['text', 'text', 'text', 'text', 'text', 'text']
category_path = 'categories.csv'
load_csv_sqlite(category_path, category_table_name, category_column_types,
insert_cursor)
#Create datawarehouse table structure
create_dw(insert_cursor)
#Load transaction data to datawarehouse
transaction_etl(select_cursor, insert_cursor)
# Load category data to datawarehouse
sector_extract = ''' SELECT DISTINCT sector_id, sector_name FROM category_raw_data '''
sector_load = ''' INSERT OR IGNORE INTO sector (sector_id, sector_name) VALUES (?, ?) '''
etl(select_cursor, insert_cursor, sector_extract, sector_load)
category_extract = ''' SELECT DISTINCT category_id, category_name, sector_id FROM category_raw_data'''
category_load = ''' INSERT OR IGNORE INTO category (category_id, category_name, sector_id) VALUES (?, ?, ?) '''
etl(select_cursor, insert_cursor, category_extract, category_load)
subcategory_extract = ''' SELECT DISTINCT subcategory_id, subcategory_name, category_id FROM category_raw_data '''
subcategory_load = ''' INSERT OR IGNORE INTO subcategory (subcategory_id, subcategory_name, category_id) VALUES (?, ?, ?) '''
etl(select_cursor, insert_cursor, subcategory_extract, subcategory_load)
conn.commit()
conn.close()
from etl import etl
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from mpl_toolkits.mplot3d import Axes3D
points = etl()
Kmean = KMeans(n_clusters=2)
Kmean.fit(points)
labels = Kmean.labels_
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
for item, label in zip(points, labels):
if label == 0:
ax.scatter(item[0], item[1], item[2], c='r', marker='o')
else:
ax.scatter(item[0], item[1], item[2], c='b', marker='s')
ax.set_xlabel('changeOverTime')
ax.set_ylabel('changePercent')
ax.set_zlabel('volume')
plt.show()
""" Main Module"""
from config_module import config
from create_tables import main as create_tables
from etl import main as etl
from load_csv_files import main as load_csv_file
from question import main as question
from sql_queries.oltp import create_table_queries, drop_table_queries
"""
3 thins done in this project:
1- Creating original database and tables
2- Loading csv files data to original table
3- Creating new database and tables which is a fact and multiple dimension tables through sql
queries for transformation in etl process. Queries can be check in sql_queries.oltp module
"""
if __name__ == '__main__':
obj = config.resolve()
create_tables(obj, "OLTP_DB", create_table_queries, drop_table_queries)
load_csv_file()
etl()
question()
def updateData():
"""Function to update dataset."""
print("[INFO] Updating data")
data = etl()
data.to_csv('data.csv', index=False)
print(f"[INFO] Data has been updated on {datetime.now}.")