def test_no_duplicated(self):
db_manage = DataManager()
folder_path = os.path.dirname(os.path.abspath(__file__))
db_manage.load_data(folder_path)
stores = db.session.query(Store).all()
self.assertEquals(len(stores), 6)
def test_probability_landscape(self):
data_mngr = DataManager(self.db_path, self.mfa_path)
prob = data_mngr.probability_landscape("dummy_01", 7)
self.assertEqual(prob, [
0.00010000000000021103, 0.3334000000000006, 1.0,
0.6667000000000001, 0.6667000000000001, 0.3334000000000006,
0.1667500000000004
])
def setUp(self):
db.create_all()
self.app = app.test_client()
self.app_context = app.app_context()
self.app_context.push()
db_manage = DataManager()
folder_path = os.path.dirname(os.path.abspath(__file__))
db_manage.load_data(folder_path)
def test_pathmgr_model(self, mock_register):
paths = util.PathManager(unittest_flag=True)
case = DataManager(self.default_case)
d = paths.modelPaths(case)
self.assertEqual(d['MODEL_DATA_DIR'], 'TEST_MODEL_DATA_ROOT/A')
self.assertEqual(d['MODEL_WK_DIR'],
'TEST_WORKING_DIR/MDTF_A_1900_2100')
def fine_tune_model():
print(f'\n\nfine tuning model...')
data_manager = DataManager()
plotter = Plotter()
train_manager = get_train_manager(data_manager)
n_filters_list = []
train_losses, test_losses = [], []
for i in range(1, 11):
n_filters = 4 * i
net = Net(conv1_out_channels=n_filters)
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
train_manager.init_model(net, optimizer)
train_and_save_model(train_manager)
(train_loss,
train_accuracy), (test_loss,
test_accuracy) = train_manager.get_losses()
n_filters_list.append(n_filters)
train_losses.append(train_loss)
test_losses.append(test_loss)
plotter.plot_filters_losses(n_filters_list, train_losses, test_losses)
return train_losses, test_losses, n_filters_list
async def create_shortlink(
origin: str, data_manager: DataManager = Depends(get_datamanager)) -> str:
"""
Создать короткую ссылку
"""
short = data_manager.shortlink_create(origin)
return short
def test_pathmgr_model(self):
config = util_mdtf.ConfigManager()
case = DataManager(self.case_dict)
d = config.paths.model_paths(case)
self.assertEqual(d['MODEL_DATA_DIR'], 'TEST_MODEL_DATA_ROOT/A')
self.assertEqual(d['MODEL_WK_DIR'],
'TEST_WORKING_DIR/MDTF_A_1900_2100')
def get_datamanager() -> DataManager:
"""
Используется как зависимость FastAPI
"""
db = _db_connect(lazy=True)
data_manager = DataManager(db)
return data_manager
def test_no_spatial_structure():
print(f'testing no spatial structure...')
fixed_shuffle_data_manager = DataManager(shuffle_type='fixed')
fresh_shuffle_data_manager = DataManager(shuffle_type='fresh')
for name, data_manager in [('fixed_shuffle', fixed_shuffle_data_manager),
('fresh_shuffle', fresh_shuffle_data_manager)]:
print(f'\ntesting {name}...')
train_manager = get_train_manager(data_manager)
net = Net()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
train_manager.init_model(net, optimizer)
train_manager.train()
print(f'\ntesting no spatial...')
train_res, test_res = train_manager.get_losses()
print(f'train loss: {train_res[0]}, train accuracy: {train_res[1]}\n'
f'test loss: {test_res[0]}, test accuracy: {test_res[1]}')
def test_pathmgr_pod(self):
config = util_mdtf.ConfigManager()
case = DataManager(self.case_dict)
pod = Diagnostic('AA')
d = config.paths.pod_paths(pod, case)
self.assertEqual(d['POD_CODE_DIR'], 'TEST_CODE_ROOT/diagnostics/AA')
self.assertEqual(d['POD_OBS_DATA'], 'TEST_OBS_DATA_ROOT/AA')
self.assertEqual(d['POD_WK_DIR'],
'TEST_WORKING_DIR/MDTF_A_1900_2100/AA')
def test_locality_of_receptive_field():
print(f'testing locality of receptive field...')
noshuffle_data_manager = DataManager(shuffle_type='none')
shuffled_data_manager = DataManager(shuffle_type='fixed')
for name, data_manager in [('no_shuffle', noshuffle_data_manager),
('shuffle', shuffled_data_manager)]:
print(f'\ntesting {name}...')
train_manager = get_train_manager(data_manager)
net = Net()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
train_manager.init_model(net, optimizer)
train_manager.train()
print(f'\ntesting local field...')
train_res, test_res = train_manager.get_losses()
print(f'train loss: {train_res[0]}, train accuracy: {train_res[1]}\n'
f'test loss: {test_res[0]}, test accuracy: {test_res[1]}')
async def get_shortlinks(
limit: int = 1000,
offset: int = 0,
datamanager: DataManager = Depends(get_datamanager)
) -> Dict[str, Any]:
"""
Получить список всех ссылок (метод не для прода)
"""
shortlinks = datamanager.shortlinks_get(limit, offset)
return shortlinks
async def get_shortlink(
short: str, data_manager: DataManager = Depends(get_datamanager)
) -> Dict[str, str]:
"""
Получить полную ссылку
"""
try:
link = data_manager.shortlink_get(short)
except ShortlinkNotFound as e:
raise HTTPException(status_code=404, detail=str(e))
return {short: link}
def test_chromosomes(self):
# Setup
data_mngr = DataManager(self.db_path, self.mfa_path)
data_mngr.select_chromosomes_from_database = MagicMock(
return_value=[("dummy_01", 150, None, 10, 5, float("inf"), 0,
'dummy')])
data_mngr.select_transcription_regions_from_database = MagicMock(
return_value=[(25, 80, 'dummy_01')])
data_mngr.probability_landscape = MagicMock(
return_value="dummy_landscape")
# Call
chrm = data_mngr.chromosomes("dummy")
# Assertions
data_mngr.select_chromosomes_from_database.assert_called_once_with(
organism="dummy")
data_mngr.select_transcription_regions_from_database.assert_called_once_with(
chromosome_code="dummy_01")
self.assertEqual(chrm,
[{
'code': 'dummy_01',
'length': 150,
'probability_landscape': 'dummy_landscape',
'transcription_regions': [{
'start': 25,
'end': 80
}],
'constitutive_origins': []
}])
class TestDiagnosticSetUp(unittest.TestCase):
# pylint: disable=maybe-no-member
default_pod = {'settings': {}, 'varlist': []}
default_case = {
'CASENAME': 'A',
'model': 'B',
'FIRSTYR': 1900,
'LASTYR': 2100,
'pod_list': ['C']
}
dummy_paths = {
'CODE_ROOT': 'A',
'OBS_DATA_ROOT': 'B',
'MODEL_DATA_ROOT': 'C',
'WORKING_DIR': 'D',
'OUTPUT_DIR': 'E'
}
dummy_var_translate = {
'convention_name': 'not_CF',
'var_names': {
'pr_var': 'PRECT',
'prc_var': 'PRECC'
}
}
@mock.patch('src.util_mdtf.util.read_json',
return_value=dummy_var_translate)
def setUp(self, mock_read_json):
setUp_ConfigManager(config=self.default_case,
paths=self.dummy_paths,
pods={'DUMMY_POD': self.default_pod})
_ = util_mdtf.VariableTranslator(unittest=True)
def tearDown(self):
tearDown_ConfigManager()
# ---------------------------------------------------
@unittest.skip("")
@mock.patch.multiple(DataManager, __abstractmethods__=set())
@mock.patch('os.path.exists', return_value=True)
def test_set_pod_env_vars_paths(self, mock_exists):
# check definition of pod paths
case = DataManager(self.default_case)
pod = Diagnostic('DUMMY_POD')
case._setup_pod(pod)
pod.POD_WK_DIR = 'A'
pod._set_pod_env_vars()
self.assertEqual(pod.pod_env_vars['POD_HOME'],
'TEST_CODE_ROOT/diagnostics/C')
self.assertEqual(pod.pod_env_vars['OBS_DATA'], 'TEST_OBS_DATA_ROOT/C')
self.assertEqual(pod.pod_env_vars['WK_DIR'], 'A')
class TestDataManagerSetup(unittest.TestCase):
# pylint: disable=abstract-class-instantiated
default_case = {
'CASENAME': 'A', 'model': 'B', 'FIRSTYR': 1900, 'LASTYR': 2100,
'pod_list': ['C']
}
default_pod_CF = {
'settings':{},
'varlist':[{'var_name': 'pr_var', 'freq':'mon'}]
}
dummy_paths = {
'CODE_ROOT':'A', 'OBS_DATA_ROOT':'B', 'MODEL_DATA_ROOT':'C',
'WORKING_DIR':'D', 'OUTPUT_DIR':'E'
}
dummy_var_translate = {
'convention_name':'not_CF',
'var_names':{'pr_var': 'PRECT', 'prc_var':'PRECC'}
}
@mock.patch('src.util_mdtf.util.read_json', return_value=dummy_var_translate)
def setUp(self, mock_read_json):
setUp_ConfigManager(
config=self.default_case,
paths=self.dummy_paths,
pods={'C': self.default_pod_CF}
)
_ = util_mdtf.VariableTranslator(unittest = True)
def tearDown(self):
tearDown_ConfigManager()
# ---------------------------------------------------
def test_setup_model_paths(self):
pass
# expect failure because variable name env vars set by POD now
@unittest.expectedFailure
def test_set_model_env_vars(self):
# set env vars for model
case = DataManager(self.default_case)
case.convention = 'not_CF'
dummy = {'envvars':{}}
case._set_model_env_vars(dummy)
self.assertEqual(os.environ['pr_var'], 'PRECT')
self.assertEqual(os.environ['prc_var'], 'PRECC')
class TestDataManagerSetupNonCFPod(unittest.TestCase):
# pylint: disable=abstract-class-instantiated
default_case = {
'CASENAME': 'A', 'model': 'B', 'FIRSTYR': 1900, 'LASTYR': 2100,
'pod_list': ['C']
}
default_pod_not_CF = {
'settings': {'variable_convention':'not_CF'},
'varlist': [{'var_name': 'PRECT', 'freq':'mon'}]
}
dummy_paths = {
'CODE_ROOT':'A', 'OBS_DATA_ROOT':'B', 'MODEL_DATA_ROOT':'C',
'WORKING_DIR':'D', 'OUTPUT_DIR':'E'
}
dummy_var_translate = {
'convention_name':'not_CF',
'var_names':{'pr_var': 'PRECT', 'prc_var':'PRECC'}
}
@mock.patch('src.util_mdtf.util.read_json', return_value=dummy_var_translate)
def setUp(self, mock_read_json):
setUp_ConfigManager(
config=self.default_case,
paths=self.dummy_paths,
pods={'C': self.default_pod_not_CF}
)
_ = util_mdtf.VariableTranslator(unittest = True)
def tearDown(self):
tearDown_ConfigManager()
def test_setup_pod_custom_cf(self):
case = DataManager(self.default_case)
pod = Diagnostic('C')
case._setup_pod(pod)
self.assertEqual(pod.varlist[0].CF_name, 'pr_var')
self.assertEqual(pod.varlist[0].name_in_model, 'pr_var')
def test_non_linearities():
print(f'testing non linearities...')
non_linear_net = Net(fc1_out_channels=120, non_linear=True)
linear_net = Net(fc1_out_channels=120, non_linear=False)
big_linear_net = Net(fc1_out_channels=240, non_linear=False)
nets = [
NetInfo('non_linear', non_linear_net),
NetInfo('linear', linear_net),
NetInfo('big_linear', big_linear_net)
]
for net in nets:
data_manager = DataManager()
train_manager = get_train_manager(data_manager)
# if net is not nets[1]: continue
print(f'\ntesting {net.name}...')
optimizer = optim.SGD(net.object.parameters(), lr=0.001, momentum=0.9)
train_manager.init_model(net.object, optimizer)
train_manager.train()
train_loss, test_loss = train_manager.get_losses()
print(f'train loss: {train_loss}\ntest loss: {test_loss}')
import logging
import os
import sys
from flask import Flask, request, make_response, jsonify
from flask_restx import Api, Resource
sys.path.append(os.path.dirname(os.path.abspath("./app")))
import config.config as cfg
from src.setup.setup import logger_setup
from src.setup.flask_app import run_flask_app
from src.data_manager import DataManager
logger = logging.getLogger(__name__)
data_manager = DataManager(cfg)
app = Flask(__name__)
API = Api(
app,
title="TEST UPCITI API",
description="Documentation of the API of the data scraper module",
doc="/doc",
prefix="/test_upciti/api/v1",
)
STATUS_NS = API.namespace("STATUS", path="/")
POPULATE_DB_NS = API.namespace("Populate database", path="/")
GET_CELL_TOWER_NS = API.namespace("Get a list of cell towers", path="/")
from flask import Flask, render_template, request, redirect
import cPickle as pickle
import requests
import socket
import time
from datetime import datetime
import json
from src.data_manager import DataManager
from src.model_utils import Fraud
app = Flask(__name__)
data_broker = DataManager()
fraud_model = Fraud()
fraud_model.load_model('data/model.pkl')
SERVER_IP = '10.3.35.189'
PORT = 5353
REG_URL = 'http://' + SERVER_IP + ':5000/register'
@app.route('/home')
def index():
return render_template('home.html')
@app.route('/score', methods=['POST', 'GET'])
def fraud_detection_score():
if request.method == 'GET':
return ""
elif request.method == 'POST':
# print type(request.json)
# print json.dump(request.json)
def test_select_chromosomes_from_database(self):
data_mngr = DataManager(self.db_path, self.mfa_path)
data = data_mngr.select_chromosomes_from_database(organism="dummy")
# Comparison based on the known data inside db
self.assertEqual(
data, [('dummy_01', 150, None, 10, 5, float("inf"), 0, 'dummy')])
def test_select_transcription_regions_from_database(self):
data_mngr = DataManager(self.db_path, self.mfa_path)
data = data_mngr.select_transcription_regions_from_database(
chromosome_code='dummy_01')
self.assertEqual(data, [(25, 80, 'dummy_01')])
import json
from flask import abort, Flask, Blueprint, jsonify, request
from flask_cors import cross_origin
from src.data_manager import DataManager
from src.const import PREFECTURES
from src.swagger_settings import set_config
data_manager = DataManager()
app = Flask(__name__)
apiv1 = Blueprint('apiv1', __name__, url_prefix='/api/v1')
@app.route('/')
def index():
return jsonify({
'message':
'Web API to get COVID-19(coronavirus) information of each prefecture in Japan.'
})
@apiv1.route('/total')
@cross_origin(apiv1)
def total():
""" Get the total number of infected people and deaths in Japan
---
parameters:
- in: query
name: history
type: boolean
# save the image
img_name = "img#" + str(image_index) + "_cell#" + str(
i) + export_img_extension
filepath = os.path.join(out_path, img_name)
io.imsave(filepath, cell)
logging.info("extracted cell: {}".format(img_name))
if __name__ == "__main__":
logging.basicConfig(
level=logging.INFO,
format="[%(asctime)s] - [%(name)s] - [%(levelname)s] - %(message)s")
config = configparser.ConfigParser()
config.read("config.ini")
data_manager = DataManager.from_file()
start_time = time.monotonic()
inpath = data_manager.get_input_path()
outpath = data_manager.get_cells_path()
logging.info("input path: {}".format(inpath))
logging.info("extracted cells will be saved in: {}".format(outpath))
files = data_manager.get_input_images()
if not files:
logging.error(
"{} directory is empty! No image to process".format(inpath))
for i, infile in enumerate(files):
def test_constructor(self):
data_mngr = DataManager(self.db_path, self.mfa_path)
self.assertEqual(self.db_path, data_mngr.database_path)
self.assertEqual(self.mfa_path, data_mngr.mfa_seq_folder_path)
class TestDataManagerSetup(unittest.TestCase):
@mock.patch('src.util.read_yaml',
return_value={
'convention_name': 'not_CF',
'var_names': {
'pr_var': 'PRECT',
'prc_var': 'PRECC'
}
})
def setUp(self, mock_read_yaml):
# set up translation dictionary without calls to filesystem
temp = util.VariableTranslator(unittest_flag=True)
temp = util.PathManager(unittest_flag=True)
def tearDown(self):
# call _reset method deleting clearing Translator for unit testing,
# otherwise the second, third, .. tests will use the instance created
# in the first test instead of being properly initialized
temp = util.VariableTranslator(unittest_flag=True)
temp._reset()
temp = util.PathManager(unittest_flag=True)
temp._reset()
# ---------------------------------------------------
default_case = {
'CASENAME': 'A',
'model': 'B',
'FIRSTYR': 1900,
'LASTYR': 2100,
'pod_list': []
}
default_pod_CF = {
'settings': {},
'varlist': [{
'var_name': 'pr_var',
'freq': 'mon'
}]
}
default_pod_not_CF = {
'settings': {
'variable_convention': 'not_CF'
},
'varlist': [{
'var_name': 'PRECT',
'freq': 'mon'
}]
}
def test_setup_model_paths(self):
pass
@mock.patch.multiple(DataManager, __abstractmethods__=set())
def test_set_model_env_vars(self):
# set env vars for model
case = DataManager(self.default_case)
case.convention = 'not_CF'
dummy = {'envvars': {}}
case._set_model_env_vars(dummy)
self.assertEqual(os.environ['pr_var'], 'PRECT')
self.assertEqual(os.environ['prc_var'], 'PRECC')
def test(model: Model, data: DataManager, verbose=True):
if verbose: print('Start predicting..', end='')
y_pred = model.predict(data.X)
if verbose: print('Done')
result = calc_metrics(data.y,
y_pred,
len(data.X_cols),
data.log_target,
return_str=True)
print(result)
if __name__ == '__main__':
args = parse_args()
# Data loading
if args.verbose: print('Data loading..', end='')
data_manager = DataManager(args.data_path, args.log_target,
args.feature_engineering)
if args.verbose: print('Done')
# Model loading
if args.verbose: print('Model loading..', end='')
model = Model(args.model_name)
if args.verbose: print('Done')
test(model, data_manager, args.verbose)
def get_train_manager(data_manager: DataManager):
trainloader, testloader, classes = data_manager.load_cifar10()
critertion = nn.CrossEntropyLoss()
train_manager = TrainManager(trainloader, testloader, critertion)
return train_manager
async def delete_shortlink(
short: str, data_manager: DataManager = Depends(get_datamanager)):
"""
Удалить короткую ссылку (удаляет упреждающе, без проверки на наличие)
"""
data_manager.shortlink_delete(short)
pass
@mock.patch.multiple(DataManager, __abstractmethods__=set())
def test_set_model_env_vars(self):
# set env vars for model
case = DataManager(self.default_case)
case.convention = 'not_CF'
dummy = {'envvars': {}}
case._set_model_env_vars(dummy)
self.assertEqual(os.environ['pr_var'], 'PRECT')
self.assertEqual(os.environ['prc_var'], 'PRECC')
@mock.patch.multiple(DataManager, __abstractmethods__=set())
def test_set_model_env_vars_no_model(self):
# exit if can't find model
case = DataManager(self.default_case)
case.convention = 'nonexistent'
dummy = {'envvars': {}}
self.assertRaises(AssertionError, case._set_model_env_vars, dummy)
def test_setup_html(self):
pass
@mock.patch.multiple(DataManager, __abstractmethods__=set())
@mock.patch('src.shared_diagnostic.util.read_yaml',
return_value=default_pod_CF)
@mock.patch('os.path.exists', return_value=True)
def test_setup_pod_cf_cf(self, mock_exists, mock_read_yaml):
case = DataManager(self.default_case)
pod = Diagnostic('C')
case._setup_pod(pod)
