def setUp(self):
init_logger()
try:
os.chdir(PACKAGE_DIRECTORY_COM)
except OSError:
pass
# pylint: disable=line-too-long
storage_command_generator.USER_NAME = "test-user"
storage_command_generator.JOB_NAME = "job"
storage_command_generator.STORAGE_CONFIGS = "[\"STORAGE_NFS\", \"STORAGE_TEST\", \"STORAGE_SAMBA\", \"STORAGE_AZURE_FILE\", \"STORAGE_AZURE_BLOB\"]"
storage_command_generator.KUBE_APISERVER_ADDRESS = "http://api_server_url:8080"
def plugin_init():
init_logger()
parser = argparse.ArgumentParser()
parser.add_argument("plugin_config",
help="plugin config for runtime plugin in yaml")
parser.add_argument("pre_script", help="script for pre commands")
parser.add_argument("post_script", help="script for post commands")
args = parser.parse_args()
plugin_config = yaml.safe_load(args.plugin_config)
return [plugin_config, args.pre_script, args.post_script]
:param test_size: size of test set. It must be a float between 0 and 1
:param random_state: random state / bean
:param shuffle: boolean that determines if data should be shuffled
:param stratify: If True, data is split in a stratified fashion
:return:
"""
train_x, test_x, train_y, test_y = \
self.get_train_test_split(test_size=test_size,
random_state=random_state,
shuffle=shuffle,
stratify=stratify)
self.save_train_test_sets(inputs=(train_x, test_x),
targets=(train_y, test_y),
names=('training', 'test'))
if __name__ == '__main__':
cfg = load_config('./conf/conf.yaml')
init_logger(cfg['logging'], cfg['logging']['name'])
prep = Preprocessor(cfg['data']['raw_path'],
cfg['preprocess']['criteria'],
cfg['data']['file_name_format'],
cfg['data']['classes_list'],
cfg['preprocess']['classes_ranges'],
cfg['preprocess']['dest_path'])
prep.run(test_size=cfg['preprocess']['test_size'],
random_state=cfg['preprocess']['random_state'],
shuffle=cfg['preprocess']['shuffle'],
stratify=True)
import sys
import unittest
import yaml
# pylint: disable=wrong-import-position
sys.path.append(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "../src"))
sys.path.append(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "../src/init.d"))
import image_checker
from common.utils import init_logger
# pylint: enable=wrong-import-position
PACKAGE_DIRECTORY_COM = os.path.dirname(os.path.abspath(__file__))
init_logger()
# pylint: disable=protected-access
def prepare_image_check(job_config_path):
def decorator(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
os.environ["FC_TASKROLE_NAME"] = "worker"
if os.path.exists(job_config_path):
with open(job_config_path, 'r') as f:
self.config = yaml.load(f, Loader=yaml.FullLoader)
func(self, *args, **kwargs)
del os.environ["FC_TASKROLE_NAME"]
return wrapper
def main():
parser = argparse.ArgumentParser()
parser.add_argument("job_config", help="job config yaml")
parser.add_argument("secret_file", help="secret file path")
args = parser.parse_args()
LOGGER.info("get job config from %s", args.job_config)
with open(args.job_config) as config:
job_config = yaml.safe_load(config)
if not os.path.isfile(args.secret_file):
job_secret = None
else:
with open(args.secret_file) as f:
job_secret = yaml.safe_load(f.read())
LOGGER.info("Start checking docker image")
image_checker = ImageChecker(job_config, job_secret)
try:
if not image_checker.is_docker_image_accessible():
sys.exit(1)
except Exception: #pylint: disable=broad-except
LOGGER.warning("Failed to check image", exc_info=True)
if __name__ == "__main__":
utils.init_logger()
main()
block_list = Block.objects.get()
logger.debug('Block list count {}'.format(len(block_list)))
block_chain = BlockchainFactory.build_blockchain(block_list)
logger.debug('Last block hash {}'.format(block_chain.last_block_hash))
file = open(settings['builder_tree_path'], 'w')
file.write(json.dumps(block_chain.__dict__))
logger.info('Finished blockchain tree building')
except Exception as e:
logger.error(e)
if __name__ == "__main__":
init_logger(settings)
logger.info('Starting the application')
app = tornado.web.Application([
(r"/", BuilderListener),
])
app.listen(settings['builder_server']['port'])
io_loop = tornado.ioloop.IOLoop.current()
scheduler = tornado.ioloop.PeriodicCallback(build_tree, 60000)
logger.info('Listening for connections on {}:{}'.format(
settings['miner_server']['ip'], settings['miner_server']['port']))
scheduler.start()
io_loop.start()
'''
Created on May 22, 2016
@author: ajaniv
'''
import logging
import time
import Queue
from common import utils
logger = logging.getLogger(__name__)
utils.init_logger(logger)
def read_file_using_queues(i, work_queue, results_queue):
while not work_queue.empty():
logger.debug('%s: fetching next file', i)
try:
file_name = work_queue.get(block=False)
logger.debug('%s: loading: %s', i, file_name)
time.sleep(i + 2)
results_queue.put((0, i, file_name))
work_queue.task_done()
except Queue.Empty:
pass
logger.debug('%s: exiting', i)
def read_file_wrapper(a_b):
return read_file(*a_b)
from models.train import distributed_train, test
from models.utils import get_model
from viz.training_plots import training_plots
print = functools.partial(print, flush=True)
torch.set_printoptions(linewidth=120)
# ------------------------------------------------------------------------------
# Setups
# ------------------------------------------------------------------------------
args = Arguments(argparser())
hook = sy.TorchHook(torch)
device = get_device(args)
paths = get_paths(args, distributed=True)
log_file, std_out = init_logger(paths.log_file, args.dry_run, args.load_model)
if os.path.exists(paths.tb_path):
shutil.rmtree(paths.tb_path)
tb = SummaryWriter(paths.tb_path)
print('+' * 80)
print(paths.model_name)
print('+' * 80)
print(args.__dict__)
print('+' * 80)
# prepare graph and data
_, workers = get_fl_graph(hook, args.num_workers)
print('Loading data: {}'.format(paths.data_path))
X_trains, _, y_trains, _, meta = pkl.load(open(paths.data_path, 'rb'))
ap.add_argument("--dataset", required=True, type=str)
ap.add_argument("--num-workers", required=True, type=int)
ap.add_argument("--non-iid", required=True, type=int)
ap.add_argument("--repeat", required=False, type=int, default=True)
ap.add_argument("--shuffle", required=False, type=booltype, default=True)
ap.add_argument("--stratify", required=False, type=booltype, default=True)
ap.add_argument("--uniform-data", required=False, type=booltype, default=False)
ap.add_argument("--dry-run", required=False, type=booltype, default=False)
args = vars(ap.parse_args())
args = Struct(**args)
filename = get_data_path(cfg.ckpt_path, args)
folder = '{}_{}'.format(args.dataset, args.num_workers)
log_file, std_out = init_logger(
os.path.join(cfg.ckpt_path, folder,
'logs/data_non_iid_{}.log'.format(args.non_iid)))
num_train = cfg.num_trains[args.dataset]
num_test = cfg.num_tests[args.dataset]
num_classes = cfg.output_sizes[args.dataset]
kwargs = {}
train_loader = get_trainloader(args.dataset, num_train)
test_loader = get_testloader(args.dataset, num_test)
for data, target in train_loader:
X_train = data
y_train = target
from models.multi_class_hinge_loss import multiClassHingeLoss
from models.utils import get_model
from models.train import test, sdirs_approximation
print = functools.partial(print, flush=True)
torch.set_printoptions(linewidth=120)
# ------------------------------------------------------------------------------
# Setups
# ------------------------------------------------------------------------------
args = Arguments(argparser())
hook = sy.TorchHook(torch)
device = get_device(args)
paths = get_paths(args)
log_file, std_out = init_logger(paths.log_file, args.dry_run)
if os.path.exists(paths.tb_path):
shutil.rmtree(paths.tb_path)
tb = SummaryWriter(paths.tb_path)
print('+' * 80)
print(paths.model_name)
print('+' * 80)
print(args.__dict__)
print('+' * 80)
if args.batch_size == 0:
args.batch_size = args.num_train
print("Resetting batch size: {}...".format(args.batch_size))
