def init_org(conf):
service = conf
for peer in service['peers']:
# remove ugly sample files defined here https://github.com/hyperledger/fabric/tree/master/sampleconfig
# except core.yaml from binded volume
# call('cd $FABRIC_CA_CLIENT_HOME && rm -rf msp orderer.yaml configtx.yaml', shell=True)
##################################################################################################################
# Although a peer may use the same TLS key and certificate file for both inbound and outbound TLS, #
# we generate a different key and certificate for inbound and outbound TLS simply to show that it is permissible #
##################################################################################################################
setup_peer_msp_dir = generateMSPandTLS(peer, service)
# copy the admincerts from the admin user for being able to install chaincode
# https://stackoverflow.com/questions/48221810/what-is-difference-between-admincerts-and-signcerts-in-hyperledge-fabric-msp
# https://lists.hyperledger.org/g/fabric/topic/17549225#1250
# https://github.com/hyperledger/fabric-sdk-go/blob/master/internal/github.com/hyperledger/fabric/msp/mspimpl.go#L460
# https://jira.hyperledger.org/browse/FAB-3840
admin = service['users']['admin']
org_admin_msp_dir = admin['home'] + '/msp'
dst_admincerts_dir = setup_peer_msp_dir + '/admincerts'
create_directory(dst_admincerts_dir)
copyfile(org_admin_msp_dir + '/signcerts/cert.pem',
'%s/%s-cert.pem' % (dst_admincerts_dir, admin['name']))
def init(conf, enrollmentAdmin):
if 'peers' in conf:
init_org(conf, enrollmentAdmin)
if 'orderers' in conf:
init_orderer(conf)
create_directory(conf['broadcast_dir']['external'])
generateGenesis(conf)
def substra_org(org, orderer=None):
org_name = org['name']
print(f'Prepare Node : {org_name}')
create_directory(f"{SUBSTRA_PATH}/data/orgs/{org_name}")
create_directory(f"{SUBSTRA_PATH}/conf/{org_name}")
# CA files
create_ca_server_config(org)
create_ca_client_config(org)
# Configtx file
config_filepath = org['misc']['configtx-config-path']
create_configtx(org, config_filepath)
# Org Config files
if 'peers' in org:
create_peer_config(org)
# create_fabric_ca_peer_config(org)
# Docker-compose for org
docker_compose = generate_docker_compose_org(org, orderer, SUBSTRA_PATH, SUBSTRA_NETWORK)
intern_stop(docker_compose['path'])
start(org, docker_compose)
# Orderer Config files
if 'orderers' in org:
create_orderer_config(org)
docker_compose = generate_docker_compose_orderer(org,
SUBSTRA_PATH,
SUBSTRA_NETWORK)
intern_stop(docker_compose['path'])
start(org, docker_compose)
def init(conf):
if 'peers' in conf:
init_org(conf)
if 'orderers' in conf:
init_orderer(conf)
create_directory(conf['broadcast_dir'])
generateGenesis(conf)
def init_orderer(conf):
service = conf
# remove ugly sample files defined here https://github.com/hyperledger/fabric/tree/master/sampleconfig
# except orderer.yaml from binded volume
# call('cd $FABRIC_CA_CLIENT_HOME && rm -rf msp core.yaml configtx.yaml', shell=True)
for orderer in service['orderers']:
setup_orderer_msp_dir = generateMSPandTLS(orderer, service)
# copy the admincerts from the admin user for being able to launch orderer
# https://stackoverflow.com/questions/48221810/what-is-difference-between-admincerts-and-signcerts-in-hyperledge-fabric-msp
# https://lists.hyperledger.org/g/fabric/topic/17549225#1250
dst_admincerts_dir = setup_orderer_msp_dir + '/admincerts'
create_directory(dst_admincerts_dir)
copyfile('%s/signcerts/cert.pem' % setup_orderer_msp_dir,
'%s/%s-cert.pem' % (dst_admincerts_dir, orderer['name']))
def generateMSPandTLS(node, service):
enrollment_url = 'https://%(name)s:%(pass)s@%(host)s:%(port)s' % {
'name': node['name'],
'pass': node['pass'],
'host': service['ca']['host'],
'port': service['ca']['port']['internal']
}
# create external folder
tls_server_dir = node['tls']['dir']['external'] + '/' + node['tls'][
'server']['dir']
tls_client_dir = node['tls']['dir']['external'] + '/' + node['tls'][
'client']['dir']
create_directory(tls_server_dir)
create_directory(tls_client_dir)
# Generate server TLS cert and key pair in container
genTLSCert(node['host'],
'%s/%s' % (tls_server_dir, node['tls']['server']['cert']),
'%s/%s' % (tls_server_dir, node['tls']['server']['key']),
'%s/%s' % (tls_server_dir, node['tls']['server']['ca']),
enrollment_url)
# Generate client TLS cert and key pair for the peer CLI (will be used by external tools)
# in a binded volume
genTLSCert(node['name'],
'%s/%s' % (tls_client_dir, node['tls']['client']['cert']),
'%s/%s' % (tls_client_dir, node['tls']['client']['key']),
'%s/%s' % (tls_client_dir, node['tls']['client']['ca']),
enrollment_url)
# Enroll the node to get an enrollment certificate and set up the core's local MSP directory for starting node
setup_node_msp_dir = service['core_dir']['internal'] + '/' + node[
'name'] + '/msp'
call([
'fabric-ca-client', 'enroll', '-d', '-u', enrollment_url, '-M',
setup_node_msp_dir
])
return setup_node_msp_dir
def substra_network(orgs):
# Stop all
docker_compose_paths = glob.glob(os.path.join(SUBSTRA_PATH, 'dockerfiles/*.yaml'))
# Remove all
remove_all_docker()
for docker_compose_path in docker_compose_paths:
intern_stop(docker_compose_path)
# Create Network
call(['docker', 'network', 'create', SUBSTRA_NETWORK])
for orderer in [x for x in orgs if 'orderers' in x]:
substra_org(orderer)
else:
# Prepare each org
for org in [x for x in orgs if 'peers' in x]:
substra_org(org, orderer)
# substrabac
create_directory(f"{SUBSTRA_PATH}/dryrun/{org['name']}")
create_substrabac_config(org, orderer)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--period',
type=str,
default='all',
help='specifies which period extract features from',
choices=[
'first4days', 'first8days', 'last12hours',
'first25percent', 'first50percent', 'all'
])
parser.add_argument('--features',
type=str,
default='all',
help='specifies what features to extract',
choices=['all', 'len', 'all_but_len'])
parser.add_argument('--data',
type=str,
help='Path to the data of in-hospital mortality task',
default=os.path.join(
os.path.dirname(__file__),
'../../../data/in-hospital-mortality/'))
parser.add_argument(
'--output_dir',
type=str,
help='Directory relative which all output files are stored',
default='.')
args = parser.parse_args()
print(args)
train_reader = InHospitalMortalityReader(
dataset_dir=os.path.join(args.data, 'train'),
listfile=os.path.join(args.data, 'train_listfile.csv'),
period_length=48.0)
val_reader = InHospitalMortalityReader(
dataset_dir=os.path.join(args.data, 'train'),
listfile=os.path.join(args.data, 'val_listfile.csv'),
period_length=48.0)
test_reader = InHospitalMortalityReader(
dataset_dir=os.path.join(args.data, 'test'),
listfile=os.path.join(args.data, 'test_listfile.csv'),
period_length=48.0)
#print("shape->",train_reader.read_example(100)['X'].shape)
print('Reading data and extracting features ...')
(train_X, train_y,
train_names) = read_and_extract_features(train_reader, args.period,
args.features)
(val_X, val_y,
val_names) = read_and_extract_features(val_reader, args.period,
args.features)
(test_X, test_y,
test_names) = read_and_extract_features(test_reader, args.period,
args.features)
print(' train data shape = {}'.format(train_X.shape))
print(' validation data shape = {}'.format(val_X.shape))
print(' test data shape = {}'.format(test_X.shape))
#print("feature sample->", train_X[11])
print('Imputing missing values ...')
imputer = Imputer(missing_values=np.nan,
strategy='mean',
axis=0,
verbose=0,
copy=True)
imputer.fit(train_X)
train_X = np.array(imputer.transform(train_X), dtype=np.float32)
val_X = np.array(imputer.transform(val_X), dtype=np.float32)
test_X = np.array(imputer.transform(test_X), dtype=np.float32)
print('Normalizing the data to have zero mean and unit variance ...')
scaler = StandardScaler()
scaler.fit(train_X)
train_X = scaler.transform(train_X)
val_X = scaler.transform(val_X)
test_X = scaler.transform(test_X)
file_name = 'xgboost_{}.{}.'.format(args.period, args.features)
xgreg = xgb.XGBRegressor(colsample_bytree=0.4,
gamma=0,
learning_rate=0.07,
max_depth=3,
min_child_weight=1.5,
n_estimators=10000,
reg_alpha=0.75,
reg_lambda=0.45,
subsample=0.6,
seed=42)
xgreg.fit(train_X, train_y)
result_dir = os.path.join(args.output_dir, 'results')
common_utils.create_directory(result_dir)
with open(os.path.join(result_dir, 'train_{}.json'.format(file_name)),
'w') as res_file:
ret = print_metrics_binary(train_y, xgreg.predict(train_X))
ret = {k: float(v) for k, v in ret.items()}
json.dump(ret, res_file)
with open(os.path.join(result_dir, 'val_{}.json'.format(file_name)),
'w') as res_file:
ret = print_metrics_binary(val_y, xgreg.predict(val_X))
ret = {k: float(v) for k, v in ret.items()}
json.dump(ret, res_file)
prediction = xgreg.predict(test_X)
with open(os.path.join(result_dir, 'test_{}.json'.format(file_name)),
'w') as res_file:
ret = print_metrics_binary(test_y, prediction)
ret = {k: float(v) for k, v in ret.items()}
json.dump(ret, res_file)
save_results(
test_names, prediction, test_y,
os.path.join(args.output_dir, 'predictions', file_name + '.csv'))
def save_results(names, ts, pred, y_true, path):
common_utils.create_directory(os.path.dirname(path))
with open(path, 'w') as f:
f.write("stay,period_length,prediction,y_true\n")
for (name, t, x, y) in zip(names, ts, pred, y_true):
f.write("{},{:.6f},{:.6f},{:.6f}\n".format(name, t, x, y))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--period',
type=str,
default='all',
help='specifies which period extract features from',
choices=[
'first4days', 'first8days', 'last12hours',
'first25percent', 'first50percent', 'all'
])
parser.add_argument('--features',
type=str,
default='all',
help='specifies what features to extract',
choices=['all', 'len', 'all_but_len'])
parser.add_argument('--data',
type=str,
help='Path to the data of in-hospital mortality task',
default=os.path.join(os.path.dirname(__file__),
'../../../data/phenotyping/'))
parser.add_argument(
'--output_dir',
type=str,
help='Directory relative which all output files are stored',
default='.')
args = parser.parse_args()
print(args)
train_reader = PhenotypingReader(
dataset_dir=os.path.join(args.data, 'train'),
listfile=os.path.join(args.data, 'train_listfile.csv'))
val_reader = PhenotypingReader(
dataset_dir=os.path.join(args.data, 'train'),
listfile=os.path.join(args.data, 'val_listfile.csv'))
test_reader = PhenotypingReader(
dataset_dir=os.path.join(args.data, 'test'),
listfile=os.path.join(args.data, 'test_listfile.csv'))
#print("shape->",train_reader.read_example(100)['X'].shape)
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
print('Reading data and extracting features ...')
(train_X, train_y, train_names,
train_ts) = read_and_extract_features(train_reader, args.period,
args.features)
train_y = np.array(train_y)
(val_X, val_y, val_names,
val_ts) = read_and_extract_features(val_reader, args.period,
args.features)
val_y = np.array(val_y)
(test_X, test_y, test_names,
test_ts) = read_and_extract_features(test_reader, args.period,
args.features)
test_y = np.array(test_y)
print(' train data shape = {}'.format(train_X.shape))
print(' validation data shape = {}'.format(val_X.shape))
print(' test data shape = {}'.format(test_X.shape))
#print("feature sample->", train_X[11])
print('Imputing missing values ...')
imputer = Imputer(missing_values=np.nan,
strategy='mean',
axis=0,
verbose=0,
copy=True)
imputer.fit(train_X)
train_X = np.array(imputer.transform(train_X), dtype=np.float32)
val_X = np.array(imputer.transform(val_X), dtype=np.float32)
test_X = np.array(imputer.transform(test_X), dtype=np.float32)
print('Normalizing the data to have zero mean and unit variance ...')
scaler = StandardScaler()
scaler.fit(train_X)
train_X = scaler.transform(train_X)
val_X = scaler.transform(val_X)
test_X = scaler.transform(test_X)
file_name = 'xgboost_{}.{}.'.format(args.period, args.features)
result_dir = os.path.join(args.output_dir, 'results')
common_utils.create_directory(result_dir)
n_tasks = 25
train_activations = np.zeros(shape=train_y.shape, dtype=float)
val_activations = np.zeros(shape=val_y.shape, dtype=float)
test_activations = np.zeros(shape=test_y.shape, dtype=float)
for task_id in range(n_tasks):
print('Starting task {}'.format(task_id))
xgreg = lgb.LGBMRegressor(objective='regression',
num_leaves=11,
learning_rate=0.07,
n_estimators=10000)
eval_set = [(train_X, train_y[:, task_id]), (val_X, val_y[:, task_id])]
xgreg.fit(train_X,
train_y[:, task_id],
eval_metric='auc',
eval_set=eval_set,
verbose=True,
early_stopping_rounds=10)
train_preds = xgreg.predict(train_X)
print("train_preds shape ", train_preds.shape)
print("train_activations shape", train_activations.shape)
train_activations[:, task_id] = train_preds
val_preds = xgreg.predict(val_X)
val_activations[:, task_id] = val_preds
time_start = time.time()
test_preds = xgreg.predict(test_X)
time_elapse = time.time() - time_start
print("Processing time on Test set :", time_elapse, " s for task ",
task_id)
test_activations[:, task_id] = test_preds
with open(os.path.join(result_dir, 'train_{}.json'.format(file_name)),
'w') as f:
ret = print_metrics_multilabel(train_y, train_activations)
ret = {k: float(v) for k, v in ret.items() if k != 'auc_scores'}
json.dump(ret, f)
with open(os.path.join(result_dir, 'val_{}.json'.format(file_name)),
'w') as f:
ret = print_metrics_multilabel(val_y, val_activations)
ret = {k: float(v) for k, v in ret.items() if k != 'auc_scores'}
json.dump(ret, f)
with open(os.path.join(result_dir, 'test_{}.json'.format(file_name)),
'w') as f:
ret = print_metrics_multilabel(test_y, test_activations)
ret = {k: float(v) for k, v in ret.items() if k != 'auc_scores'}
json.dump(ret, f)
save_results(
test_names, test_ts, test_activations, test_y,
os.path.join(args.output_dir, 'predictions', file_name + '.csv'))
xgreg = lgb.LGBMRegressor(objective='regression',
num_leaves=11,
learning_rate=0.07,
n_estimators=10000)
eval_set = [(train_raw_reshape, train_raw[1]), (val_raw_reshape, val_raw[1])]
xgreg.fit(train_raw_reshape,
train_raw[1],
eval_metric='auc,auroc',
eval_set=eval_set,
verbose=True,
early_stopping_rounds=80)
result_dir = os.path.join(args.output_dir, 'results')
common_utils.create_directory(result_dir)
with open(os.path.join(result_dir, 'train_{}.json'.format(file_name)),
'w') as res_file:
ret = print_metrics_binary(train_raw[1], xgreg.predict(train_raw_reshape))
ret = {k: float(v) for k, v in ret.items()}
json.dump(ret, res_file)
with open(os.path.join(result_dir, 'val_{}.json'.format(file_name)),
'w') as res_file:
ret = print_metrics_binary(val_raw[1], xgreg.predict(val_raw_reshape))
ret = {k: float(v) for k, v in ret.items()}
json.dump(ret, res_file)
time_start = time.time()
prediction = xgreg.predict(test_raw_reshape)
parser.add_argument('--no-backup', action='store_true', default=False,
help="Remove backup binded volume. Launch from scratch")
args = vars(parser.parse_args())
# Stop all docker
remove_all_docker()
if args['no_backup']:
# create directory with correct rights
call(['rm', '-rf', f'{SUBSTRA_PATH}/data'])
call(['rm', '-rf', f'{SUBSTRA_PATH}/conf'])
call(['rm', '-rf', f'{SUBSTRA_PATH}/backup'])
call(['rm', '-rf', f'{SUBSTRA_PATH}/dryrun'])
call(['rm', '-rf', f'{SUBSTRA_PATH}/dockerfiles'])
create_directory(f'{SUBSTRA_PATH}/data/log')
create_directory(f'{SUBSTRA_PATH}/conf/')
create_directory(f'{SUBSTRA_PATH}/conf/config')
create_directory(f'{SUBSTRA_PATH}/dryrun/')
create_directory(f'{SUBSTRA_PATH}/dockerfiles/')
if not glob.glob(f'{SUBSTRA_PATH}/conf/config/conf-*.json'):
# Global configuration
if args['config']:
call(['python3', args['config']])
else:
call(['python3', os.path.join(dir_path, 'conf/2orgs.py')])
files = glob.glob(f'{SUBSTRA_PATH}/conf/config/conf-*.json')
files.sort(key=os.path.getmtime)
orgs = [json.load(open(file_path, 'r')) for file_path in files]
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--C',
type=float,
default=1.0,
help='inverse of L1 / L2 regularization')
parser.add_argument('--l1', dest='l2', action='store_false')
parser.add_argument('--l2', dest='l2', action='store_true')
parser.set_defaults(l2=True)
parser.add_argument('--period',
type=str,
default='all',
help='specifies which period extract features from',
choices=[
'first4days', 'first8days', 'last12hours',
'first25percent', 'first50percent', 'all'
])
parser.add_argument('--features',
type=str,
default='all',
help='specifies what features to extract',
choices=['all', 'len', 'all_but_len'])
parser.add_argument('--data',
type=str,
help='Path to the data of in-hospital mortality task',
default=os.path.join(
os.path.dirname(__file__),
'../../../data/in-hospital-mortality/'))
parser.add_argument(
'--output_dir',
type=str,
help='Directory relative which all output files are stored',
default='.')
args = parser.parse_args()
print(args)
train_reader = InHospitalMortalityReader(
dataset_dir=os.path.join(args.data, 'train'),
listfile=os.path.join(args.data, 'train_listfile.csv'),
period_length=48.0)
val_reader = InHospitalMortalityReader(
dataset_dir=os.path.join(args.data, 'train'),
listfile=os.path.join(args.data, 'val_listfile.csv'),
period_length=48.0)
test_reader = InHospitalMortalityReader(
dataset_dir=os.path.join(args.data, 'test'),
listfile=os.path.join(args.data, 'test_listfile.csv'),
period_length=48.0)
print('Reading data and extracting features ...')
(train_X, train_y,
train_names) = read_and_extract_features(train_reader, args.period,
args.features)
(val_X, val_y,
val_names) = read_and_extract_features(val_reader, args.period,
args.features)
(test_X, test_y,
test_names) = read_and_extract_features(test_reader, args.period,
args.features)
print(' train data shape = {}'.format(train_X.shape))
print(' validation data shape = {}'.format(val_X.shape))
print(' test data shape = {}'.format(test_X.shape))
print('Imputing missing values ...')
imputer = Imputer(missing_values=np.nan,
strategy='mean',
axis=0,
verbose=0,
copy=True)
imputer.fit(train_X)
train_X = np.array(imputer.transform(train_X), dtype=np.float32)
val_X = np.array(imputer.transform(val_X), dtype=np.float32)
test_X = np.array(imputer.transform(test_X), dtype=np.float32)
print('Normalizing the data to have zero mean and unit variance ...')
scaler = StandardScaler()
scaler.fit(train_X)
train_X = scaler.transform(train_X)
val_X = scaler.transform(val_X)
test_X = scaler.transform(test_X)
penalty = ('l2' if args.l2 else 'l1')
file_name = '{}.{}.{}.C{}'.format(args.period, args.features, penalty,
args.C)
logreg = LogisticRegression(penalty=penalty, C=args.C, random_state=42)
logreg.fit(train_X, train_y)
result_dir = os.path.join(args.output_dir, 'results')
common_utils.create_directory(result_dir)
with open(os.path.join(result_dir, 'train_{}.json'.format(file_name)),
'w') as res_file:
ret = print_metrics_binary(train_y, logreg.predict_proba(train_X))
ret = {k: float(v) for k, v in ret.items()}
json.dump(ret, res_file)
with open(os.path.join(result_dir, 'val_{}.json'.format(file_name)),
'w') as res_file:
ret = print_metrics_binary(val_y, logreg.predict_proba(val_X))
ret = {k: float(v) for k, v in ret.items()}
json.dump(ret, res_file)
time_start = time.time()
prediction = logreg.predict_proba(test_X)[:, 1]
time_elapse = time.time() - time_start
print("Processing time on Test set :", time_elapse, " s")
with open(os.path.join(result_dir, 'test_{}.json'.format(file_name)),
'w') as res_file:
ret = print_metrics_binary(test_y, prediction)
ret = {k: float(v) for k, v in ret.items()}
json.dump(ret, res_file)
save_results(
test_names, prediction, test_y,
os.path.join(args.output_dir, 'predictions', file_name + '.csv'))