def init_org(conf): service = conf for peer in service['peers']: # remove ugly sample files defined here https://github.com/hyperledger/fabric/tree/master/sampleconfig # except core.yaml from binded volume # call('cd $FABRIC_CA_CLIENT_HOME && rm -rf msp orderer.yaml configtx.yaml', shell=True) ################################################################################################################## # Although a peer may use the same TLS key and certificate file for both inbound and outbound TLS, # # we generate a different key and certificate for inbound and outbound TLS simply to show that it is permissible # ################################################################################################################## setup_peer_msp_dir = generateMSPandTLS(peer, service) # copy the admincerts from the admin user for being able to install chaincode # https://stackoverflow.com/questions/48221810/what-is-difference-between-admincerts-and-signcerts-in-hyperledge-fabric-msp # https://lists.hyperledger.org/g/fabric/topic/17549225#1250 # https://github.com/hyperledger/fabric-sdk-go/blob/master/internal/github.com/hyperledger/fabric/msp/mspimpl.go#L460 # https://jira.hyperledger.org/browse/FAB-3840 admin = service['users']['admin'] org_admin_msp_dir = admin['home'] + '/msp' dst_admincerts_dir = setup_peer_msp_dir + '/admincerts' create_directory(dst_admincerts_dir) copyfile(org_admin_msp_dir + '/signcerts/cert.pem', '%s/%s-cert.pem' % (dst_admincerts_dir, admin['name']))
def init(conf, enrollmentAdmin): if 'peers' in conf: init_org(conf, enrollmentAdmin) if 'orderers' in conf: init_orderer(conf) create_directory(conf['broadcast_dir']['external']) generateGenesis(conf)
def substra_org(org, orderer=None): org_name = org['name'] print(f'Prepare Node : {org_name}') create_directory(f"{SUBSTRA_PATH}/data/orgs/{org_name}") create_directory(f"{SUBSTRA_PATH}/conf/{org_name}") # CA files create_ca_server_config(org) create_ca_client_config(org) # Configtx file config_filepath = org['misc']['configtx-config-path'] create_configtx(org, config_filepath) # Org Config files if 'peers' in org: create_peer_config(org) # create_fabric_ca_peer_config(org) # Docker-compose for org docker_compose = generate_docker_compose_org(org, orderer, SUBSTRA_PATH, SUBSTRA_NETWORK) intern_stop(docker_compose['path']) start(org, docker_compose) # Orderer Config files if 'orderers' in org: create_orderer_config(org) docker_compose = generate_docker_compose_orderer(org, SUBSTRA_PATH, SUBSTRA_NETWORK) intern_stop(docker_compose['path']) start(org, docker_compose)
def init(conf): if 'peers' in conf: init_org(conf) if 'orderers' in conf: init_orderer(conf) create_directory(conf['broadcast_dir']) generateGenesis(conf)
def init_orderer(conf): service = conf # remove ugly sample files defined here https://github.com/hyperledger/fabric/tree/master/sampleconfig # except orderer.yaml from binded volume # call('cd $FABRIC_CA_CLIENT_HOME && rm -rf msp core.yaml configtx.yaml', shell=True) for orderer in service['orderers']: setup_orderer_msp_dir = generateMSPandTLS(orderer, service) # copy the admincerts from the admin user for being able to launch orderer # https://stackoverflow.com/questions/48221810/what-is-difference-between-admincerts-and-signcerts-in-hyperledge-fabric-msp # https://lists.hyperledger.org/g/fabric/topic/17549225#1250 dst_admincerts_dir = setup_orderer_msp_dir + '/admincerts' create_directory(dst_admincerts_dir) copyfile('%s/signcerts/cert.pem' % setup_orderer_msp_dir, '%s/%s-cert.pem' % (dst_admincerts_dir, orderer['name']))
def generateMSPandTLS(node, service): enrollment_url = 'https://%(name)s:%(pass)s@%(host)s:%(port)s' % { 'name': node['name'], 'pass': node['pass'], 'host': service['ca']['host'], 'port': service['ca']['port']['internal'] } # create external folder tls_server_dir = node['tls']['dir']['external'] + '/' + node['tls'][ 'server']['dir'] tls_client_dir = node['tls']['dir']['external'] + '/' + node['tls'][ 'client']['dir'] create_directory(tls_server_dir) create_directory(tls_client_dir) # Generate server TLS cert and key pair in container genTLSCert(node['host'], '%s/%s' % (tls_server_dir, node['tls']['server']['cert']), '%s/%s' % (tls_server_dir, node['tls']['server']['key']), '%s/%s' % (tls_server_dir, node['tls']['server']['ca']), enrollment_url) # Generate client TLS cert and key pair for the peer CLI (will be used by external tools) # in a binded volume genTLSCert(node['name'], '%s/%s' % (tls_client_dir, node['tls']['client']['cert']), '%s/%s' % (tls_client_dir, node['tls']['client']['key']), '%s/%s' % (tls_client_dir, node['tls']['client']['ca']), enrollment_url) # Enroll the node to get an enrollment certificate and set up the core's local MSP directory for starting node setup_node_msp_dir = service['core_dir']['internal'] + '/' + node[ 'name'] + '/msp' call([ 'fabric-ca-client', 'enroll', '-d', '-u', enrollment_url, '-M', setup_node_msp_dir ]) return setup_node_msp_dir
def substra_network(orgs): # Stop all docker_compose_paths = glob.glob(os.path.join(SUBSTRA_PATH, 'dockerfiles/*.yaml')) # Remove all remove_all_docker() for docker_compose_path in docker_compose_paths: intern_stop(docker_compose_path) # Create Network call(['docker', 'network', 'create', SUBSTRA_NETWORK]) for orderer in [x for x in orgs if 'orderers' in x]: substra_org(orderer) else: # Prepare each org for org in [x for x in orgs if 'peers' in x]: substra_org(org, orderer) # substrabac create_directory(f"{SUBSTRA_PATH}/dryrun/{org['name']}") create_substrabac_config(org, orderer)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--period', type=str, default='all', help='specifies which period extract features from', choices=[ 'first4days', 'first8days', 'last12hours', 'first25percent', 'first50percent', 'all' ]) parser.add_argument('--features', type=str, default='all', help='specifies what features to extract', choices=['all', 'len', 'all_but_len']) parser.add_argument('--data', type=str, help='Path to the data of in-hospital mortality task', default=os.path.join( os.path.dirname(__file__), '../../../data/in-hospital-mortality/')) parser.add_argument( '--output_dir', type=str, help='Directory relative which all output files are stored', default='.') args = parser.parse_args() print(args) train_reader = InHospitalMortalityReader( dataset_dir=os.path.join(args.data, 'train'), listfile=os.path.join(args.data, 'train_listfile.csv'), period_length=48.0) val_reader = InHospitalMortalityReader( dataset_dir=os.path.join(args.data, 'train'), listfile=os.path.join(args.data, 'val_listfile.csv'), period_length=48.0) test_reader = InHospitalMortalityReader( dataset_dir=os.path.join(args.data, 'test'), listfile=os.path.join(args.data, 'test_listfile.csv'), period_length=48.0) #print("shape->",train_reader.read_example(100)['X'].shape) print('Reading data and extracting features ...') (train_X, train_y, train_names) = read_and_extract_features(train_reader, args.period, args.features) (val_X, val_y, val_names) = read_and_extract_features(val_reader, args.period, args.features) (test_X, test_y, test_names) = read_and_extract_features(test_reader, args.period, args.features) print(' train data shape = {}'.format(train_X.shape)) print(' validation data shape = {}'.format(val_X.shape)) print(' test data shape = {}'.format(test_X.shape)) #print("feature sample->", train_X[11]) print('Imputing missing values ...') imputer = Imputer(missing_values=np.nan, strategy='mean', axis=0, verbose=0, copy=True) imputer.fit(train_X) train_X = np.array(imputer.transform(train_X), dtype=np.float32) val_X = np.array(imputer.transform(val_X), dtype=np.float32) test_X = np.array(imputer.transform(test_X), dtype=np.float32) print('Normalizing the data to have zero mean and unit variance ...') scaler = StandardScaler() scaler.fit(train_X) train_X = scaler.transform(train_X) val_X = scaler.transform(val_X) test_X = scaler.transform(test_X) file_name = 'xgboost_{}.{}.'.format(args.period, args.features) xgreg = xgb.XGBRegressor(colsample_bytree=0.4, gamma=0, learning_rate=0.07, max_depth=3, min_child_weight=1.5, n_estimators=10000, reg_alpha=0.75, reg_lambda=0.45, subsample=0.6, seed=42) xgreg.fit(train_X, train_y) result_dir = os.path.join(args.output_dir, 'results') common_utils.create_directory(result_dir) with open(os.path.join(result_dir, 'train_{}.json'.format(file_name)), 'w') as res_file: ret = print_metrics_binary(train_y, xgreg.predict(train_X)) ret = {k: float(v) for k, v in ret.items()} json.dump(ret, res_file) with open(os.path.join(result_dir, 'val_{}.json'.format(file_name)), 'w') as res_file: ret = print_metrics_binary(val_y, xgreg.predict(val_X)) ret = {k: float(v) for k, v in ret.items()} json.dump(ret, res_file) prediction = xgreg.predict(test_X) with open(os.path.join(result_dir, 'test_{}.json'.format(file_name)), 'w') as res_file: ret = print_metrics_binary(test_y, prediction) ret = {k: float(v) for k, v in ret.items()} json.dump(ret, res_file) save_results( test_names, prediction, test_y, os.path.join(args.output_dir, 'predictions', file_name + '.csv'))
def save_results(names, ts, pred, y_true, path): common_utils.create_directory(os.path.dirname(path)) with open(path, 'w') as f: f.write("stay,period_length,prediction,y_true\n") for (name, t, x, y) in zip(names, ts, pred, y_true): f.write("{},{:.6f},{:.6f},{:.6f}\n".format(name, t, x, y))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--period', type=str, default='all', help='specifies which period extract features from', choices=[ 'first4days', 'first8days', 'last12hours', 'first25percent', 'first50percent', 'all' ]) parser.add_argument('--features', type=str, default='all', help='specifies what features to extract', choices=['all', 'len', 'all_but_len']) parser.add_argument('--data', type=str, help='Path to the data of in-hospital mortality task', default=os.path.join(os.path.dirname(__file__), '../../../data/phenotyping/')) parser.add_argument( '--output_dir', type=str, help='Directory relative which all output files are stored', default='.') args = parser.parse_args() print(args) train_reader = PhenotypingReader( dataset_dir=os.path.join(args.data, 'train'), listfile=os.path.join(args.data, 'train_listfile.csv')) val_reader = PhenotypingReader( dataset_dir=os.path.join(args.data, 'train'), listfile=os.path.join(args.data, 'val_listfile.csv')) test_reader = PhenotypingReader( dataset_dir=os.path.join(args.data, 'test'), listfile=os.path.join(args.data, 'test_listfile.csv')) #print("shape->",train_reader.read_example(100)['X'].shape) os.environ["CUDA_VISIBLE_DEVICES"] = "1" print('Reading data and extracting features ...') (train_X, train_y, train_names, train_ts) = read_and_extract_features(train_reader, args.period, args.features) train_y = np.array(train_y) (val_X, val_y, val_names, val_ts) = read_and_extract_features(val_reader, args.period, args.features) val_y = np.array(val_y) (test_X, test_y, test_names, test_ts) = read_and_extract_features(test_reader, args.period, args.features) test_y = np.array(test_y) print(' train data shape = {}'.format(train_X.shape)) print(' validation data shape = {}'.format(val_X.shape)) print(' test data shape = {}'.format(test_X.shape)) #print("feature sample->", train_X[11]) print('Imputing missing values ...') imputer = Imputer(missing_values=np.nan, strategy='mean', axis=0, verbose=0, copy=True) imputer.fit(train_X) train_X = np.array(imputer.transform(train_X), dtype=np.float32) val_X = np.array(imputer.transform(val_X), dtype=np.float32) test_X = np.array(imputer.transform(test_X), dtype=np.float32) print('Normalizing the data to have zero mean and unit variance ...') scaler = StandardScaler() scaler.fit(train_X) train_X = scaler.transform(train_X) val_X = scaler.transform(val_X) test_X = scaler.transform(test_X) file_name = 'xgboost_{}.{}.'.format(args.period, args.features) result_dir = os.path.join(args.output_dir, 'results') common_utils.create_directory(result_dir) n_tasks = 25 train_activations = np.zeros(shape=train_y.shape, dtype=float) val_activations = np.zeros(shape=val_y.shape, dtype=float) test_activations = np.zeros(shape=test_y.shape, dtype=float) for task_id in range(n_tasks): print('Starting task {}'.format(task_id)) xgreg = lgb.LGBMRegressor(objective='regression', num_leaves=11, learning_rate=0.07, n_estimators=10000) eval_set = [(train_X, train_y[:, task_id]), (val_X, val_y[:, task_id])] xgreg.fit(train_X, train_y[:, task_id], eval_metric='auc', eval_set=eval_set, verbose=True, early_stopping_rounds=10) train_preds = xgreg.predict(train_X) print("train_preds shape ", train_preds.shape) print("train_activations shape", train_activations.shape) train_activations[:, task_id] = train_preds val_preds = xgreg.predict(val_X) val_activations[:, task_id] = val_preds time_start = time.time() test_preds = xgreg.predict(test_X) time_elapse = time.time() - time_start print("Processing time on Test set :", time_elapse, " s for task ", task_id) test_activations[:, task_id] = test_preds with open(os.path.join(result_dir, 'train_{}.json'.format(file_name)), 'w') as f: ret = print_metrics_multilabel(train_y, train_activations) ret = {k: float(v) for k, v in ret.items() if k != 'auc_scores'} json.dump(ret, f) with open(os.path.join(result_dir, 'val_{}.json'.format(file_name)), 'w') as f: ret = print_metrics_multilabel(val_y, val_activations) ret = {k: float(v) for k, v in ret.items() if k != 'auc_scores'} json.dump(ret, f) with open(os.path.join(result_dir, 'test_{}.json'.format(file_name)), 'w') as f: ret = print_metrics_multilabel(test_y, test_activations) ret = {k: float(v) for k, v in ret.items() if k != 'auc_scores'} json.dump(ret, f) save_results( test_names, test_ts, test_activations, test_y, os.path.join(args.output_dir, 'predictions', file_name + '.csv'))
xgreg = lgb.LGBMRegressor(objective='regression', num_leaves=11, learning_rate=0.07, n_estimators=10000) eval_set = [(train_raw_reshape, train_raw[1]), (val_raw_reshape, val_raw[1])] xgreg.fit(train_raw_reshape, train_raw[1], eval_metric='auc,auroc', eval_set=eval_set, verbose=True, early_stopping_rounds=80) result_dir = os.path.join(args.output_dir, 'results') common_utils.create_directory(result_dir) with open(os.path.join(result_dir, 'train_{}.json'.format(file_name)), 'w') as res_file: ret = print_metrics_binary(train_raw[1], xgreg.predict(train_raw_reshape)) ret = {k: float(v) for k, v in ret.items()} json.dump(ret, res_file) with open(os.path.join(result_dir, 'val_{}.json'.format(file_name)), 'w') as res_file: ret = print_metrics_binary(val_raw[1], xgreg.predict(val_raw_reshape)) ret = {k: float(v) for k, v in ret.items()} json.dump(ret, res_file) time_start = time.time() prediction = xgreg.predict(test_raw_reshape)
parser.add_argument('--no-backup', action='store_true', default=False, help="Remove backup binded volume. Launch from scratch") args = vars(parser.parse_args()) # Stop all docker remove_all_docker() if args['no_backup']: # create directory with correct rights call(['rm', '-rf', f'{SUBSTRA_PATH}/data']) call(['rm', '-rf', f'{SUBSTRA_PATH}/conf']) call(['rm', '-rf', f'{SUBSTRA_PATH}/backup']) call(['rm', '-rf', f'{SUBSTRA_PATH}/dryrun']) call(['rm', '-rf', f'{SUBSTRA_PATH}/dockerfiles']) create_directory(f'{SUBSTRA_PATH}/data/log') create_directory(f'{SUBSTRA_PATH}/conf/') create_directory(f'{SUBSTRA_PATH}/conf/config') create_directory(f'{SUBSTRA_PATH}/dryrun/') create_directory(f'{SUBSTRA_PATH}/dockerfiles/') if not glob.glob(f'{SUBSTRA_PATH}/conf/config/conf-*.json'): # Global configuration if args['config']: call(['python3', args['config']]) else: call(['python3', os.path.join(dir_path, 'conf/2orgs.py')]) files = glob.glob(f'{SUBSTRA_PATH}/conf/config/conf-*.json') files.sort(key=os.path.getmtime) orgs = [json.load(open(file_path, 'r')) for file_path in files]
def main(): parser = argparse.ArgumentParser() parser.add_argument('--C', type=float, default=1.0, help='inverse of L1 / L2 regularization') parser.add_argument('--l1', dest='l2', action='store_false') parser.add_argument('--l2', dest='l2', action='store_true') parser.set_defaults(l2=True) parser.add_argument('--period', type=str, default='all', help='specifies which period extract features from', choices=[ 'first4days', 'first8days', 'last12hours', 'first25percent', 'first50percent', 'all' ]) parser.add_argument('--features', type=str, default='all', help='specifies what features to extract', choices=['all', 'len', 'all_but_len']) parser.add_argument('--data', type=str, help='Path to the data of in-hospital mortality task', default=os.path.join( os.path.dirname(__file__), '../../../data/in-hospital-mortality/')) parser.add_argument( '--output_dir', type=str, help='Directory relative which all output files are stored', default='.') args = parser.parse_args() print(args) train_reader = InHospitalMortalityReader( dataset_dir=os.path.join(args.data, 'train'), listfile=os.path.join(args.data, 'train_listfile.csv'), period_length=48.0) val_reader = InHospitalMortalityReader( dataset_dir=os.path.join(args.data, 'train'), listfile=os.path.join(args.data, 'val_listfile.csv'), period_length=48.0) test_reader = InHospitalMortalityReader( dataset_dir=os.path.join(args.data, 'test'), listfile=os.path.join(args.data, 'test_listfile.csv'), period_length=48.0) print('Reading data and extracting features ...') (train_X, train_y, train_names) = read_and_extract_features(train_reader, args.period, args.features) (val_X, val_y, val_names) = read_and_extract_features(val_reader, args.period, args.features) (test_X, test_y, test_names) = read_and_extract_features(test_reader, args.period, args.features) print(' train data shape = {}'.format(train_X.shape)) print(' validation data shape = {}'.format(val_X.shape)) print(' test data shape = {}'.format(test_X.shape)) print('Imputing missing values ...') imputer = Imputer(missing_values=np.nan, strategy='mean', axis=0, verbose=0, copy=True) imputer.fit(train_X) train_X = np.array(imputer.transform(train_X), dtype=np.float32) val_X = np.array(imputer.transform(val_X), dtype=np.float32) test_X = np.array(imputer.transform(test_X), dtype=np.float32) print('Normalizing the data to have zero mean and unit variance ...') scaler = StandardScaler() scaler.fit(train_X) train_X = scaler.transform(train_X) val_X = scaler.transform(val_X) test_X = scaler.transform(test_X) penalty = ('l2' if args.l2 else 'l1') file_name = '{}.{}.{}.C{}'.format(args.period, args.features, penalty, args.C) logreg = LogisticRegression(penalty=penalty, C=args.C, random_state=42) logreg.fit(train_X, train_y) result_dir = os.path.join(args.output_dir, 'results') common_utils.create_directory(result_dir) with open(os.path.join(result_dir, 'train_{}.json'.format(file_name)), 'w') as res_file: ret = print_metrics_binary(train_y, logreg.predict_proba(train_X)) ret = {k: float(v) for k, v in ret.items()} json.dump(ret, res_file) with open(os.path.join(result_dir, 'val_{}.json'.format(file_name)), 'w') as res_file: ret = print_metrics_binary(val_y, logreg.predict_proba(val_X)) ret = {k: float(v) for k, v in ret.items()} json.dump(ret, res_file) time_start = time.time() prediction = logreg.predict_proba(test_X)[:, 1] time_elapse = time.time() - time_start print("Processing time on Test set :", time_elapse, " s") with open(os.path.join(result_dir, 'test_{}.json'.format(file_name)), 'w') as res_file: ret = print_metrics_binary(test_y, prediction) ret = {k: float(v) for k, v in ret.items()} json.dump(ret, res_file) save_results( test_names, prediction, test_y, os.path.join(args.output_dir, 'predictions', file_name + '.csv'))