def load_env(args): env = 'dev' if len(args) > 1: arg = args[1].split('=') if len(arg) > 1 and arg[0] == '--active' and arg[1] == 'prod': env = arg[1] logger.info('当前环境为: ' + env) config.load_config(env)
def get_repository(config, name): if config is None: config = load_config() section_name = repo_section_name(name) if not config.has_section(section_name): raise ValueError("No repository found: " + name) return config[section_name]
def main(config_filepath): config = load_config(config_filepath) if os.path.isfile(config.x_train_output_path): click.confirm(f"Overwrite {config.x_train_output_path}?", abort=True) if os.path.isfile(config.y_train_output_path): click.confirm(f"Overwrite {config.y_train_output_path}?", abort=True) if os.path.isfile(config.x_test_output_path): click.confirm(f"Overwrite {config.x_test_output_path}?", abort=True) if os.path.isfile(config.y_test_output_path): click.confirm(f"Overwrite {config.y_test_output_path}?", abort=True) if os.path.isfile(config.covariance_output_path): click.confirm(f"Overwrite {config.covariance_output_path}?", abort=True) if os.path.isfile(config.means_output_path): click.confirm(f"Overwrite {config.means_output_path}?", abort=True) if os.path.isfile(config.clip_values_output_path): click.confirm(f"Overwrite {config.clip_values_output_path}?", abort=True) np.random.seed(config.seed) torch.manual_seed(config.seed) # Load data ( (x_train, y_train), (x_test, y_test), min_pixel_value, max_pixel_value, ) = load_mnist() clip_values = { "min_pixel_value": min_pixel_value, "max_pixel_value": max_pixel_value, } # Swap axes to PyTorch's NCHW format x_train = np.swapaxes(x_train, 1, 3).astype(np.float32) x_test = np.swapaxes(x_test, 1, 3).astype(np.float32) x_obvs = torch.Tensor(x_train) means = x_obvs.mean(dim=0) means = means.unsqueeze(dim=0) x_obvs = x_obvs.reshape( (-1, 1, 28 * 28) ) # flatten so we can get a real covariance matrix covariance = torch.Tensor(np.cov(x_obvs[:, 0, :], rowvar=False)) # Save data torch.save(x_train, config.x_train_output_path) torch.save(y_train, config.y_train_output_path) torch.save(x_test, config.x_test_output_path) torch.save(y_test, config.y_test_output_path) torch.save(covariance, config.covariance_output_path) torch.save(means, config.means_output_path) with open(config.clip_values_output_path, mode="w") as f: json.dump(clip_values, f)
def load_srv_config(): """ read configs from config file, and simulate a static config class """ log = logging.getLogger('server') config_filepath = "srv_config.conf" dic = load_config(log, os.path.abspath(config_filepath)) for k,v in dic.items(): _dict[k] = v
def load_client_config(): """ read configs from config file, and simulate a static config class """ log = logging.getLogger('client') config_filepath = os.path.abspath("../client/client_config.conf") dic = load_config(log, config_filepath) for k, v in dic.items(): _dict[k] = v
def install_update(self): if not self._update_ready or self._status == UPDATER_STATUS_INSTALLING: return False self._status = UPDATER_STATUS_INSTALLING self.emit_status() logger.info('Installing update') try: assert self._update_file_path and isfile(self._update_file_path) logger.debug("self._update_file_path %s", self._update_file_path) path, name = split(self._update_file_path) old_cwd = os.getcwd() os.chdir(path) system = get_platform() if system == 'Windows': from common.config import load_config config = load_config() root = config.sync_directory log_basename = time.strftime('%Y%m%d_%H%M%S.log') log_filename = get_bases_filename(root, log_basename) if not self._is_ascii(log_filename): log_filename = log_basename args = [name, '/verysilent', '/Log={}'.format(log_filename)] if is_portable(): args.append('/PATH={}'.format(get_application_path())) subprocess.Popen( args, creationflags=0x00000200 # CREATE_NEW_PROCESS_GROUP | 0x00000008, # DETACHED_PROCESS close_fds=True) elif system == 'Darwin': bundle_path = normpath( join(get_application_path(), '..', '..', '..', '..')) logger.debug("bundle_path: %s", bundle_path) subprocess.call( ['ditto', '-xk', self._update_file_path, bundle_path]) subprocess.call( ['xattr', '-d', '-r', 'com.apple.quarantine', bundle_path]) logger.debug("Update completed, restart") remove_file(get_cfg_filename('lock')) if is_portable(): launcher_path = normpath( join(bundle_path, "..", "Pvtbox-Mac.command")) else: launcher_path = bundle_path subprocess.call(['open', launcher_path]) os.chdir(old_cwd) Application.exit() except Exception as e: logger.warning("Can't install update. Reason: %s", e) self._status = UPDATER_STATUS_INSTALL_ERROR self.emit_status() return False self._status = UPDATER_STATUS_INSTALLED self.emit_status() return True
def delete_repository(name): """ Deletes a configured Maven repository. """ config = load_config() section_name = repo_section_name(name) if config.has_section(section_name): config.remove_section(section_name) save_config(config)
def view_stagedir(): """ Views the output directory where mvnfeed will be staged. """ config = load_config() if "general" not in config: return "" return config.get("general", STAGE_DIR_CONFIGNAME)
def main(config_filepath): config = load_config(config_filepath) if os.path.isfile(config.metrics_output_path): click.confirm(f"Overwrite {config.metrics_output_path}?", abort=True) np.random.seed(config.seed) torch.manual_seed(config.seed) # Load data x = torch.load(config.x_filepath) y = torch.load(config.y_filepath) # Flatten x = x.reshape(x.shape[0], -1) model = torch.load(config.trained_model_filepath) clip_values = {} with open(config.clip_values_filepath, "r") as f: clip_values = json.load(f) clip_values = ( clip_values.get("min_pixel_value"), clip_values.get("max_pixel_value"), ) classifier = PyTorchClassifier( model=model, clip_values=clip_values, loss=model.criterion, optimizer=model.optimizer, input_shape=(1, 28, 28), nb_classes=10, ) # TODO: move these parameters to config # Evaluate the classifier on benign data predictions = classifier.predict(x) # Convert one-hots to numbers for metrics y = utils.one_hot_to_num(y) predictions = utils.one_hot_to_num(predictions) accuracy = { "Accuracy": metrics.accuracy_score(y, predictions), "Confusion Matrix": metrics.confusion_matrix(y, predictions).tolist(), } # Save data with open(config.metrics_output_path, "w") as f: json.dump( accuracy, f, ensure_ascii=False, sort_keys=True, indent=4, separators=(",", ": "), )
def get_stagedir(config): """ Returns the value of the stage directory. """ if config is None: config = load_config() if "general" not in config: return None return config.get("general", STAGE_DIR_CONFIGNAME)
def list_repositories(): """ Lists the configured Maven repositories. """ config = load_config() for section in config.sections(): if section.startswith(REPOSITORY): repo = config[section] print(section[11:]) if URL in repo: print(" url : " + repo[URL])
def main(config_filepath): config = load_config(config_filepath) if os.path.isfile(config.output_path): click.confirm(f"Overwrite {config.output_path}?", abort=True) metrics = {} with open(config.metrics_filepath, "r") as f: metrics = json.load(f) accuracy = metrics.get("Accuracy") misclass = 1 - accuracy cm = np.array(metrics.get("Confusion Matrix")) cmap = plt.get_cmap("Blues") plt.figure(figsize=(8, 6)) plt.imshow(cm, interpolation="nearest", cmap=cmap) plt.title(config.title) plt.colorbar() target_names = [str(t) for t in range(10)] tick_marks = np.arange(len(target_names)) plt.xticks(tick_marks, target_names, rotation=45) plt.yticks(tick_marks, target_names) if config.normalize: cm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis] thresh = cm.max() / 1.5 if config.normalize else cm.max() / 2 for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): if config.normalize: plt.text( j, i, f"{cm[i, j]:0.4f}", horizontalalignment="center", color="white" if cm[i, j] > thresh else "black", ) else: plt.text( j, i, f"{cm[i, j]:,}", horizontalalignment="center", color="white" if cm[i, j] > thresh else "black", ) # plt.tight_layout() plt.ylabel("True label") plt.xlabel( f"Predicted label\naccuracy={accuracy:0.4f}; misclass={misclass:0.4f}") plt.savefig(config.output_path)
def main(config_filepath): config = load_config(config_filepath) if os.path.isfile(config.x_adv_output_path): click.confirm(f"Overwrite {config.x_adv_output_path}?", abort=True) np.random.seed(config.seed) torch.manual_seed(config.seed) # Load data x = torch.load(config.x_filepath) x_shape = x.shape x = x.reshape(x.shape[0], -1) y = torch.load(config.y_filepath) ace = torch.load(config.ace_filepath).cpu() interventions = torch.load(config.interventions_filepath).cpu() model = torch.load(config.model_filepath) clip_values = {} with open(config.clip_values_filepath, "r") as f: clip_values = json.load(f) clip_values = ( clip_values.get("min_pixel_value"), clip_values.get("max_pixel_value"), ) classifier = PyTorchClassifier( model=model, clip_values=clip_values, loss=model.criterion, optimizer=model.optimizer, input_shape=(1, 28, 28), nb_classes=10, ) # TODO: move these parameters to config # Target the attacks to a particular class y_adv = torch.zeros_like(torch.from_numpy(y)) y_adv[:, 0] = 1.0 # Generate attacks x_adv = ace_attack( ace, interventions, torch.from_numpy(x), target_classes=y_adv, norm=2, budget=5.0, ) # TODO: move these parameters to config # Unflatten for saving x_adv = x_adv.reshape(x_shape) torch.save(x_adv, config.x_adv_output_path)
def set_stagedir(path): """ Sets the output directory where dependencies will be staged. :param path: path where the downloaded mvnfeed will be staged """ config = load_config() if "general" not in config: config.add_section("general") config.set("general", STAGE_DIR_CONFIGNAME, path) save_config(config)
def main(config_filepath): config = load_config(config_filepath) if os.path.isfile(config.interventional_expectations_output_path): click.confirm( f"Overwrite {config.interventional_expectations_output_path}?", abort=True) if os.path.isfile(config.average_causal_effects_output_path): click.confirm( f"Overwrite {config.average_causal_effects_output_path}?", abort=True) if os.path.isfile(config.interventions_output_path): click.confirm(f"Overwrite {config.interventions_output_path}?", abort=True) model = torch.load(config.model_filepath) cov = torch.load(config.covariance_filepath) mean = torch.load(config.means_filepath) # Flatten the mean mean = mean.reshape(1, -1) clip_values = {} with open(config.clip_values_filepath, "r") as f: clip_values = json.load(f) clip_values = ( clip_values.get("min_pixel_value"), clip_values.get("max_pixel_value"), ) num_alphas = 10 interventions = torch.Tensor(np.linspace(*clip_values, num_alphas)) ie = ace.interventional_expectation( model, mean, cov, interventions, epsilon=0.000001, method="hessian_diag", progress=True, ) avg_ce = ace.average_causal_effect(ie) # Remove names for now since named tensors aren't serializable # Also detach grads so they are not saved. ie = ie.rename(None).detach() avg_ce = avg_ce.rename(None).detach() torch.save(ie, config.interventional_expectations_output_path) torch.save(avg_ce, config.average_causal_effects_output_path) torch.save(torch.Tensor(interventions), config.interventions_output_path)
def check_sync_folder_removed(): from application.app_config import load_config as load_main_config from common.config import load_config from common.file_path import FilePath from common.utils import get_bases_dir main_cfg = load_main_config(check=False) if not main_cfg.get_setting('user_email'): return False config = load_config() root = FilePath(config.sync_directory).longpath return not isdir(root) or not isdir(get_bases_dir(root))
def app(ctx): """ Use the EC-instance connect tools, mssh and msftp, with instance names instead of instance ids with the commands ec2 ssh and ec2 sftp. Boto3 is used to retrieve instance ids and correct os user given instance names. This only works if your instances have unique names, not containing whitespace. """ # Entry point for CLI. # We just want some object for the context, to carry variables. ctx.obj = EmptyObj() public_key = Path.home() / ".ssh/id_rsa.pub" if not public_key.exists(): public_key = None click.echo( "Rsa key not found. Please generate one with: ssh-keygen -t rsa -f ~/.ssh/id_rsa. Some commands will be unavailable." ) else: public_key = public_key.read_text() ctx.obj.public_key = public_key # Load custom config. try: ctx.obj.cfg, ctx.obj.config_file = load_config( Path(click.get_app_dir(APP_NAME))) except json.decoder.JSONDecodeError: raise click.ClickException( f"Your configuration file is invalid JSON. Please fix: {Path(click.get_app_dir(APP_NAME))}" ) # Confirm AWS credentials. ctx.obj.ec2 = boto3.client("ec2") ctx.obj.ec2ic = boto3.client("ec2-instance-connect") try: _ = ctx.obj.ec2.describe_instances(DryRun=True) except ClientError as e: if "DryRunOperation" not in str(e): raise click.ClickException( "Failure. Your AWS credentials do not authorize: describe_instances" ) try: _ = ctx.obj.ec2.describe_images(DryRun=True) except ClientError as e: if "DryRunOperation" not in str(e): raise click.ClickException( "Failure. Your AWS credentials do not authorize: describe_images" )
def download_artifact(name, repo_name, download_deps=False): """ Downloads a single artifact. :param name: name of the artifact to download, following the group_id:artifact_id:version format :param repo_name: name of the repository to look for artifacts :param download_deps: True if the dependencies must be downloaded """ logging.info("downloading %s", name) config = load_config() repository = get_repository(config, repo_name) stage_dir = get_stagedir(config) _download_single_artifact(name, repository, stage_dir, download_deps)
def start(self, app_start_ts, args): ''' Performs application initialization and launch @raise SystemExit ''' logger.debug("Launching service with args (%s)...", args) clear_old_logs(logger) # Load configuration file self._cfg = config.load_config() set_max_log_size_mb(logger, max(self._cfg.max_log_size, 0.02)) if self._cfg.copies_logging: copies_logger = logging.getLogger('copies_logger') set_max_log_size_mb(copies_logger, max(self._cfg.max_log_size, 0.02)) if 'sync_directory' in args and args['sync_directory']: self._cfg.set_settings( {'sync_directory': args['sync_directory'].decode('utf-8')}) if 'wipe_internal' in args and args['wipe_internal']: try: wipe_internal(self._cfg.sync_directory) except Exception as e: logger.warning("Can't wipe internal info. Reason: %s", e) raise SystemExit(0) if self._cfg.tracking_address: self._tracker = Tracker('service_stats.db', self._cfg.sync_directory, self._cfg.tracking_address) init_crash_handler(self._tracker) self._tracker_thread = QThread() self._tracker.moveToThread(self._tracker_thread) self._tracker_thread.started.connect(self._tracker.start.emit) self._tracker_thread.start(QThread.IdlePriority) else: init_crash_handler(logger=logger) self._worker = ApplicationWorker(self._cfg, self._tracker, app_start_ts, args) self._worker.exited.connect(self._on_exit, Qt.QueuedConnection) self._worker.start_work() self.exec_() logger.debug("Service exiting...")
def main(config_filepath): config = load_config(config_filepath) if os.path.isfile(config.x_adv_output_path): click.confirm(f"Overwrite {config.x_adv_output_path}?", abort=True) seed = 45616451 np.random.seed(seed) torch.manual_seed(seed) # Load data x = torch.load(config.x_filepath) x_shape = x.shape y = torch.load(config.y_filepath) # Flatten test set x = x.reshape(x.shape[0], -1) model = torch.load(config.model_filepath) clip_values = {} with open(config.clip_values_filepath, "r") as f: clip_values = json.load(f) clip_values = ( clip_values.get("min_pixel_value"), clip_values.get("max_pixel_value"), ) classifier = PyTorchClassifier( model=model, clip_values=clip_values, loss=model.criterion, optimizer=model.optimizer, input_shape=(1, 28, 28), nb_classes=10, ) # TODO: move these parameters to config # Generate attacks attack = FastGradientMethod( classifier=classifier, eps=0.2) # TODO: move these parameters to config x_adv = attack.generate(x=x) # Reshape adversarial examples back to original test data shape x_adv = torch.from_numpy(x_adv.reshape(x_shape)) torch.save(x_adv, config.x_adv_output_path)
def main(config_filepath): config = load_config(config_filepath) if (os.path.isfile(config.x_train_output_path) or os.path.isfile(config.y_train_output_path) or os.path.isfile(config.covariance_output_path) or os.path.isfile(config.means_output_path)): click.confirm(f"Overwrite files?", abort=True) np.random.seed(config.seed) torch.manual_seed(config.seed) dataset = pd.read_csv(config.input_filepath) dataset = pd.get_dummies(dataset, columns=["species"]) # One Hot Encoding values = list(dataset.columns.values) y = dataset[values[-3:]] y = np.array(y, dtype="float32") x = dataset[values[:-3]] x = np.array(x, dtype="float32") # Shuffle Data indices = np.random.choice(len(x), len(x), replace=False) x_values = x[indices] scaler = MinMaxScaler() test_size = 30 x_train = x_values[:-test_size] x_train = scaler.fit_transform(x_train) x_values = scaler.transform(x_values) y_values = y[indices] x_tensor = torch.Tensor(x_values) y_tensor = torch.Tensor(y_values) # Calculate observational statistics covariance = torch.Tensor(np.cov(x_values, rowvar=False)) means = torch.Tensor(np.mean(x_values, axis=0)) # Save data torch.save(x_tensor, config.x_train_output_path) torch.save(y_tensor, config.y_train_output_path) torch.save(covariance, config.covariance_output_path) torch.save(means, config.means_output_path)
def main(config_filepath): config = load_config(config_filepath) if os.path.isfile(config.model_output_path): click.confirm(f"Overwrite {config.model_output_path}?", abort=True) np.random.seed(config.seed) torch.manual_seed(config.seed) # Load data x = torch.load(config.x_filepath) y = torch.load(config.y_filepath) # Flatten training set x = x.reshape(x.shape[0], -1) clip_values = {} with open(config.clip_values_filepath, "r") as f: clip_values = json.load(f) clip_values = ( clip_values.get("min_pixel_value"), clip_values.get("max_pixel_value"), ) model = get_model_from_module(mnist.models, config.model_class_name) if not model: sys.exit(f"Could not load provided model {config.model_class_name}") classifier = PyTorchClassifier( model=model, clip_values=clip_values, loss=model.criterion, optimizer=model.optimizer, input_shape=(784), nb_classes=10, ) # TODO: move these parameters to config # Train classifier classifier.fit(x, y, batch_size=config.batch_size, nb_epochs=config.num_epochs) # Save data torch.save(model, config.model_output_path)
def main(config_filepath): config = load_config(config_filepath) if os.path.isfile(config.metrics_output_path): click.confirm(f"Overwrite {config.metrics_output_path}?", abort=True) ace = torch.load(config.ace_filepath).cpu().numpy() num_monotonic = 0 num_nonmonotonic = 0 extrema_dict = {} for x in range(ace.shape[0]): for y in range(ace.shape[1]): alphas = ace[x, y, :] diff = np.diff(alphas) if np.all(diff >= 0) or np.all(diff <= 0): num_monotonic += 1 else: num_nonmonotonic += 1 maxima = argrelextrema(alphas, np.greater)[0] minima = argrelextrema(alphas, np.less)[0] extrema_count = len(maxima) + len(minima) if extrema_count in extrema_dict.keys(): extrema_dict[extrema_count] += 1 else: extrema_dict[extrema_count] = 1 metrics = { "Monotonic": num_monotonic, "Non-monotonic": num_nonmonotonic, "Extrema": extrema_dict, } with open(config.metrics_output_path, "w") as f: json.dump( metrics, f, ensure_ascii=False, sort_keys=True, indent=4, separators=(",", ": "), )
def download_bulk(filename, repo_name, download_deps=False): """ Downloads artifacts from a file, one artifact per line. :param filename: name of the file containing the artifacts to download :param repo_name: name of the repository to look for artifacts :param download_deps: True if the dependencies must be downloaded """ logging.info("downloading from file %s", filename) config = load_config() repository = get_repository(config, repo_name) stage_dir = get_stagedir(config) with open(filename, "r") as file: lines = file.readlines() for line in lines: line = line.strip().rstrip() if line: _download_single_artifact(line, repository, stage_dir, download_deps)
def add_repository(name, username, url=None): """ Adds an external Maven repository. :param name: internal name of the repository :param username: name of the user for the basic authentication to the repository :param url: url of the repository """ if username is None: raise ValueError("Username must be defined") if url is None: raise ValueError("Url must be defined") password = getpass.getpass() encoded = base64.b64encode((username + ":" + password).encode("utf-8")) authorization = "Basic " + encoded.decode("utf-8") config = load_config() config[repo_section_name(name)] = { URL: _default_value(url), AUTHORIZATION: _default_value(authorization), } save_config(config)
def test_find_equivalent_variant_whole_seq(fetch_seq_mock_data): with patch.object(bioutils.seqfetcher, 'fetch_seq', side_effect=lambda ac, s, e: fetch_seq_mock_data[ (str(ac), str(s), str(e))]): gene_config_path = os.path.join(pwd, 'test_files', 'gene_config_test.txt') cfg = load_config(gene_config_path) regions = extract_gene_regions_dict(cfg, 'start_hg38_legacy_variants', 'end_hg38_legacy_variants').keys() seq_wrapper = seq_utils.SeqRepoWrapper(regions_preload=regions) # empty case assert [] == find_equivalent_variants_whole_seq({}, seq_wrapper) # a bunch of variants. If they appear in the same set, they are considered equivalent example_variants = [ frozenset({'chr13:g.32355030:A>AA'}), frozenset({'chr13:g.32339774:GAT>G', 'chr13:g.32339776:TAT>T'}), frozenset({'chr17:g.43090921:G>GCA', 'chr17:g.43090921:GCA>GCACA'}) ] # construct variant dict (flattening example_variants!) variant_dict = { v: VCFVariant(int(v.split(':')[0].lstrip('chr')), int(v.split(':')[1].lstrip('g.')), v.split(':')[2].split('>')[0], v.split(':')[2].split('>')[1]) for eq_variants in example_variants for v in eq_variants } whole_seq_provider = seq_utils.WholeSeqSeqProvider(seq_wrapper) assert frozenset(example_variants) == frozenset( find_equivalent_variants_whole_seq(variant_dict, whole_seq_provider))
def main(config_filepath): config = load_config(config_filepath) if ( os.path.isfile(config.interventional_expectations_output_path) or os.path.isfile(config.average_causal_effects_output_path) or os.path.isfile(config.interventions_output_path) ): click.confirm(f"Overwrite files?", abort=True) model = torch.load(config.model_filepath) wrapped_model = wrap_model_activation( model ) # Iris uses a softmax activation, but I didn't want to build that assumption into the ace calculation. cov = torch.load(config.covariance_filepath) mean = torch.load(config.means_filepath) interventions = torch.Tensor(np.linspace(0, 1, 1000)) ie = ace.interventional_expectation( wrapped_model, mean, cov, interventions, epsilon=config.epsilon, method=config.method, progress=config.show_progress, ) avg_ce = ace.average_causal_effect(ie) # Also detach grads so they are not saved. ie = ie.detach() avg_ce = avg_ce.detach() torch.save(ie, config.interventional_expectations_output_path) torch.save(avg_ce, config.average_causal_effects_output_path) torch.save(torch.Tensor(interventions), config.interventions_output_path)
""" database seeding for init db data """ import psycopg2 from common.config import load_config CONF = load_config() CONN = psycopg2.connect(host=CONF["postgres"]["host"], port=CONF["postgres"]["port"], dbname=CONF["postgres"]["db"], user=CONF["postgres"]["user"], password=CONF["postgres"]["password"]) CURSOR = CONN.cursor() with open("../bin/db_seed/addresses.csv", "r", encoding="utf-8") as f: CURSOR.copy_from(f, "addresses", sep=",") CONN.commit() with open("../bin/db_seed/users.csv", "r", encoding="utf-8") as f: CURSOR.copy_from(f, "users", sep=",") CONN.commit() with open("../bin/db_seed/schools.csv", "r", encoding="utf-8") as f: CURSOR.copy_from(f, "schools", sep=",") CONN.commit() with open("../bin/db_seed/students.csv", "r", encoding="utf-8") as f: CURSOR.copy_from(f, "students", sep=",") CONN.commit()
from database import db, SQLALCHEMY_DATABASE_URI def create_app(): app = Flask(__name__) app.config.update(SQLALCHEMY_DATABASE_URI=SQLALCHEMY_DATABASE_URI) CORS(app) app.config.from_object(Config) db.init_app(app) Migrate(app, db) from models import user, student, school, address, transfer, attendance, donation, extrafund app.register_blueprint(api.api) return app conf = load_config() load_logging_conf(conf["common"]["log"]["conf"]) app = create_app() jwt = JWTManager(app) api.jwt = jwt if __name__ == "__main__": try: app.run(host=conf["common"]["server"]["host"], port=conf["common"]["server"]["port"], threaded=True, debug=True) finally: pass
import ast import glob import os import re from pathlib import Path from time import time from termcolor import colored from sympy import factor, Interval ## Importing my code from common.config import load_config from common.convert import parse_numbers, parse_interval_bounds from common.files import pickle_load from common.my_storm import parse_2D_refinement_into_space, merge_2D_refinements, merge_multidim_refinements spam = load_config() data_path = spam["data"] results_dir = spam["results"] prism_results = spam["prism_results"] storm_results = spam["storm_results"] del spam ############################### ### RATIONAL FUNCTIONS ### ############################### def load_mc_result(file_path, tool="unknown", factorize=True, rewards_only=False,
# 删除机器目录 def _delete_file(ip, target_dir): logger.info('开始删除目录: ' + ip) del_str = 'ls' if target_dir and '/home/centos/' in target_dir: del_str = 'rm -rf ' + target_dir # 删除文件 del_cmd = str.format('ssh {} {}@{} sudo {}', conf().IDENTITY, conf().USER_NAME, ip, del_str) res_del = shell_util.exec_cmd(del_cmd) logger.info('删除: ' + str(res_del)) if __name__ == '__main__': config.load_config('dev') name = 'TTANode1' p2p_str = 'fd69dfa8baa7cb0dc91a10cdc1eac00506d05b5e@TTANode3:26656' genesis_dict = { 'genesis_time': '2019-12-21T08:15:49.6296137Z', 'chain_id': 'test-chain-hVOXVd', 'consensus_params': { 'block': { 'max_bytes': '22020096', 'max_gas': '-1', 'time_iota_ms': '1000' }, 'evidence': { 'max_age': '100000'
def setUp(self): self.columns = [ 'Source', 'Gene_symbol_ENIGMA', 'Genomic_Coordinate', 'Chr', 'Pos', 'Ref', 'Alt', 'Reference_sequence_ENIGMA', 'HGVS_cDNA_ENIGMA', 'BIC_Nomenclature_ENIGMA', 'Abbrev_AA_change_ENIGMA', 'URL_ENIGMA', 'Condition_ID_type_ENIGMA', 'Condition_ID_value_ENIGMA', 'Condition_category_ENIGMA', 'Clinical_significance_ENIGMA', 'Date_last_evaluated_ENIGMA', 'Assertion_method_ENIGMA', 'Assertion_method_citation_ENIGMA', 'Clinical_significance_citations_ENIGMA', 'Comment_on_clinical_significance_ENIGMA', 'Collection_method_ENIGMA', 'Allele_origin_ENIGMA', 'ClinVarAccession_ENIGMA', 'HGVS_protein_ENIGMA', 'BX_ID_ENIGMA', 'Clinical_Significance_ClinVar', 'Date_Last_Updated_ClinVar', 'BX_ID_ClinVar', 'HGVS_ClinVar', 'Submitter_ClinVar', 'Protein_ClinVar', 'SCV_ClinVar', 'Allele_Origin_ClinVar', 'Method_ClinVar', 'Description_ClinVar', 'Summary_Evidence_ClinVar', 'Review_Status_ClinVar', 'Condition_Type_ClinVar', 'Condition_Value_ClinVar', 'Condition_DB_ID_ClinVar', 'Individuals_LOVD', 'BX_ID_LOVD', 'Variant_effect_LOVD', 'Variant_frequency_LOVD', 'HGVS_cDNA_LOVD', 'HGVS_protein_LOVD', 'Genetic_origin_LOVD', 'RNA_LOVD', 'Submitters_LOVD', 'DBID_LOVD', 'Created_date_LOVD', 'Edited_date_LOVD', 'Submission_ID_LOVD', 'Remarks_LOVD', 'Classification_LOVD', 'BX_ID_ESP', 'Minor_allele_frequency_percent_ESP', 'EA_Allele_Frequency_ESP', 'AA_Allele_Frequency_ESP', 'Allele_Frequency_ESP', 'polyPhen2_result_ESP', 'EUR_Allele_frequency_1000_Genomes', 'AFR_Allele_frequency_1000_Genomes', 'AMR_Allele_frequency_1000_Genomes', 'EAS_Allele_frequency_1000_Genomes', 'BX_ID_1000_Genomes', 'Allele_frequency_1000_Genomes', 'SAS_Allele_frequency_1000_Genomes', 'Allele_frequency_ExAC', 'BX_ID_ExAC', 'BX_ID_BIC', 'Patient_nationality_BIC', 'Clinical_importance_BIC', 'Clinical_classification_BIC', 'BIC_Designation_BIC', 'Literature_citation_BIC', 'Number_of_family_member_carrying_mutation_BIC', 'Germline_or_Somatic_BIC', 'Ethnicity_BIC', 'Mutation_type_BIC', 'IARC_class_exLOVD', 'BIC_Nomenclature_exLOVD', 'Sum_family_LR_exLOVD', 'Combined_prior_probablility_exLOVD', 'BX_ID_exLOVD', 'HGVS_cDNA_exLOVD', 'Literature_source_exLOVD', 'Co_occurrence_LR_exLOVD', 'Posterior_probability_exLOVD', 'Missense_analysis_prior_probability_exLOVD', 'Segregation_LR_exLOVD', 'HGVS_protein_exLOVD', "Allele_count_AFR_ExAC", "Allele_number_AFR_ExAC", "Homozygous_count_AFR_ExAC", "Allele_count_AMR_ExAC", "Allele_number_AMR_ExAC", "Homozygous_count_AMR_ExAC", "Allele_count_EAS_ExAC", "Allele_number_EAS_ExAC", "Homozygous_count_EAS_ExAC", "Allele_count_FIN_ExAC", "Allele_number_FIN_ExAC", "Homozygous_count_FIN_ExAC", "Allele_count_NFE_ExAC", "Allele_number_NFE_ExAC", "Homozygous_count_NFE_ExAC", "Allele_count_OTH_ExAC", "Allele_number_OTH_ExAC", "Homozygous_count_OTH_ExAC", "Allele_count_SAS_ExAC", "Allele_number_SAS_ExAC", "Homozygous_count_SAS_ExAC", "Allele_frequency_AFR_ExAC", "Allele_frequency_AMR_ExAC", "Allele_frequency_EAS_ExAC", "Allele_frequency_FIN_ExAC", "Allele_frequency_NFE_ExAC", "Allele_frequency_OTH_ExAC", "Allele_frequency_SAS_ExAC", "DateSignificanceLastEvaluated_ClinVar", "SCV_Version_ClinVar", "Synonyms_ClinVar", "HGVS_Nucleotide_Findlay_BRCA1_Ring_Function_Scores", "Log_RNA_Depletion_Findlay_BRCA1_Ring_Function_Scores", "Functional_Enrichment_Score_Findlay_BRCA1_Ring_Function_Scores", "BX_ID_Findlay_BRCA1_Ring_Function_Scores", "HGVS_cDNA_GnomAD", "HGVS_GnomAD", "HGVS_protein_GnomAD", "Flags_GnomAD", "Consequence_GnomAD", "Variant_id_GnomAD", "Allele_count_genome_AFR_GnomAD", "Allele_count_hemi_genome_AFR_GnomAD", "Allele_count_hom_genome_AFR_GnomAD", "Allele_number_genome_AFR_GnomAD", "Allele_frequency_genome_AFR_GnomAD", "Allele_count_genome_AMR_GnomAD", "Allele_count_hemi_genome_AMR_GnomAD", "Allele_count_hom_genome_AMR_GnomAD", "Allele_number_genome_AMR_GnomAD", "Allele_frequency_genome_AMR_GnomAD", "Allele_count_genome_ASJ_GnomAD", "Allele_count_hemi_genome_ASJ_GnomAD", "Allele_count_hom_genome_ASJ_GnomAD", "Allele_number_genome_ASJ_GnomAD", "Allele_frequency_genome_ASJ_GnomAD", "Allele_count_genome_EAS_GnomAD", "Allele_count_hemi_genome_EAS_GnomAD", "Allele_count_hom_genome_EAS_GnomAD", "Allele_number_genome_EAS_GnomAD", "Allele_frequency_genome_EAS_GnomAD", "Allele_count_genome_FIN_GnomAD", "Allele_count_hemi_genome_FIN_GnomAD", "Allele_count_hom_genome_FIN_GnomAD", "Allele_number_genome_FIN_GnomAD", "Allele_frequency_genome_FIN_GnomAD", "Allele_count_genome_NFE_GnomAD", "Allele_count_hemi_genome_NFE_GnomAD", "Allele_count_hom_genome_NFE_GnomAD", "Allele_number_genome_NFE_GnomAD", "Allele_frequency_genome_NFE_GnomAD", "Allele_count_genome_OTH_GnomAD", "Allele_count_hemi_genome_OTH_GnomAD", "Allele_count_hom_genome_OTH_GnomAD", "Allele_number_genome_OTH_GnomAD", "Allele_frequency_genome_OTH_GnomAD", "Allele_count_genome_SAS_GnomAD", "Allele_count_hemi_genome_SAS_GnomAD", "Allele_count_hom_genome_SAS_GnomAD", "Allele_number_genome_SAS_GnomAD", "Allele_frequency_genome_SAS_GnomAD", "Allele_count_genome_GnomAD", "Allele_number_genome_GnomAD", "Allele_frequency_genome_GnomAD", "Allele_count_exome_AFR_GnomAD", "Allele_count_exome_AFR_GnomAD", "Allele_count_hom_exome_AFR_GnomAD", "Allele_number_exome_AFR_GnomAD", "Allele_frequency_exome_AFR_GnomAD", "Allele_count_exome_AMR_GnomAD", "Allele_count_hemi_exome_AMR_GnomAD", "Allele_count_hom_exome_AMR_GnomAD", "Allele_number_exome_AMR_GnomAD", "Allele_frequency_exome_AMR_GnomAD", "Allele_count_exome_ASJ_GnomAD", "Allele_count_hemi_exome_ASJ_GnomAD", "Allele_count_hom_exome_ASJ_GnomAD", "Allele_number_exome_ASJ_GnomAD", "Allele_frequency_exome_ASJ_GnomAD", "Allele_count_exome_EAS_GnomAD", "Allele_count_hemi_exome_EAS_GnomAD", "Allele_count_hom_exome_EAS_GnomAD", "Allele_number_exome_EAS_GnomAD", "Allele_frequency_exome_EAS_GnomAD", "Allele_count_exome_FIN_GnomAD", "Allele_count_hemi_exome_FIN_GnomAD", "Allele_count_hom_exome_FIN_GnomAD", "Allele_number_exome_FIN_GnomAD", "Allele_frequency_exome_FIN_GnomAD", "Allele_count_exome_NFE_GnomAD", "Allele_count_hemi_exome_NFE_GnomAD", "Allele_count_hom_exome_NFE_GnomAD", "Allele_number_exome_NFE_GnomAD", "Allele_frequency_exome_NFE_GnomAD", "Allele_count_exome_OTH_GnomAD", "Allele_count_hemi_exome_OTH_GnomAD", "Allele_count_hom_exome_OTH_GnomAD", "Allele_number_exome_OTH_GnomAD", "Allele_frequency_exome_OTH_GnomAD", "Allele_count_exome_SAS_GnomAD", "Allele_count_hemi_exome_SAS_GnomAD", "Allele_count_hom_exome_SAS_GnomAD", "Allele_number_exome_SAS_GnomAD", "Allele_frequency_exome_SAS_GnomAD", "Allele_number_exome_GnomAD", "Allele_count_exome_GnomAD", "Allele_frequency_exome_GnomAD", "BX_ID_GnomAD" ] self.sources = aggregate_reports.FIELD_DICT.keys() + ["ENIGMA"] self.vcf_test_file = VCF_TESTDATA_FILENAME self.tsv_test_file = TSV_TESTDATA_FILENAME pwd = os.path.dirname(os.path.realpath(__file__)) gene_config_df = config.load_config( os.path.join(pwd, 'test_files', 'gene_config_test.txt')) self.genome_regions_symbol_dict = config.get_genome_regions_symbol_dict( gene_config_df, 'start_hg38_legacy_variants', 'end_hg38_legacy_variants')