def migrate(self, mig): if mig: pull.print("*", "Migration Phase. Initializing File & Configurations.", pull.YELLOW) config = CONFIG() config.read_variables() config.generate_key() config.write() time.sleep(3) application = GETWSGI() pull.print("^", "Configuration Done. Uprnning Migrations Now. ", pull.DARKCYAN) DJANGOCALL('makemigrations', stdout=(sys.stdout if self.debug else self.npointer)) pull.verbose("*", "Files Modified. Running Into Final Stage. ", self.verbose, pull.YELLOW) DJANGOCALL('migrate', stdout=(sys.stdout if self.debug else self.npointer)) pull.halt("Migrations Applied Successfuly. Exiting Now!", True, pull.GREEN) return True return False
def initialize(self, addr=""): config = CONFIG() if os.path.isfile(config.SETTPATH) or self.migrate: config.read() config.extend(addr, self.debug) config.generate() else: pull.halt("Migrations aren't applied. Apply them first!", True, pull.RED)
def resnet34(pretrained=False, **kwargs): """Constructs a ResNet-34 model. Args: pretrained (bool): If True, returns a model pre-trained on ImageNet """ cfg = CONFIG() model = ResNet(BasicBlock, [3, 4, 6, 3], cfg.num_class, **kwargs) if pretrained: model.load_state_dict(model_zoo.load_url(model_urls['resnet34'])) return model
def __init__(self, file_ap=None, file_sta=None, mbus=None): self.status = {"ap": False, "sta": False} self.file_sta = file_sta self.file_ap = file_ap self.config = CONFIG() self.run = False self.mbus = mbus
def get_feature(wav_dir, extract_breath=False): config = CONFIG() data = {} for d in os.listdir(wav_dir): speaker_dir = os.path.join(wav_dir, d) if not os.path.isdir(speaker_dir): continue data[d] = [] for file in os.listdir(speaker_dir): if file.endswith('.WAV'): y, sr = librosa.load(os.path.join(speaker_dir, file), config.sampling_frequency) if extract_breath: y = get_unvoiced_intervals(y, sr) mfcc = librosa.feature.mfcc(y, sr, n_mfcc=config.feature_size) data[d].append(mfcc) return data
def main(args): model = args.model config = CONFIG( model_name = model, data_name = args.dataset ) # clear Existing Model if args.clear_model: try: shutil.rmtree(config.checkpoint_dir) except Exception as e: print('Error! {} occured at model cleaning'.format(e)) else: print( '{} model cleaned'.format(config.checkpoint_dir) ) # build estimator build_estimator = getattr(importlib.import_module('model.{}.{}'.format(model, model)), 'build_estimator') estimator = build_estimator(config) # train or predict if args.step == 'train': early_stopping = tf.estimator.experimental.stop_if_no_decrease_hook( estimator, metric_name="loss", max_steps_without_decrease= 20 * 100 ) train_spec = tf.estimator.TrainSpec( input_fn = input_fn( step = 'train', is_predict = 0, config = config), hooks = [early_stopping]) eval_spec = tf.estimator.EvalSpec( input_fn = input_fn( step ='valid', is_predict = 1, config = config ), steps = 200, throttle_secs = 60) tf.estimator.train_and_evaluate( estimator, train_spec, eval_spec) if args.step =='predict': prediction = estimator.predict( input_fn = input_fn( step='valid', is_predict = 1, config = config) ) predict_prob = pd.DataFrame({'predict_prob': [i['prediction_prob'][1] for i in prediction ]}) predict_prob.to_csv('./result/prediction_{}.csv'.format(model))
def resnet50(pretrained=False, **kwargs): """Constructs a ResNet-50 model. Args: pretrained (bool): If True, returns a model pre-trained on ImageNet """ cfg = CONFIG() model = ResNet(Bottleneck, [3, 4, 6, 3], cfg.num_class, **kwargs) if pretrained: pretrained_dict = model_zoo.load_url(model_urls['resnet50']) model_dict = model.state_dict() pretrained_dict = { k: v for k, v in pretrained_dict.items() if k in model_dict } model_dict.update(pretrained_dict) model.load_state_dict(model_dict) #model.load_state_dict(model_zoo.load_url(model_urls['resnet50'])) return model
def configure(self, conf): if conf: toset = pull.input( "-", "Do you Want To Configure Your Database Credentials? [Y/n] ", ("y", "n"), pull.BLUE) if toset: dbase = pull.input("?", "Enter Your Database Name: ", False, pull.GREEN) serve = pull.input("?", "Enter Your Server Name [localhost]: ", False, pull.GREEN) uname = pull.input("?", "Enter Database Username: "******"?", "Enter Database Password: "******"^", "Checking Database Connection. Connecting!", pull.DARKCYAN) try: pymysql.connect(serve, uname, passw, dbase) except pymysql.err.OperationalError: pull.halt( "Access Denied for the user. Check Credentials and Server Status!", True, pull.RED) config = CONFIG() config.read_variables() config.db_create(dbase, serve, uname, passw) config.write_variables() else: pull.print( "*", "Skipped Configuration of Database. Proceeding now. ", pull.YELLOW) pull.halt("Done Configurations. Exiting", True, pull.RED)
import random import time import sys import os import torch import torch.nn as nn import numpy as np from utils.utils import * from models.gcn import GCN from models.mlp import MLP from config import CONFIG cfg = CONFIG() if len(sys.argv) != 2: sys.exit("Use: python train.py <dataset>") datasets = ['20ng', 'R8', 'R52', 'ohsumed', 'mr'] dataset = sys.argv[1] if dataset not in datasets: sys.exit("wrong dataset name") cfg.dataset = dataset # Set random seed seed = random.randint(1, 200) seed = 2019 np.random.seed(seed)
import tensorflow as tf import numpy as np import os import pickle import cv2 import logging import sys import matplotlib.pyplot as plt from config import CONFIG CONFIG = CONFIG() from model import CNN_Encoder, RNN_Decoder import utils as utils from prepare_img_features import model_config_dict from tools.timer import timer from tools.logging_helper import LOGGING_CONFIG logging.basicConfig(**LOGGING_CONFIG.print_kwargs) logger = logging.getLogger(__name__) logger.info('Logging has begun!') os.environ["CUDA_VISIBLE_DEVICES"] = "" import gradio as gr import requests class InstgramCaptioner: def __init__(self, checkpoint_path, tokenizer_path, CONFIG): """Load weights of encoder-decoder model from checkpoint. Load saved tokenizer.
from ansys_model import ANSYS_MODEL from NAGA_2 import MyProblem, NSGA2_ANSYS from config import CONFIG import pyansys # 初始化配置文件 opt = CONFIG(root_path='/root/', Encoding='BG', NIND=20, MAXGEN=300, Drawing=1, plot_use=False) # ansys建模 ansys = pyansys.Mapdl(run_location=opt.root_path, override=True, interactive_plotting=True, loglevel="ERROR") model = ANSYS_MODEL(ansys=ansys, opt=opt) model.model() # 多目标优化 ansys_nsga2 = NSGA2_ANSYS(MyProblem(ansys), opt.Encoding, opt.NIND, opt.MAXGEN, opt.Drawing) ansys_nsga2.run_nsga()
def main(): os.environ["WANDB_IGNORE_GLOBS"]="*.ckpt" logging.info("empezando setup del experimento") torch.backends.cudnn.benchmark = True config=CONFIG() config_dict=asdict(config) wandb.init( project="Hierarchical-label-transformers", entity='dcastf01', config=config_dict ) config=wandb.config print(config) wandb.run.name=config.experiment_name[:5]+" "+\ datetime.datetime.utcnow().strftime("%b %d %X") wandb.run.notes=config.notes # wandb.run.save() wandb_logger=WandbLogger( #offline=True, log_model =False ) #get transform_fn transfrom_fn,transform_fn_test,collate_fn=get_transform_collate_function( config.transform_name, config.IMG_SIZE, config.collate_fn_name, config.experiment_name, ) #get datamodule dm=get_datamodule(config.dataset_name, config.batch_size, transfrom_fn, transform_fn_test, collate_fn ) #get losses losses=get_losses_fn(config) #get callbacks callbacks=get_callbacks(config,dm) #get system model=get_system( datamodule=dm, criterions=losses, architecture_type=config.architecture_name, model_choice= config.experiment_name, optim=config.optim_name, scheduler_name=config.scheduler_name, lr= config.lr, img_size=config.IMG_SIZE, pretrained=config.PRETRAINED_MODEL, epochs=config.NUM_EPOCHS, steps_per_epoch=len(dm.train_dataloader()), ) #create trainer trainer=get_trainer(wandb_logger,callbacks,config) model=autotune_lr(trainer, model, dm, get_auto_lr=config.AUTO_LR, model_name=config.experiment_name, dataset_name=config.dataset_name ) logging.info("empezando el entrenamiento") trainer.fit(model,datamodule=dm) trainer.test(test_dataloaders=dm.test_dataloader())
from UBM.GMM_UBM import GMM_UBM from config import CONFIG from get_breath_sound.UnvoicedIntervals import get_feature import numpy as np import os config = CONFIG() def get_ubm(ubm_data_dir): ubm = GMM_UBM() if os.path.exists(r'.\models\ubm.npy'): ubm.read_feature(r'.\models\ubm.npy') else: features = get_feature(ubm_data_dir) data = [] for value in features.values(): for i in value: data.append(i) ubm.gmm_em(data, 512, 10, 1) ubm.save_feature(r'.\models\ubm') return ubm def adapt(ubm, train_dir): features = get_feature(train_dir, True) gmm = GMM_UBM() for key in features.keys(): data = [] for d in features[key]: data.append(d) gmm.map_adapt(data, ubm, config.tau)
def main(): print("empezando setup del experimento") torch.backends.cudnn.benchmark = True config=CONFIG() config_dict=create_config_dict(config) wandb.init( project='TFM-classification', entity='dcastf01', name=config.experiment_name+" "+ datetime.datetime.utcnow().strftime("%Y-%m-%d %X"), config=config_dict) wandb_logger = WandbLogger( # offline=True, ) config =wandb.config dataloaders,NUM_CLASSES=choice_loader_and_splits_dataset( config.dataset_name, batch_size=config.batch_size, NUM_WORKERS=config.NUM_WORKERS, use_tripletLoss=config.USE_TRIPLETLOSS,#config_experiment.use_tripletLoss ) logging.info("DEVICE",config.DEVICE) train_loader=dataloaders["train"] test_loader=dataloaders["test"] ##callbacks early_stopping=EarlyStopping(monitor='_val_loss',verbose=True) # checkpoint_callback = ModelCheckpoint( # monitor='_val_loss', # dirpath=config.PATH_CHECKPOINT, # filename= '-{epoch:02d}-{val_loss:.6f}', # mode="min", # save_last=True, # save_top_k=3, # ) learning_rate_monitor=LearningRateMonitor(logging_interval="epoch") backbone=build_model( config.experiment_name, NUM_CLASSES=NUM_CLASSES, pretrained=config.PRETRAINED_MODEL, transfer_learning=config.transfer_learning, ) model=LitClassifier(backbone, # loss_fn=loss_fn, lr=config.lr, NUM_CLASSES=NUM_CLASSES, optim=config.optim_name ) # model=model.model.load_from_checkpoint("/home/dcast/object_detection_TFM/classification/model/checkpoint/last.ckpt") wandb_logger.watch(model.model) trainer=pl.Trainer( logger=wandb_logger, gpus=-1, max_epochs=config.NUM_EPOCHS, precision=config.precision_compute, # limit_train_batches=0.1, #only to debug # limit_val_batches=0.05, #only to debug # val_check_interval=1, auto_lr_find=config.AUTO_LR, log_gpu_memory=True, # distributed_backend='ddp', # accelerator="dpp", # plugins=DDPPlugin(find_unused_parameters=False), callbacks=[ # early_stopping , # checkpoint_callback, # confusion_matrix_wandb, learning_rate_monitor ], progress_bar_refresh_rate=5, ) model=autotune_lr(trainer,model,test_loader,get_auto_lr=config.AUTO_LR) logging.info("empezando el entrenamiento") trainer.fit(model,train_loader,test_loader)
def resnet152(pretrained=False, **kwargs): cfg = CONFIG() model = ResNet(Bottleneck, [3, 8, 36, 3], cfg.num_class, **kwargs) return model
""" This is the main driver of the CVE Process class. """ import gzip import json import requests from config import CONFIG from controllers import dynamoDBO, mitre_db from models import cve settings = CONFIG() dbo = dynamoDBO.dynamo(config=settings.settings['dbo']) nvd_cve = mitre_db.nvd_lookup() def process_easy_os(): """ This funciton will process any of the Easy OS CVE's that are present :return: """ items = dbo.query_cve_meta_keys(key="type", value="operating_system") #pre-process filter stats={ "ToProcess": {"count": 0, "critical": 0, "high": 0, "medium": 0, "low": 0, "toSkip": 0}, "Processed": {"count": 0, "critical": 0, "high": 0, "medium": 0, "low": 0, "toSkip": 0}, "Errors": {} } for item in items:
import argparse, os, re, string, pickle import tweepy from pandas import DataFrame from config import CONFIG from datetime import datetime from nltk.tokenize import word_tokenize from Sastrawi.Stemmer.StemmerFactory import StemmerFactory # Configuration config = CONFIG('CONFIG.json').data # Create Sastrawi stemmer STEMMER = StemmerFactory().create_stemmer() # Create Stopword with open("kamus/Stopword.txt", "r") as f: stop_words = f.readline().split() # Cleanner def cleaning(text): text = text[2:] text = text.replace('\\n', ' ') return text # Preprocessor def preprocessor(text): # Convert to lower case text = text.lower() # Remove additional code