def run(): host = get_config("azimu_api.host") port = get_config("azimu_api.port") path = get_config("azimu_repo_path") endpoint = "/data/git" url = f"{host}:{port}/{endpoint}" for repo in __get_repos(path): __process_repository(repo, url) __process_commits(repo, url) __process_refs(repo, url)
def run(endpoint): repositories = get_config("bitbucket.repositories") host = get_config("azimu_api.host") port = get_config("azimu_api.port") url = f"{host}:{port}{endpoint}" for repository in repositories: process_repository(repository, url) # process_commits(repository, url) process_pull_requests(repository, url, "OPEN") process_pull_requests(repository, url, "MERGED") process_refs(repository, url) result = True return result
def generate_header(self, master_key = None): header = { "Authentication": get_config(key="TEST_TOKEN"), "Content-Type": "application/json" } return header
def process_bitbucket(): result = False git_provider_name = get_config("git") if git_provider_name == "bitbucket": endpoint = "/data/git" result = bitbucket_runner.run(endpoint) return result
def main(): args = parse_args() set_global_seeds(666) config = get_config(args.config) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') runner = PytorchInference(device) test_loader = make_data(**config['data_params'], mode='test') weights_paths = collect_weight_paths(args.path, args.fold) os.makedirs(args.output, exist_ok=True) predictions = defaultdict(lambda: {'mask': 0, 'empty': 0}) for i, weights_path in enumerate(weights_paths): torch.cuda.empty_cache() print(f'weights: {weights_path}') config['train_params']['weights'] = weights_path factory = Factory(config['train_params']) model = factory.make_model(device) for result in tqdm(iterable=runner.predict(model, test_loader), total=len(test_loader) * config['data_params']['batch_size']): ensemble_mask = (predictions[result['image_id']]['mask'] * i + result['mask']) / (i + 1) ensemble_empty = (predictions[result['image_id']]['empty'] * i + result['empty']) / (i + 1) predictions[result['image_id']]['mask'] = ensemble_mask predictions[result['image_id']]['empty'] = ensemble_empty mmcv.dump(dict(predictions), osp.join(args.output, f'fold_{args.fold}.pkl'))
def wrapper(*args, **kwargs): try: request_limits = get_config(key='REQUEST_LIMITS') per_ip_limit = request_limits['PER_IP_LIMIT'] parse_limit = request_limits['PARSE_LIMIT'] # Limit number of requests per IP adress check_request_limit( requests=per_ip_limit['NUM_REQUESTS'], window=per_ip_limit['INTERVAL'], by='ip', group=None ) # Limit number of requests per second check_request_limit( requests=parse_limit['NUM_REQUESTS'], window=parse_limit['INTERVAL'], by='parse', group='parse' ) except IOError as err: return err.message return wrapped(*args, **kwargs)
def __init__(self): config = get_config() intents = discord.Intents.all() super().__init__(command_prefix=config["bot"]["prefix"], intents=intents) self.config = config self.con = connect_database(self.config["database"]["path"])
def init(): twillio_info = get_config(key='TWILLIO_INFO') client = twilio.rest.TwilioRestClient( twillio_info['TWILLIO_ACCOUNT'], twillio_info['TWILLIO_AUTH_TOKEN'] ) return client
def send_email(email_to, subject, content, subs, email_from=None): if get_config(key='SENDGRID_INFO') == {}: return 200, 'Not send email' sg, message = sendgrid_init(email_to, subject, subs, email_from) message.set_html(content) return sg.send(message)
def get_model(model_name, config_file): if model_name == 'unet_resnet34': model = Unet("resnet34", encoder_weights="imagenet", activation=None) elif model_name == 'hrnet': config = get_config(config_file) model = get_hrseg_model(config) elif model_name == 'segnet': model = segnet(pretrained=True) return model
def validate_auth_token(auth): try: return jwt.decode(jwt=auth, key=get_config(key='JWT_KEY')) except jwt.ExpiredSignature: return {'error': 'Token is expired'} except jwt.DecodeError: return {'error': 'Token signature is invalid'} except Exception: return {'error': 'Problem parsing token'}
def send_reset_password_email(email, password, username=None): if username is None: username = '******'.format(email.split("@", 1)[0], email) email_to = '{}<{}>'.format(username, email) subject = 'Reset for password for {}'.format(get_config(key='APP_NAME')) content = "<html><head><title></title><link href='https://fonts.googleapis.com/css?family=Ubuntu' rel='stylesheet' type='text/css'><style type='text/css'></style></head><body style='background-color: #fDfDfD;'><div style='font-family: Ubuntu, sans-serif;background-color: white;width: 80%;margin: auto;padding: 20px;border: 1px solid #E5E5E5;'> <h4 style='font-size:24px'>Hi {{name}},</h4><p style='font-size:18px'>We have received a request to set a new password for your {{app}} account. </p><p style='font-size:18px'>Your new password is shown in the bow below.</p><div style='padding-top: 15px;padding-bottom: 15px;width: 100%;margin: auto;text-align:center'><div style='margin: auto;background: #3BCA96;background-image: -webkit-linear-gradient(top, #3BCA96, #3BCA90);background-image: -moz-linear-gradient(top, #3BCA96, #3BCA90);background-image: -ms-linear-gradient(top, #3BCA96, #3BCA90);background-image: -o-linear-gradient(top, #3BCA96, #3BCA90);background-image: linear-gradient(to bottom, #3BCA96, #3BCA90);-webkit-border-radius: 3;-moz-border-radius: 3;border-radius: 3px;text-shadow: 1px 1px 3px #666666;font-family: Arial;width: 400px;color: #ffffff;font-size: 23px;padding: 10px 20px 10px 20px;text-decoration: none;'>{{password}}</div></div><p style='font-size:18px'>Please login to <a href='{{link}}'>{{app}}</a> and change your password</p><p style='font-size:18px'>Best regards,</p><p style='font-size:18px'>The {{app}} team</p></div></body></html>" subs = { '{{name}}': username, '{{app}}': get_config(key='APP_NAME'), '{{link}}': get_config(key='APP_LINK'), '{{password}}': password } return send_email(email_to=email_to, email_from=None, subject=subject, subs=subs, content=content)
def train(hparams: dict): config_list = ["--dataset-config", "--model-config", "--runner-config"] config: DictConfig = get_config(hparams=hparams, options=config_list) # TODO. 임시방편 OmegaConf.set_readonly(config, False) for key, value in config.hyperparameter.items(): config.runner.experiments.name += "-" + str(key) + "_" + str(value) OmegaConf.set_readonly(config, True) log_dir = get_log_dir(config=config) log_dir.mkdir(parents=True, exist_ok=True) train_dataloader, test_dataloader = get_data_loaders(config=config) model: nn.Module = build_model(model_conf=config.model) training_container: pl.LightningModule = TrainingContainer(model=model, config=config) checkpoint_callback = get_checkpoint_callback(log_dir=log_dir, config=config) wandb_logger = get_wandb_logger(log_dir=log_dir, config=config) wandb_logger.watch(model, log="gradients", log_freq=100) lr_logger = LearningRateMonitor() early_stop_callback = get_early_stopper( early_stopping_config=config.runner.earlystopping.params) with (log_dir / Path("config.yaml")).open("w") as f: OmegaConf.save(config=config, f=f) trainer = Trainer( accelerator=config.runner.trainer.distributed_backend, fast_dev_run=False, gpus=config.runner.trainer.params.gpus, amp_level="O2", logger=wandb_logger, callbacks=[early_stop_callback, lr_logger], checkpoint_callback=checkpoint_callback, max_epochs=config.runner.trainer.params.max_epochs, weights_summary="top", reload_dataloaders_every_epoch=False, resume_from_checkpoint=None, benchmark=False, deterministic=True, num_sanity_val_steps=0, overfit_batches=0.0, precision=32, profiler=True, limit_train_batches=1.0, ) trainer.fit( model=training_container, train_dataloader=train_dataloader, val_dataloaders=test_dataloader, )
def tearup_conv_config() -> DictConfig: config_path = { "--data-config": "conf/conv/data/data.yml", "--model-config": "conf/conv/model/model.yml", "--training-config": "conf/conv/training/training.yml", } config_list = ["--data-config", "--model-config", "--training-config"] config: DictConfig = get_config(hparams=config_path, options=config_list) return config
def send_reset_password_email(email, password, username=None): if username is None: username = '******'.format(email.split("@", 1)[0], email) email_to = '{}<{}>'.format(username, email) subject = 'Reset for password for {}'.format(get_config(key='APP_NAME')) content = "<html><head><title></title><link href='https://fonts.googleapis.com/css?family=Ubuntu' rel='stylesheet' type='text/css'><style type='text/css'></style></head><body style='background-color: #fDfDfD;'><div style='font-family: Ubuntu, sans-serif;background-color: white;width: 80%;margin: auto;padding: 20px;border: 1px solid #E5E5E5;'> <h4 style='font-size:24px'>Hi {{name}},</h4><p style='font-size:18px'>We have received a request to set a new password for your {{app}} account. </p><p style='font-size:18px'>Your new password is shown in the bow below.</p><div style='padding-top: 15px;padding-bottom: 15px;width: 100%;margin: auto;text-align:center'><div style='margin: auto;background: #3BCA96;background-image: -webkit-linear-gradient(top, #3BCA96, #3BCA90);background-image: -moz-linear-gradient(top, #3BCA96, #3BCA90);background-image: -ms-linear-gradient(top, #3BCA96, #3BCA90);background-image: -o-linear-gradient(top, #3BCA96, #3BCA90);background-image: linear-gradient(to bottom, #3BCA96, #3BCA90);-webkit-border-radius: 3;-moz-border-radius: 3;border-radius: 3px;text-shadow: 1px 1px 3px #666666;font-family: Arial;width: 400px;color: #ffffff;font-size: 23px;padding: 10px 20px 10px 20px;text-decoration: none;'>{{password}}</div></div><p style='font-size:18px'>Please login to <a href='{{link}}'>{{app}}</a> and change your password</p><p style='font-size:18px'>Best regards,</p><p style='font-size:18px'>The {{app}} team</p></div></body></html>" subs = { '{{name}}': username, '{{app}}': get_config(key='APP_NAME'), '{{link}}': get_config(key='APP_LINK'), '{{password}}': password } return send_email( email_to=email_to, email_from=None, subject=subject, subs=subs, content=content)
def test_signup(self): string = self.random_string(length=20) payload = { 'username': string, 'password': get_config(key="TEST_PASSWORD"), 'email': "*****@*****.**" % string } res = self.post_data(url='signup', data=payload) assert 'createdAt' in res assert 'token' in res
def test_signup_with_existing_username(self): params = {'keys': 'username'} res = self.get_data(url='users', params=params) payload = { 'username': res['results'][0]['username'], 'password': get_config(key="TEST_PASSWORD") } res = self.post_data(url='signup', data=payload) assert 'error' in res
def send_sms(to_number, body, media_url=None): try: twillio_info = get_config(key='TWILLIO_INFO') client = init() message = client.messages.create(to=to_number, from_=twillio_info['TWILLIO_NUMBER'], body=body, media_url=media_url) return message.error_code except twilio.TwilioRestException as e: return e
def __process_repository(repo, url): table_name = 'git_repository' repo_info = vars(get_repository_info(repo)) print(vars(repo_info)) print(f'http://{url}/{table_name}') r = requests.post( f'http://{url}/{table_name}', json={ "repository": json.dumps(repo_info, sort_keys=True, default=str) }, headers={"AUTH_TOKEN": get_config("azimu_api.auth_token")})
def test_delete(self): string = self.random_string(length=20) payload = { 'username': string, 'password': get_config(key="TEST_PASSWORD"), 'email': "*****@*****.**" % string } res = self.post_data(url='signup', data=payload) url = 'users/' + res['objectId'] res = self.delete_data(url=url) assert 'error' not in res
def send_activation_email(email, objectId, username=None): if username is None: username = '******'.format(email.split("@", 1)[0], email) app_info = get_config(key='APP_INFO') email_to = '{}<{}>'.format(username, email) subject = 'Activation' content = "<html><head><title></title><linkhref='https://fonts.googleapis.com/css?family=Ubuntu' rel='stylesheet' type='text/css'><style type='text/css'></style></head><body style='background-color: #fDfDfD;'><div style='font-family: Ubuntu, sans-serif;background-color: white;width: 80%;margin: auto;padding: 20px;border: 1px solid #E5E5E5;'> <h4 style='font-size:24px'>Hi {{name}} ,</h4><p style='font-size:18px'>Congrats! Your {{app}} account has been created successfully.</p><p style='font-size:18px'>Click on the button below to activate your account.</p><div style='padding-top: 15px;padding-bottom: 15px;width: 100%;margin: auto;text-align:center'><a href='{{link}}' style='margin: auto;background: #3BCA96;background-image: -webkit-linear-gradient(top, #3BCA96, #3BCA90);background-image: -moz-linear-gradient(top, #3BCA96, #3BCA90);background-image: -ms-linear-gradient(top, #3BCA96, #3BCA90);background-image: -o-linear-gradient(top, #3BCA96, #3BCA90);background-image: linear-gradient(to bottom, #3BCA96, #3BCA90);-webkit-border-radius: 3;-moz-border-radius: 3;border-radius: 3px;text-shadow: 1px 1px 3px #666666;font-family: Arial;color: #ffffff;font-size: 23px;padding: 10px 20px 10px 20px;text-decoration: none;'>Activate Your Account</a></div><p style='font-size:18px'>Best regards,</p><p style='font-size:18px'>The {{app}} team</p></div></body></html>" subs = { '{{name}}': username, '{{app}}': app_info['APP_NAME'], '{{link}}': '{}activate/{}>'.format(get_config(key='API_LINK'), objectId) } return send_email(email_to=email_to, email_from=None, subject=subject, subs=subs, content=content)
def send_activation_email(email, objectId, username=None): if username is None: username = '******'.format(email.split("@", 1)[0], email) app_info = get_config(key='APP_INFO') email_to = '{}<{}>'.format(username, email) subject = 'Activation' content = "<html><head><title></title><linkhref='https://fonts.googleapis.com/css?family=Ubuntu' rel='stylesheet' type='text/css'><style type='text/css'></style></head><body style='background-color: #fDfDfD;'><div style='font-family: Ubuntu, sans-serif;background-color: white;width: 80%;margin: auto;padding: 20px;border: 1px solid #E5E5E5;'> <h4 style='font-size:24px'>Hi {{name}} ,</h4><p style='font-size:18px'>Congrats! Your {{app}} account has been created successfully.</p><p style='font-size:18px'>Click on the button below to activate your account.</p><div style='padding-top: 15px;padding-bottom: 15px;width: 100%;margin: auto;text-align:center'><a href='{{link}}' style='margin: auto;background: #3BCA96;background-image: -webkit-linear-gradient(top, #3BCA96, #3BCA90);background-image: -moz-linear-gradient(top, #3BCA96, #3BCA90);background-image: -ms-linear-gradient(top, #3BCA96, #3BCA90);background-image: -o-linear-gradient(top, #3BCA96, #3BCA90);background-image: linear-gradient(to bottom, #3BCA96, #3BCA90);-webkit-border-radius: 3;-moz-border-radius: 3;border-radius: 3px;text-shadow: 1px 1px 3px #666666;font-family: Arial;color: #ffffff;font-size: 23px;padding: 10px 20px 10px 20px;text-decoration: none;'>Activate Your Account</a></div><p style='font-size:18px'>Best regards,</p><p style='font-size:18px'>The {{app}} team</p></div></body></html>" subs = { '{{name}}': username, '{{app}}': app_info['APP_NAME'], '{{link}}': '{}activate/{}>'.format(get_config(key='API_LINK'), objectId) } return send_email( email_to=email_to, email_from=None, subject=subject, subs=subs, content=content)
def test_reset_password(self): string = self.random_string(length=20) payload = { 'username': string, 'password': get_config(key="TEST_PASSWORD"), 'email': "*****@*****.**" % string } user = self.post_data(url='signup', data=payload) res = self.post_data(url='resetpassword', data=payload) assert 'updatedAt' in res assert res['email']['status'] is 200
def send_sms(to_number, body, media_url=None): try: twillio_info = get_config(key='TWILLIO_INFO') client = init() message = client.messages.create( to=to_number, from_=twillio_info['TWILLIO_NUMBER'], body=body, media_url=media_url ) return message.error_code except twilio.TwilioRestException as e: return e
def test_validate_cookie(): with mock.patch('src.utils.get_jwt_secret', return_value='foobar') as jwt_sec: token_in = {'foo': 'bar'} cookie_name = utils.get_config().login_cookie_name cookie = jwt.encode(token_in, jwt_sec.return_value, 'HS256').decode('ascii') token_out = utils.validate_login_cookie( {'headers': { 'Cookie': f'{cookie_name}={cookie}' }}) assert token_in == token_out
def generate_header(self, master_key=None): parse_info = get_config(key='PARSE_INFO') header = { 'X-Parse-Application-Id': parse_info['PARSE_APP_ID'], 'X-Parse-REST-API-Key': parse_info['PARSE_REST_KEY'], 'Content-Type': 'application/json' } if master_key: header['X-Parse-Master-Key'] = parse_info['PARSE_MASTER_KEY'] return header
def generate_url(self, collection, object_id=None): parse_info = get_config(key='PARSE_INFO') base_url = parse_info['PARSE_URL'] if collection in self._parse_special_classes: url = '{}/{}'.format(base_url, collection) else: url = '{}/classes/{}'.format(base_url, collection) if object_id is not None: url = '{}/{}'.format(url, object_id) return url
def test_login(self): string = self.random_string(length=20) payload = { 'username': string, 'password': get_config(key="TEST_PASSWORD"), 'email': "*****@*****.**" % string } res = self.post_data(url='signup', data=payload) params = payload res = self.get_data(url='login', params=params) assert 'objectId' in res assert 'token' in res
def process_repository(repository, url): table_name = "bitbucket_repository" user_name = get_config("bitbucket.user_name") user_name, repo_name = repository.split("/") repository = Repository.find_repository_by_name_and_owner( repository_name=repo_name, owner=user_name, client=bitbucket_client) logger.debug("Bitbucket repository - {}".format(repository)) repository_info = vars(get_repository_info(repository.data)) logger.debug("BitBucket repository owner - {}".format( repository_info['owner_uuid'])) process_user(repository_info['owner_uuid'], url) print(f'https://{url}/{table_name}') r = requests.post( f'https://{url}/{table_name}', json={ "repository": json.dumps(repository_info, sort_keys=True, default=str) }, headers={"AUTH_TOKEN": get_config("azimu_api.auth_token")}) if r.status_code != 200: raise Exception(r.status_code) return r.json()
def test_put(self): params = {'keys': 'phone,username'} user = self.get_data(url='users', params=params) url = 'users/' + user['results'][0]['objectId'] payload = { 'username': user['results'][0]['username'], 'password': get_config(key="TEST_PASSWORD"), 'phone': 'phone has changed' } res = self.put_data(url=url, data=payload) user = self.get_data(url='users/' + user['results'][0]['objectId']) assert user['results'][0]['phone'] == 'phone has changed' assert 'updatedAt' in res
def __process_refs(repo, url): table_name = 'git_ref' for ref in repo.refs: ref_info = vars(get_ref_info(ref)) if ref_info: r = requests.post( f'http://{url}/{table_name}', json={ "ref": json.dumps(ref_info, sort_keys=True, default=str) }, headers={"AUTH_TOKEN": get_config("azimu_api.auth_token")}) if r.status_code != 200: raise Exception(r.status_code) return r.json()
def __init__(self): self.config = get_config() if not os.isdir("logs"): os.mkdir("logs") super().__init__( command_prefix=self.config["bot"]["prefix"], intents=discord.Intents.all(), activity=discord.Game(self.config["bot"]["status"]), help_command=None, ) self.discord = Logger.discordLogger() self.logger = Logger.defaultLogger("EasyGameBot") self.con = connect_database(self.config["database"]["path"])
async def on_ready(self): self.allowed_mentions = discord.AllowedMentions( replied_user=self.config["bot"]["reply_mention"]) init_stock( self.con, self.config["game"]["stock"]["stocks"], self.config["game"]["stock"]["stock_default_price"], ) for extension in get_config()["bot"]["extensions"]: self.load_extension(extension) self.logger.info("준비 완료!")
def sendgrid_init(email_to, subject, subs, email_from): sendgrid_info = get_config(key="SENDGRID_INFO") sg = sendgrid.SendGridClient(sendgrid_info['SENDGRID_API_KEY']) message = sendgrid.Mail() message.add_to(email_to) message.set_subject(subject) if email_from is None: email_from = '{}<{}>'.format(sendgrid_info['SENDGRID_NAME'], sendgrid_info['SENDGRID_EMAIL']) message.set_from(email_from) for key in subs: message.add_substitution(key, subs[key]) return sg, message
def main(): args = parse_args() config = get_config(args.config_dir) set_global_seeds(456) for fold in range(4): config['train_params']['fold'] = fold netname = f'{args.netname}/fold{fold}' log_save_dir = Path( config['dumps']['path']) / config['dumps']['logs'] / netname model_save_dir = Path( config['dumps']['path']) / config['dumps']['weights'] / netname os.makedirs(log_save_dir, exist_ok=True) os.makedirs(model_save_dir, exist_ok=True) config['train_params']['netname'] = netname config['train_params']['model_path'] = model_save_dir pprint(config) data_loaders = make_data_fold(**config['data_params'], fold=config['train_params']['fold']) model = get_model(args.netname, config) callbacks = create_callbacks(netname, config['dumps']) model_trainer = Trainer(model, callbacks, data_loaders, **config['train_params']) model_trainer.start() # PLOT TRAINING losses = model_trainer.losses dice_scores = model_trainer.dice_scores # overall dice iou_scores = model_trainer.iou_scores def plot(scores, name=args.netname, is_save=True): plt.figure(figsize=(15, 5)) plt.plot(range(len(scores["train"])), scores["train"], label=f'train {name}') plt.plot(range(len(scores["train"])), scores["val"], label=f'val {name}') plt.title(f'{name} plot') plt.xlabel('Epoch') plt.ylabel(f'{name}') plt.legend() if is_save: plt.savefig(os.path.join(log_save_dir, name + '.png')) plot(losses, "Loss") plot(dice_scores, "Dice score") plot(iou_scores, "IoU score")
def sendgrid_init(email_to, subject, subs, email_from): sendgrid_info = get_config(key="SENDGRID_INFO") sg = sendgrid.SendGridClient(sendgrid_info['SENDGRID_API_KEY']) message = sendgrid.Mail() message.add_to(email_to) message.set_subject(subject) if email_from is None: email_from = '{}<{}>'.format( sendgrid_info['SENDGRID_NAME'], sendgrid_info['SENDGRID_EMAIL'] ) message.set_from(email_from) for key in subs: message.add_substitution(key, subs[key]) return sg, message
def open(self, *args, **kw): """ Overriding open to add in Auth token header """ headers = [ ("Authentication", get_config(key="TEST_TOKEN")) ] # set default header to be json content_type = "application/json" if 'data' in kw and 'file' in kw['data'] and 'profile' not in \ kw['data']: content_type = "multipart/form-data" headers.append(("Content-Type", content_type)) if kw.get('headers'): kw['headers'].append(headers) else: kw['headers'] = headers return super(TestClient, self).open(*args, **kw)
def generate_auth_token(payload): return jwt.encode(payload=payload, key=get_config(key='JWT_KEY'))
def setUp(self): self.app = TestClient(app, app.response_class) self.prefix_url = get_config(key="TEST_API_URL") + "/"
def generate_url(self, url): url = "{}/{}".format( get_config(key="TEST_API_URL"), url ) return url