def main():
args = parse_args()
cfg = Config.fromfile(args.config)
# set training environment, e.g. distribution, cudnn_benchmark, random_seed for re-prodution
env.set_env(cfg.env_config)
# init logger before other steps
logger = log.get_root_logger(cfg.log_level)
logger.info('Distributed training: {}'.format(True))
if cfg.checkpoint_config is not None:
# save satdet version in checkpoints as meta data
cfg.checkpoint_config.meta = dict(satdet_version=__version__,
config=cfg.text)
model = build_detector(cfg.model,
train_cfg=cfg.train_cfg,
test_cfg=cfg.test_cfg)
train_dataset = get_dataset(cfg.data.train)
val_dataset = get_dataset(cfg.data.val)
train_detector(model, [train_dataset, val_dataset], cfg, logger=logger)
def config() -> Config:
c = Config()
set_env(c, EXPAND)
c.set_optimizer(lambda params: Adam(params, lr=2.5e-4, eps=1.0e-4))
c.set_net_fn('actor-critic', a2c_conv())
c.grad_clip = 0.5
c.episode_log_freq = 100
c.eval_deterministic = False
return c
def config() -> Config:
c = vae.patched_config()
# vae parameters
c.vae_loss_weight = 1.0
c.vae_loss = vae.BetaVaeLoss(beta=4.0, decoder_type='categorical_binary')
set_env(c, EXPAND)
c.set_optimizer(lambda params: Adam(params, lr=2.5e-4, eps=1.0e-4))
c.set_net_fn('actor-critic', net)
c.grad_clip = 0.5
c.episode_log_freq = 100
c.eval_deterministic = False
c.network_log_freq = 100
return c
def setup(config):
gecko_repo = repos.Gecko(config)
git_gecko = gecko_repo.repo()
wpt_repo = repos.WebPlatformTests(config)
git_wpt = wpt_repo.repo()
gh_wpt = gh.GitHub(config["web-platform-tests"]["github"]["token"],
config["web-platform-tests"]["repo"]["url"])
bz = bug.Bugzilla(config)
env.set_env(config, bz, gh_wpt)
logger.info("Gecko repository: %s" % git_gecko.working_dir)
logger.info("wpt repository: %s" % git_wpt.working_dir)
logger.info("Tasks enabled: %s" %
(", ".join(config["sync"]["enabled"].keys())))
return git_gecko, git_wpt
def config() -> Config:
c = vae.patched_config()
# vae parameters
set_env(c, EXPAND)
c.vae_loss_weight = 1.0
c.vae_loss = vae.GammaVaeLoss(gamma=200.0,
capacity_start=0.0,
capacity_max=25.0,
num_epochs=c.max_steps // (100 * 8))
c.set_optimizer(lambda params: Adam(params, lr=2.5e-4, eps=1.0e-4))
c.set_net_fn('actor-critic', net)
c.grad_clip = 0.5
c.episode_log_freq = 100
c.eval_deterministic = False
c.network_log_freq = 100
return c
import mysql.connector
from mysql.connector import errorcode
from env import set_env
import os
set_env()
class MySqlDb:
def __init__(self, db_name):
self.db_name = db_name
self.conn = self.mysql_connect()
self.table = ""
def create_db(self, db_name):
# CREATE DATABASE IF NOT EXISTS `pythonlogin` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci USE `pythonlogin`;
pass
def mysql_connect(self):
try:
cnx = mysql.connector.connect(user=os.getenv('MYSQL_USER'),
password=os.getenv('MYSQL_PASS'),
host=os.getenv('MYSQL_HOST'),
database=self.db_name)
print("Connected")
return cnx
except mysql.connector.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
print("Something is wrong with your user name or password")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
print("Database does not exist")
def tsvbuild(json_path, gcsbucket, suffix, pairflag, tsv_name, default,
metaflag):
"""builds a tsv file from a directory of paired files
Retrieves location of pairs of files matching suffix and retrieves
metadata information from the parent directory located in an xml file
Assumes paired files differences occur after an underscore '_'.
Args:
json_path (str): path to the json credentials file for GCS access
gcsbucket (str): google cloud bucket name, recursively searched
suffix (str): file identifying pattern being searched
tsv_name (str): filename or tsv file
default (bool): Use the default credentials
metaflag (bool): Find and write metadata
pairflag (bool): Find and write File pair
Returns:
str: location of tsv file
Notes:
format of column separation marked by tabs
SampleName; output; predictedinsertsize; readgroup; library_name;
platformmodel; platform; sequencingcenter; Fastq1 ; Fastq2
Dev:
Add dictquery to list of links
"""
exp_dict = {}
# set google auth
if not default:
env.set_env('GOOGLE_APPLICATION_CREDENTIALS', json_path)
header = True
loop = asyncio.get_event_loop()
for gcs_url in gcloudstorage.blob_generator(gcsbucket, suffix):
meta_dict = {}
meta_dict['File'] = gcs_url
if pairflag:
gcs_pairname, gcs_pairpath, accension = pathhandling.get_fileurl(
url=gcs_url,
sep=suffix[0],
suffix=suffix,
depth=0,
pair=pairflag)
if gcloudstorage.blob_exists(gcs_pairpath):
meta_dict['File_2'] = gcs_pairpath
else:
# seeking to create a filename
gcs_fileout, gcs_filepath, parent = pathhandling.get_fileurl(
url=gcs_url, sep=suffix[0], suffix=suffix, depth=0, pair=True)
meta_dict['output'] = gcs_fileout
if metaflag:
exp_name, exp_path, exp_folder = pathhandling.get_fileurl(
url=gcs_url,
sep=suffix[0],
suffix='.experiment.xml',
depth=1,
pair=False)
try:
curr_dict = next(
dict_extract.dict_extract(value=accension,
var=exp_dict[exp_name]))
except KeyError:
xmlfile = gcloudstorage.blob_download(exp_path)
exp_dict[exp_name] = xmldictconv.xmldictconv(xmlfile)
curr_dict = next(
dict_extract.dict_extract(value=accension,
var=exp_dict[exp_name]))
loop.run_until_complete(
dictquery.dict_endpoints(input_dict=curr_dict,
endpoint_dict=meta_dict))
tsvwriter(tsv_name, meta_dict, header)
header = False
loop.close()
if not default:
env.unset_env('GOOGLE_APPLICATION_CREDENTIALS')
return tsv_name
