def main():
cfg = train_configs.get_configs()
#======================================================
# 1. Make dirs
#======================================================
save_dir, model_dir, valid_dir, log_dir = set_dirs(cfg)
#======================================================
# 2. Set loggers & tensorboard
#======================================================
setup_logger('configs',
save_dir,
'configs',
level=logging.INFO,
screen=True)
setup_logger('valid',
save_dir,
'valid',
level=logging.INFO,
is_formatter=True,
screen=False)
config_logger = logging.getLogger('configs') # training logger
valid_logger = logging.getLogger('valid') # validation logger
# Tensorboard
board_writer = SummaryWriter(log_dir)
# Save Configurations
for k, v in cfg.items():
log = '{} : {}'.format(k, v)
config_logger.info(log)
#======================================================
# 3. Set GPU
#======================================================
is_gpu = False
gpu_ids = cfg.GPU_IDS
device = None
if gpu_ids is not None and torch.cuda.is_available():
is_gpu = True
torch.cuda.set_device(gpu_ids[0])
torch.backends.cudnn.benckmark = True
device = torch.device('cuda')
else:
device = torch.device('cpu')
cfg.is_gpu = is_gpu
config_logger.info('Training on: {:s} \t GPUs: {}'.format(
str(device), gpu_ids))
#======================================================
# 4. Random Seed, Define Models
#======================================================
set_random_seed(is_gpu)
# Generators
netG = generator.MSPL_Generator(in_channels=3,
out_channels=3,
n_feats=128,
n_blocks=[4, 4, 4, 4],
norm_type=None,
act_type='leakyrelu',
use_channel_attention=True,
use_global_residual=True,
use_tanh=False).to(device)
initializers.init_weights(netG, init_type=cfg.INIT_TYPE, scale=0.1)
if cfg.LOAD_PATH['load_path']:
load_path = os.path.join(cfg.LOAD_PATH['load_path'], 'netG_last.pth')
load_network(netG, load_path)
if is_gpu:
netG = nn.DataParallel(netG, device_ids=gpu_ids)
GENERATORS = dict(netG=netG)
# discriminators
if cfg.LOSS_ADV_WEIGHT:
netD = discriminator.MSPDiscriminator(3, True, True).to(device)
initializers.init_weights(netD, init_type=cfg.INIT_TYPE, scale=1)
if cfg.LOAD_PATH['load_path']:
load_path = os.path.join(cfg.LOAD_PATH['load_path'],
'netD_last.pth')
load_network(netD, load_path)
if is_gpu:
netD = nn.DataParallel(netD, device_ids=gpu_ids)
DISCRIMINATORS = dict(netD=netD)
else:
DISCRIMINATORS = None
#======================================================
# 5. Loss Fucntion, Optimizers, Schedulers
#======================================================
OPTIMIZERS = dict()
for name, model in GENERATORS.items():
if model is not None:
opt = torch.optim.Adam(model.parameters(),
lr=cfg.INIT_LR_G,
weight_decay=cfg.WEIGHT_DECAY_D,
betas=(0.9, 0.999))
OPTIMIZERS['{}'.format(name)] = opt
if DISCRIMINATORS is not None:
for name, model in DISCRIMINATORS.items():
if model is not None:
opt = torch.optim.Adam(model.parameters(),
lr=cfg.INIT_LR_D,
weight_decay=cfg.WEIGHT_DECAY_D,
betas=(0.9, 0.999))
OPTIMIZERS['{}'.format(name)] = opt
SCHEDULERS = []
for _, opt in OPTIMIZERS.items():
SCHEDULERS.append(lr_scheduler.ExponentialLR(opt, gamma=0.99))
if cfg.LOSS_L1_WEIGHT or cfg.LOSS_VGG_WEIGHT:
cri_l1 = nn.L1Loss().to(device)
else:
cri_l1 = None
if cfg.LOSS_ADV_WEIGHT:
cri_gan = ganloss.GAN_Loss(gan_type='vanilla',
real_label_val=1.0,
fake_label_val=0.0).to(device)
else:
cri_gan = None
# vggface Loss Network
if cfg.LOSS_VGG_WEIGHT:
vggface = vgg16faceloss.VGG16FeatureExtractor(
model_path=cfg.LOAD_PATH['VGGFace16']).to(device)
if is_gpu:
vggface = nn.DataParallel(vggface, device_ids=gpu_ids)
vggface.eval()
else:
vggface = None
LOSS_MODEL = dict(vggface=vggface)
CRITERIONS = dict(cri_l1=cri_l1, cri_gan=cri_gan)
cri_mse = nn.MSELoss().to(device)
#======================================================
# 6. Data loader
#======================================================
train_set = celeba_hq.CelebA_HQ(**cfg.TRAINSET)
train_loader = DataLoader(train_set,
batch_size=cfg.TRAIN_BATCH_SIZE,
shuffle=True,
num_workers=cfg.NUM_WORKERS,
drop_last=True,
pin_memory=True)
valid_set = celeba_hq.validface(**cfg.VALIDSET)
valid_loader = DataLoader(valid_set,
batch_size=cfg.VALID_BATCH_SIZE,
shuffle=False,
num_workers=cfg.NUM_WORKERS,
drop_last=True,
pin_memory=True)
# Blur kernel files
train_kernel_dict = get_blurkernels(cfg.TRAIN_KERNEL_PATH)
#======================================================
# 7. Resume Training
#======================================================
start_epoch = 0
total_epoch = cfg.TOTAL_EPOCH
if cfg.LOAD_PATH['load_path']:
load_path = os.path.join(cfg.LOAD_PATH['load_path'],
'train_state.state')
start_epoch = load_resume(load_path, OPTIMIZERS, SCHEDULERS)
start_epoch = start_epoch + 1
best_psnr = 0
for epoch in range(start_epoch, total_epoch):
train_loss = train(cfg, train_loader, GENERATORS, DISCRIMINATORS,
LOSS_MODEL, OPTIMIZERS, SCHEDULERS, CRITERIONS,
epoch, train_kernel_dict, device)
for loss_type, loss_val in train_loss.items():
board_writer.add_scalar('{}'.format(loss_type), loss_val.avg,
epoch + 1)
is_visual = False
if cfg.VISUAL_N_EPOCH is not None and epoch % cfg.VISUAL_N_EPOCH == 0:
is_visual = True
psnr_result = validate(cfg, valid_loader, GENERATORS, cri_mse, device,
epoch, valid_dir, is_visual)
avg_psnr = psnr_result['netG'].avg
log = '[Epoch:{}|{}] [PSNR Average : {:.4f}]'.format(
epoch + 1, total_epoch, avg_psnr)
print(log)
valid_logger.info(log)
for psnr_type, psnr_val in psnr_result.items():
board_writer.add_scalar('PSNR_{}'.format(psnr_type), psnr_val.avg,
epoch + 1)
is_best = avg_psnr > best_psnr
best_psnr = max(avg_psnr, best_psnr)
save_network(GENERATORS, DISCRIMINATORS, model_dir, is_best)
save_state(SCHEDULERS, OPTIMIZERS, model_dir, epoch)
adjust_learning_rate(SCHEDULERS)
'If this is not passed, then just the host is crawled. '
'Alternatively the word "ALL" can be used to crawl every '
'container. "ALL" will crawl all namespaces including the host '
'itself. This option is only valid for INVM crawl mode. Example: '
'--crawlContainers 5f3380d2319e,681be3e32661',
)
parser.add_argument(
'--environment',
dest='environment',
type=str,
default='cloudsight',
help='This speficies some environment specific behavior, like how '
'to name a container. The way to add a new behavior is by '
'implementing a plugin (see plugins/cloudsight_environment.py '
'as an example. Defaults to "cloudsight".',
)
misc.setup_logger('crawlutils', 'linker.log')
misc.setup_logger('yapsy', 'yapsy.log')
args = parser.parse_args()
crawler = DockerContainersLogsLinker(environment=args.environment,
user_list=args.crawlContainers,
host_namespace=args.namespace)
worker = Worker(emitters=None, frequency=args.frequency, crawler=crawler)
try:
worker.run()
except KeyboardInterrupt:
pass
'container. "ALL" will crawl all namespaces including the host '
'itself. This option is only valid for INVM crawl mode. Example: '
'--crawlContainers 5f3380d2319e,681be3e32661',
)
parser.add_argument(
'--environment',
dest='environment',
type=str,
default='cloudsight',
help='This speficies some environment specific behavior, like how '
'to name a container. The way to add a new behavior is by '
'implementing a plugin (see plugins/cloudsight_environment.py '
'as an example. Defaults to "cloudsight".',
)
misc.setup_logger('crawlutils', 'linker.log')
misc.setup_logger('yapsy', 'yapsy.log')
args = parser.parse_args()
crawler = DockerContainersLogsLinker(environment=args.environment,
user_list=args.crawlContainers,
host_namespace=args.namespace)
worker = Worker(emitters=None,
frequency=args.frequency,
crawler=crawler)
try:
worker.run()
except KeyboardInterrupt:
pass
def main():
euid = os.geteuid()
if euid != 0:
print 'Need to run this as root.'
exit(1)
parser = argparse.ArgumentParser()
parser.add_argument(
'--options',
dest='options',
type=json_parser,
default={},
help='JSON dict of crawler options used to be passed as arguments'
'to the crawler plugins.'
)
parser.add_argument(
'--url',
dest='url',
type=csv_list,
default=['stdout://'],
help='Send the snapshot data to URL. Defaults to the console.',
)
parser.add_argument(
'--namespace',
dest='namespace',
type=str,
nargs='?',
default=misc.get_host_ipaddr(),
help='Data source this crawler is associated with. Defaults to '
'/localhost',
)
parser.add_argument(
'--features',
dest='features',
type=csv_list,
default=['os', 'cpu'],
help='Comma-separated list of feature-types to crawl. Defaults to '
'os,cpu',
)
parser.add_argument(
'--frequency',
dest='frequency',
type=int,
default=-1,
help='Target time period for iterations. Defaults to -1 which '
'means only run one iteration.'
)
parser.add_argument(
'--compress',
dest='compress',
action='store_true',
default=False,
help='Whether to GZIP-compress the output frame data, must be one of '
'{true,false}. Defaults to false',
)
parser.add_argument('--logfile', dest='logfile', type=str,
default='crawler.log',
help='Logfile path. Defaults to crawler.log'
)
parser.add_argument(
'--crawlmode',
dest='crawlmode',
type=str,
choices=[
Modes.INVM,
Modes.OUTVM,
Modes.MOUNTPOINT,
Modes.OUTCONTAINER,
Modes.MESOS,
],
default=Modes.INVM,
help='The crawler mode: '
'{INVM,OUTVM,MOUNTPOINT,OUTCONTAINER}. '
'Defaults to INVM',
)
parser.add_argument(
'--mountpoint',
dest='mountpoint',
type=str,
default='/',
help='Mountpoint location used as the / for features like packages,'
'files, config'
)
parser.add_argument(
'--format',
dest='format',
type=str,
default='csv',
choices=['csv', 'graphite', 'json'],
help='Emitted data format.',
)
parser.add_argument(
'--crawlContainers',
dest='crawlContainers',
type=str,
nargs='?',
default='ALL',
help='List of containers to crawl as a list of Docker container IDs'
'(only Docker is supported at the moment). ' 'Defaults to all '
'running containers. Example: --crawlContainers aaa,bbb',
)
parser.add_argument(
'--crawlVMs',
dest='vm_descs_list',
nargs='+',
default='ALL',
help='List of VMs to crawl'
'Default is \'ALL\' VMs'
'Currently need following as input for each VM'
'\'vm_name, kernel_version_long, linux_flavour, arch\''
'Auto kernel version detection in future, when only vm names'
'(\'ALL\' by default) would need to be passed'
'Example --crawlVM'
'vm1,3.13.0-24-generic_3.13.0-24.x86_64,ubuntu,x86_64'
'vm2,4.0.3.x86_64,vanilla,x86_64',
)
parser.add_argument(
'--environment',
dest='environment',
type=str,
default='cloudsight',
help='This speficies some environment specific behavior, like how '
'to name a container. The way to add a new behavior is by '
'implementing a plugin (see plugins/cloudsight_environment.py '
'as an example. Defaults to "cloudsight".',
)
parser.add_argument(
'--plugins',
dest='plugin_places',
type=csv_list,
default=['plugins'],
help='This is a comma separated list of directories where to find '
'plugins. Each path can be an absolute, or a relative to the '
'location of the crawler.py. Default is "plugins"',
)
parser.add_argument(
'--numprocesses',
dest='numprocesses',
type=int,
default=1,
help='Number of processes used for container crawling. Defaults '
'to the number of cores. NOT SUPPORTED.'
)
parser.add_argument(
'--extraMetadata',
dest='extraMetadata',
type=json_parser,
default={},
help='Json with data to annotate all features. It can be used '
'to append a set of system identifiers to the metadata feature '
'and if the --extraMetadataForAll'
)
parser.add_argument(
'--avoidSetns',
dest='avoid_setns',
action='store_true',
default=False,
help='Avoids the use of the setns() syscall to crawl containers. '
'Some features like process will not work with this option. '
'Only applies to the OUTCONTAINER mode'
)
args = parser.parse_args()
misc.setup_logger('crawlutils', args.logfile)
misc.setup_logger('yapsy', 'yapsy.log')
options = args.options
options['avoid_setns'] = args.avoid_setns
options['mountpoint'] = args.mountpoint
emitters = EmittersManager(urls=args.url,
format=args.format,
compress=args.compress,
extra_metadata=args.extraMetadata)
if args.crawlmode == 'OUTCONTAINER':
crawler = ContainersCrawler(
features=args.features,
environment=args.environment,
user_list=args.crawlContainers,
host_namespace=args.namespace,
plugin_places=args.plugin_places,
options=options)
elif args.crawlmode == 'INVM' or args.crawlmode == 'MOUNTPOINT':
crawler = HostCrawler(
features=args.features,
namespace=args.namespace,
plugin_places=args.plugin_places,
options=options)
elif args.crawlmode == 'OUTVM':
crawler = VirtualMachinesCrawler(
features=args.features,
user_list=args.vm_descs_list,
host_namespace=args.namespace,
plugin_places=args.plugin_places,
options=options)
else:
raise NotImplementedError('Invalid crawlmode')
worker = Worker(emitters=emitters,
frequency=args.frequency,
crawler=crawler)
try:
worker.run()
except KeyboardInterrupt:
pass
# prepare envs
# --------------------------------------------------------------------------- #
if not os.path.exists(args.save_dir):
os.mkdir(args.save_dir)
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_id)
device = "cuda" if args.cuda and torch.cuda.is_available() else "cpu"
# make dataloader return fixed sequence of samples
torch.manual_seed(1337)
if device == "cuda":
print("USE GPU %s." %os.environ["CUDA_VISIBLE_DEVICES"])
torch.cuda.manual_seed(1337)
logger = setup_logger("simple-pytorch-fcn", args.save_dir)
logger.info(args)
# --------------------------------------------------------------------------- #
# build dataset
# --------------------------------------------------------------------------- #
resize = True if args.batch_size > 1 else False
# if batch size greater than 1, image and label will be resized to 500x500,
# otherwise return original size
if args.dataset == "voc":
train_loader = DataLoader(
dataset.VOCClassSegBase(root=args.dataset_root, split="train",
resize=resize),
batch_size=1, shuffle=True
)
def main():
"""
"""
args = parameter_parser()
tab_printer(args)
misc.create_directory('./_out/')
misc.setup_logger('agc', args, './_out/')
logger = logging.getLogger()
logging.getLogger().setLevel(logging.INFO)
if args.dataset in ['citeseer', 'cora', 'pubmed']:
nx_graph, adj, features, targets_id, classes = load_data(args.dataset)
elif args.dataset in ['dblp', 'reut', 'acm']:
if args.dataset == 'reut':
nx_graph, adj, features, targets_id, classes = exp_load_data(
args.dataset, 3)
else:
nx_graph, adj, features, targets_id, classes = exp_load_data(
args.dataset, None)
elif args.dataset in ['wiki']:
nx_graph = nx.from_edgelist(
pd.read_csv('./data/wiki.cites.csv').values.tolist())
features_file = pd.read_csv('./data/wiki.content.csv')
data = np.array(features_file["content"].values.tolist())
x1 = np.array(features_file["x1"].values.tolist())
x2 = np.array(features_file["x2"].values.tolist())
features = sp.csc_matrix((data, (x1, x2)))
logging.info('dataset massage:')
logging.info('feature size: {}'.format(features.shape))
# nodes, targets_id = misc.target_reader('./data/wiki.label.csv')
tar_file = pd.read_csv('./data/wiki.label.csv')
nodes = tar_file["node"].values.tolist()
targets_id = np.array(tar_file["labelId"]).reshape(-1, 1).T[0]
classes = 17
adj = sp.csr_matrix(nx.adjacency_matrix(nx_graph))
else:
raise Exception("dataset import error.")
# logging.info(features) # 三元组形式
# logging.info(targets)
# logging.info(node)
# logging.info(graph.adj) # 邻接表
# logging.info(nx.adjacency_matrix(graph)) # 临阶矩阵 ( .todense()转矩阵形式 )
# logging.info(nx.degree(graph)) # 每个点的度
start_time = time.time()
logging.info("Timing begin")
# agc
if args.dataset in ['citeseer', 'cora', 'pubmed', 'wiki']:
predict_C, epoch = agc(nx_graph, adj, features, targets_id, classes,
start_time, 1)
if args.dataset in ['dblp', 'reut', 'acm']:
predict_C, epoch = agc(nx_graph, adj, features, targets_id, classes,
start_time, 0)
# answer
logging.info('Best Clustering:')
logging.info(predict_C)
logging.info('k: {}'.format(epoch))
# 指标评价
F1_RESULT = metrics.f1_score(targets_id, predict_C, average='macro')
logging.info('F1_: {}%'.format(F1_RESULT * 100))
acc_RESULT = metrics.accuracy_score(targets_id, predict_C)
logging.info('acc: {}%'.format(acc_RESULT * 100))
NMI_RESULT = metrics.normalized_mutual_info_score(
targets_id, predict_C, average_method='arithmetic')
logging.info('NMI: {}%'.format(NMI_RESULT * 100))
ari_RESULT = ari_score(targets_id, predict_C)
logging.info('ari: {}%'.format(ari_RESULT * 100))
def main():
euid = os.geteuid()
if euid != 0:
print 'Need to run this as root.'
exit(1)
parser = argparse.ArgumentParser()
parser.add_argument(
'--options',
dest='options',
type=json_parser,
default={},
help='JSON dict of crawler options used to be passed as arguments'
'to the crawler plugins.'
)
parser.add_argument(
'--url',
dest='url',
type=csv_list,
default=['stdout://'],
help='Send the snapshot data to URL. Defaults to the console.',
)
parser.add_argument(
'--namespace',
dest='namespace',
type=str,
nargs='?',
default=misc.get_host_ipaddr(),
help='Data source this crawler is associated with. Defaults to '
'/localhost',
)
parser.add_argument(
'--features',
dest='features',
type=csv_list,
default=['os', 'cpu'],
help='Comma-separated list of feature-types to crawl. Defaults to '
'os,cpu',
)
parser.add_argument(
'--frequency',
dest='frequency',
type=int,
default=-1,
help='Target time period for iterations. Defaults to -1 which '
'means only run one iteration.'
)
parser.add_argument(
'--compress',
dest='compress',
action='store_true',
default=False,
help='Whether to GZIP-compress the output frame data, must be one of '
'{true,false}. Defaults to false',
)
parser.add_argument('--logfile', dest='logfile', type=str,
default='crawler.log',
help='Logfile path. Defaults to crawler.log'
)
parser.add_argument(
'--crawlmode',
dest='crawlmode',
type=str,
choices=[
Modes.INVM,
Modes.OUTVM,
Modes.MOUNTPOINT,
Modes.OUTCONTAINER,
Modes.MESOS,
],
default=Modes.INVM,
help='The crawler mode: '
'{INVM,OUTVM,MOUNTPOINT,OUTCONTAINER}. '
'Defaults to INVM',
)
parser.add_argument(
'--mountpoint',
dest='mountpoint',
type=str,
default='/',
help='Mountpoint location used as the / for features like packages,'
'files, config'
)
parser.add_argument(
'--format',
dest='format',
type=str,
default='csv',
choices=['csv', 'graphite', 'json', 'logstash'],
help='Emitted data format.',
)
parser.add_argument(
'--crawlContainers',
dest='crawlContainers',
type=str,
nargs='?',
default='ALL',
help='List of containers to crawl as a list of Docker container IDs'
'(only Docker is supported at the moment). ' 'Defaults to all '
'running containers. Example: --crawlContainers aaa,bbb',
)
parser.add_argument(
'--crawlVMs',
dest='vm_descs_list',
nargs='+',
default='ALL',
help='List of VMs to crawl'
'Default is \'ALL\' VMs'
'Currently need following as input for each VM'
'\'vm_name, kernel_version_long, linux_flavour, arch\''
'Auto kernel version detection in future, when only vm names'
'(\'ALL\' by default) would need to be passed'
'Example --crawlVM'
'vm1,3.13.0-24-generic_3.13.0-24.x86_64,ubuntu,x86_64'
'vm2,4.0.3.x86_64,vanilla,x86_64',
)
parser.add_argument(
'--environment',
dest='environment',
type=str,
default='cloudsight',
help='This speficies some environment specific behavior, like how '
'to name a container. The way to add a new behavior is by '
'implementing a plugin (see plugins/cloudsight_environment.py '
'as an example. Defaults to "cloudsight".',
)
parser.add_argument(
'--plugins',
dest='plugin_places',
type=csv_list,
default=['plugins'],
help='This is a comma separated list of directories where to find '
'plugins. Each path can be an absolute, or a relative to the '
'location of the crawler.py. Default is "plugins"',
)
parser.add_argument(
'--numprocesses',
dest='numprocesses',
type=int,
default=1,
help='Number of processes used for container crawling. Defaults '
'to the number of cores. NOT SUPPORTED.'
)
parser.add_argument(
'--extraMetadata',
dest='extraMetadata',
type=json_parser,
default={},
help='Json with data to annotate all features. It can be used '
'to append a set of system identifiers to the metadata feature '
'and if the --extraMetadataForAll'
)
parser.add_argument(
'--avoidSetns',
dest='avoid_setns',
action='store_true',
default=False,
help='Avoids the use of the setns() syscall to crawl containers. '
'Some features like process will not work with this option. '
'Only applies to the OUTCONTAINER mode'
)
args = parser.parse_args()
misc.setup_logger('crawlutils', args.logfile)
misc.setup_logger('yapsy', 'yapsy.log')
options = args.options
options['avoid_setns'] = args.avoid_setns
options['mountpoint'] = args.mountpoint
emitters = EmittersManager(urls=args.url,
format=args.format,
compress=args.compress,
extra_metadata=args.extraMetadata,
plugin_places=args.plugin_places)
if args.crawlmode == 'OUTCONTAINER':
crawler = ContainersCrawler(
features=args.features,
environment=args.environment,
user_list=args.crawlContainers,
host_namespace=args.namespace,
plugin_places=args.plugin_places,
options=options)
elif args.crawlmode == 'INVM' or args.crawlmode == 'MOUNTPOINT':
crawler = HostCrawler(
features=args.features,
namespace=args.namespace,
plugin_places=args.plugin_places,
options=options)
elif args.crawlmode == 'OUTVM':
crawler = VirtualMachinesCrawler(
features=args.features,
user_list=args.vm_descs_list,
host_namespace=args.namespace,
plugin_places=args.plugin_places,
options=options)
else:
raise NotImplementedError('Invalid crawlmode')
worker = Worker(emitters=emitters,
frequency=args.frequency,
crawler=crawler)
try:
worker.run()
except KeyboardInterrupt:
pass
