def init():
global args, logger, writer
args = get_parser()
logger = get_logger()
writer = SummaryWriter(args.save_path)
if args.train_gpu is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(
str(x) for x in args.train_gpu)
if args.manual_seed is not None:
cudnn.benchmark = False
cudnn.deterministic = True
random.seed(args.manual_seed)
np.random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
torch.cuda.manual_seed(args.manual_seed)
torch.cuda.manual_seed_all(args.manual_seed)
if args.train_gpu is not None and len(args.train_gpu) == 1:
args.sync_bn = False
logger.info(args)
def main():
global args, logger
args = get_parser()
logger = get_logger()
logger.info(args)
assert args.classes > 1
os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(
str(x) for x in args.test_gpu)
logger.info("=> creating model ...")
logger.info("Classes: {}".format(args.classes))
if args.arch == 'pointnet_seg':
from model.pointnet.pointnet import PointNetSeg as Model
elif args.arch == 'pointnet2_seg':
from model.pointnet2.pointnet2_seg import PointNet2SSGSeg as Model
elif args.arch == 'pointnet2_paconv_seg':
from model.pointnet2.pointnet2_paconv_seg import PointNet2SSGSeg as Model
else:
raise Exception('architecture not supported yet'.format(args.arch))
model = Model(c=args.fea_dim,
k=args.classes,
use_xyz=args.use_xyz,
args=args)
model = torch.nn.DataParallel(model.cuda())
logger.info(model)
criterion = nn.CrossEntropyLoss(ignore_index=args.ignore_label).cuda()
names = [line.rstrip('\n') for line in open(args.names_path)]
if os.path.isfile(args.model_path):
logger.info("=> loading checkpoint '{}'".format(args.model_path))
checkpoint = torch.load(args.model_path)
model.load_state_dict(checkpoint['state_dict'], strict=True)
logger.info("=> loaded checkpoint '{}'".format(args.model_path))
else:
raise RuntimeError("=> no checkpoint found at '{}'".format(
args.model_path))
test(model, criterion, names)
def setup_class(self):
self.driver = webdriver.Chrome()
self.driver.get('http://localhost:8080/jpress/user/login')
self.driver.maximize_window()
self.logger = util.get_logger()
self.logger.info('测试用户登录')
#!/usr/bin/env python3
import json
import click
from tqdm import tqdm
from util import util
logger = util.get_logger(__name__)
@click.command()
@click.argument('index')
@click.argument('output_file')
@click.option('-q',
'--query',
help='Sample selection query',
type=click.File('r'))
@click.option('-f',
'--output-format',
help='Output format (multiple may be specified)',
multiple=True,
type=click.Choice(['json', 'text']),
default='json')
@click.option('-n',
'--total-mails',
help='Total number of mails to sample',
type=int,
default=1000)
@click.option('-l', '--group-limit', help='Group sample limit', type=int)
def train(cfg):
if cfg.image.name != "normal":
print("image=imgというオプションをつけてください")
exit(0)
logger = get_logger("./log/")
# lossutil = LossUtil("./log/")
logger.info(cfg)
ROOT = Path(utils.get_original_cwd()).parent
print(ROOT)
# data_path = ROOT.joinpath("datasets/img/1024-512.png")
data_path = ROOT.joinpath("datasets/img/512-256.png")
img = np.array(Image.open(data_path), dtype=float)
img = img / 255
result_dir = Path("./result_imgs/")
if not result_dir.exists():
Path(result_dir).mkdir(parents=True)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if cfg.image.type == "denoise":
sigma = 0.1
img = get_noisy_img(img, sigma)
mask = None
tmp = Image.fromarray(np.uint8(255*img))
tmp.save(result_dir.joinpath("target.png"))
elif "inpaint" in cfg.image.type:
mask = make_mask(cfg, img)
# ターゲット画像の保存
tmp = mask*img
tmp = Image.fromarray(np.uint8(255*tmp))
tmp.save(result_dir.joinpath("target.png"))
mask = torch.from_numpy(mask)
mask = mask[None, :].permute(0, 3, 1, 2).to(device)
logger.info(mask.shape)
else:
raise NotImplementedError(
'[%s] is not Implemented' % cfg.image.type)
transform = transforms.Compose([transforms.ToTensor()])
img = transform(img)
img = img[None, :].to(device)
logger.info(img.shape)
logger.info(img.dtype)
np.random.seed(cfg.base_options.seed)
torch.manual_seed(cfg.base_options.seed)
torch.cuda.manual_seed_all(cfg.base_options.seed)
torch.backends.cudnn.benchmark = False
model = DIP(cfg, img, mask)
logger.info(model.generator)
if cfg.base_options.debugging:
summary(model.generator, (1, 1024, 512))
# summary(model.generator, (3, 256, 256))
print('デバッグ中 途中で終了します!!!')
exit(0)
input_noise = torch.randn(1, 1, img.shape[2], img.shape[3], device=device)
logger.info(input_noise.dtype)
logger.info(input_noise.shape)
for epoch in range(cfg.base_options.epochs):
if epoch % cfg.base_options.print_freq == 0:
epoch_start_time = time.time()
if not cfg.base_options.do_fix_noise:
input_noise = torch.randn(
1, 1, img.shape[2], img.shape[3], dtype=torch.float32, device=device)
model.forward(input_noise)
# logger.info(model.gimg.shape)
# logger.info(model.gimg.dtype)
model.optimize_parameters()
# model.update_learning_rate()
losses = model.get_current_losses()
if epoch % cfg.base_options.print_freq == 0:
num = cfg.base_options.print_freq
t1 = (time.time() - epoch_start_time)
t2 = float(t1) / num
print("%dエポックの所要時間%.3f 平均時間%.3f" % (num, t1, t2))
print_losses(epoch, losses)
if cfg.base_options.save_model_freq != -1 and epoch % cfg.base_options.save_model_freq == 0:
model.save_networks(epoch)
model.save_losses()
if epoch % cfg.base_options.save_img_freq == 0:
new_img = model.gimg.detach()[0]
# CHW -> HWC
new_img = new_img.permute(1, 2, 0).cpu().float().numpy()
# print(new_img.shape, new_img.device, new_img.dtype)
new_img = Image.fromarray(np.uint8(255*new_img))
new_img.save(result_dir.joinpath("%05d.png" % epoch))
if cfg.base_options.save_model_freq != -1:
model.save_networks("finish")
model.save_losses()
# 実験結果の動画
freq = cfg.base_options.save_img_freq
epochs = cfg.base_options.epochs
logger.info("making result video")
result_imgs = ["./result_imgs/%05d.png" %
(epoch) for epoch in range(epochs) if epoch % freq == 0]
make_video("./", result_imgs, img.shape[3], img.shape[2])
logger.info("making result video done!")
positive_values = np.array([])
negative_values = np.array([])
prev_keyword = cur_keyword
cur_activation = float(row[2])
if cur_activation > 0:
positive_values = np.append(positive_values, cur_activation)
else:
negative_values = np.append(negative_values, cur_activation)
write_data(cur_keyword, positive_values, negative_values, output_csv)
def meta_analysis():
mode_list = ['train']
for mode in mode_list:
input_path = '../data/results/{0}_origin_meta.csv'.format(mode)
df = pd.read_csv(input_path)
print(df.head())
# input_csv = csv.reader(open(input_path, 'r'))
#
# output_path = '../data/results/{0}_origin_meta_analysis.csv'.format(mode)
# output_csv = csv.writer(open(output_path, 'w'))
if __name__ == '__main__':
logger = util.get_logger()
# count()
meta_analysis()
from intercom import Intercom, Conversation
import os
import html2text
from util.util import get_logger
__author__ = 'Deyang'
intercomm_logger = get_logger('intercomm')
Intercom.app_id = os.environ.get('INTERCOM_APP_ID')
Intercom.app_api_key = os.environ.get('INTERCOM_APP_API_KEY')
ADMIN_DEREK_ID = '426663'
ADMIN_BOT_ID = '426928'
SUBSCRIBED_REPLY_TOPIC = 'conversation.user.replied'
SUBSCRIBED_CREATE_TOPIC = 'conversation.user.created'
def reply_to_user(conversation_id, reply_msg):
conversation = Conversation.find(id=conversation_id)
conversation.reply(
type='admin', id=str(ADMIN_BOT_ID),
message_type='comment', body=reply_msg)
def parse_notification_and_should_reply(notification):
intercomm_logger.debug("Raw notification: %s" % notification)
try:
conversation_id = notification['data']['item']['id']
assignee = notification['data']['item']['assignee']
from chatterbot import ChatBot
from wit import message
import re
import os
import json
from util.util import get_logger
__author__ = 'Deyang'
bot_logger = get_logger('bot')
wit_token = 'QUCDCX7MQX4FLYGONBEYLGDHKSTIUFTQ'
# identity_intent = [conversation[0] for conversation in identity_data]
# ask_company_intent = [conversation[0] for conversation in ask_company_data]
# ask_customer_intent = [conversation[0] for conversation in ask_customer_data]
# ask_doc_intent = [conversation[0] for conversation in ask_doc_data]
# ask_price_intent = [conversation[0] for conversation in ask_price_data]
# ask_product_intent = [conversation[0] for conversation in ask_product_data]
# ask_story_intent = [conversation[0] for conversation in ask_story_data]
GREETING_INTENT = 'greetings'
IDENTITY_INTENT = 'identity'
TRY_INTENT = 'try'
EMAIL_INTENT = 'email'
ASK_INTENT_PATTERN = re.compile('ask.*')
def load_data(file_name):
dirpath = os.path.dirname(os.path.abspath(__file__))
from chatterbot import ChatBot
from wit import message
import re
import os
import json
from util.util import get_logger
__author__ = 'Deyang'
bot_logger = get_logger('bot')
wit_token = 'QUCDCX7MQX4FLYGONBEYLGDHKSTIUFTQ'
# identity_intent = [conversation[0] for conversation in identity_data]
# ask_company_intent = [conversation[0] for conversation in ask_company_data]
# ask_customer_intent = [conversation[0] for conversation in ask_customer_data]
# ask_doc_intent = [conversation[0] for conversation in ask_doc_data]
# ask_price_intent = [conversation[0] for conversation in ask_price_data]
# ask_product_intent = [conversation[0] for conversation in ask_product_data]
# ask_story_intent = [conversation[0] for conversation in ask_story_data]
GREETING_INTENT = 'greetings'
IDENTITY_INTENT = 'identity'
TRY_INTENT = 'try'
EMAIL_INTENT = 'email'
ASK_INTENT_PATTERN = re.compile('ask.*')
def load_data(file_name):
dirpath = os.path.dirname(os.path.abspath(__file__))
filepath = os.path.join(dirpath, '..', 'data', file_name)
fd = open(filepath, 'r')
def train(cfg):
logger = get_logger("./log/")
# lossutil = LossUtil("./log/")
logger.info(cfg)
ROOT = Path(utils.get_original_cwd()).parent
print(ROOT)
data_path = ROOT.joinpath("datasets/hsimg/data.tiff")
himg = imread(str(data_path))
himg = himg / 2**16
# 分光画像の範囲を光源の範囲に制限
himg = himg[:, :, :44]
himg = np.where(himg < 0, 0, himg)
nhimg = np.empty((512, 512, himg.shape[2]))
for i in range(himg.shape[2]):
logger.info("resize %d channel..." % i)
ch = himg[:, :, i]
nhimg[:, :, i] = cv2.resize(255 * ch, (512, 512),
interpolation=cv2.INTER_LANCZOS4)
logger.info("resize %d channel... done!" % i)
himg = nhimg / 255
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if cfg.image.type == "denoise":
sigma = 0.1
himg = get_noisy_img(himg, sigma)
mask = None
elif "inpaint" in cfg.image.type:
mask = make_mask_himg(cfg, himg)
logger.info(mask.dtype)
mask = torch.from_numpy(mask)
mask = mask[None, :].permute(0, 3, 1, 2).to(device)
logger.info(mask.shape)
logger.info(mask.dtype)
else:
raise NotImplementedError('[%s] is not Implemented' % cfg.image.type)
transform = transforms.Compose([transforms.ToTensor()])
himg = transform(himg)
himg = himg[None, :].float().to(device)
dist_path = ROOT.joinpath("datasets/csvs/D65.csv")
cmf_path = ROOT.joinpath("datasets/csvs/CIE1964-10deg-XYZ.csv")
tiff2rgb = Tiff2rgb(dist_path, cmf_path)
# ターゲット画像を保存
if cfg.image.type == "denoise":
img = tiff2rgb.tiff_to_rgb(himg[0].permute(1, 2, 0))
else:
tmp = himg * mask
logger.info(tmp.shape)
logger.info(tmp.dtype)
img = tiff2rgb.tiff_to_rgb(tmp[0].permute(1, 2, 0))
result_dir = Path("./result_imgs/")
if not result_dir.exists():
Path(result_dir).mkdir(parents=True)
img.save(result_dir.joinpath("target.png"))
logger.info(himg.shape)
logger.info(himg.dtype)
np.random.seed(cfg.base_options.seed)
torch.manual_seed(cfg.base_options.seed)
torch.cuda.manual_seed_all(cfg.base_options.seed)
torch.backends.cudnn.benchmark = False
model = DIP(cfg, himg, mask)
logger.info(model.generator)
if cfg.base_options.debugging:
summary(model.generator, (1, 512, 512))
# summary(model.generator, (3, 256, 256))
print('デバッグ中 途中で終了します!!!')
exit(0)
input_noise = torch.randn(1,
1,
himg.shape[2],
himg.shape[3],
device=device)
logger.info(input_noise.dtype)
for epoch in range(cfg.base_options.epochs):
if epoch % cfg.base_options.print_freq == 0:
epoch_start_time = time.time()
if not cfg.base_options.do_fix_noise:
input_noise = torch.randn(1,
1,
himg.shape[2],
himg.shape[3],
dtype=torch.float32,
device=device)
model.forward(input_noise)
# logger.info(model.gimg.shape)
# logger.info(model.gimg.dtype)
model.optimize_parameters()
# model.update_learning_rate()
losses = model.get_current_losses()
if epoch % cfg.base_options.print_freq == 0:
num = cfg.base_options.print_freq
t1 = (time.time() - epoch_start_time)
t2 = float(t1) / num
print("%dエポックの所要時間%.3f 平均時間%.3f" % (num, t1, t2))
print_losses(epoch, losses)
if cfg.base_options.save_model_freq != -1 and epoch % cfg.base_options.save_model_freq == 0:
model.save_networks(epoch)
model.save_losses()
if epoch % cfg.base_options.save_img_freq == 0:
new_img = model.gimg.detach()[0]
# CHW -> HWC
new_img = new_img.permute(1, 2, 0)
# print(new_img.shape, new_img.device, new_img.dtype)
img = tiff2rgb.tiff_to_rgb(new_img)
img.save(result_dir.joinpath("%05d.png" % epoch))
if cfg.base_options.save_model_freq != -1:
model.save_networks("finish")
model.save_losses()
# 実験結果の動画
freq = cfg.base_options.save_img_freq
epochs = cfg.base_options.epochs
result_imgs = [
"./result_imgs/%05d.png" % (epoch) for epoch in range(epochs)
if epoch % freq == 0
]
make_video("./", result_imgs, width=himg.shape[3], height=himg.shape[3])
