def test_clone(metasync, opts, need_init=True):
"test cloning, after init"
if need_init:
test_init(metasync, opts)
dst = os.path.join(opts.tmpdir, "repo_clone")
util.mkdirs(dst)
# pick first backend
srv = metasync.config.get("backend", "services").split(",")[0]
clone = MetaSync(dst)
clone.cmd_clone("testing", srv, opts.encrypt_key)
# compare file side-by-side
for root, dirs, files in os.walk(clone.path_root):
for name in files:
dst = os.path.join(root, name)
src = metasync.path_root + dst[len(clone.path_root):]
try:
if not filecmp.cmp(dst, src):
assert dst.endswith("config")
except OSError as e:
assert name.startswith("head") or name.startswith("prev")
return clone
def test_fetch(metasync, opts):
"test fetching"
clone = test_clone(metasync, opts)
file_sizes = [1024, 2048]
for size in file_sizes:
pn = os.path.join(clone.path_root, "file-%s-2" % size)
util.create_random_file(pn, size)
clone.cmd_checkin(pn)
pn = os.path.join(clone.path_root, "dir1")
util.mkdirs(pn)
clone.cmd_checkin(pn)
pn = os.path.join(clone.path_root, "dir2")
util.mkdirs(pn)
pn = os.path.join(clone.path_root, "dir2", "file-1024")
util.create_random_file(pn, 1024)
pn = os.path.join(clone.path_root, "dir2")
clone.cmd_checkin(pn)
clone.cmd_push()
root2 = clone.get_root_blob()
metasync.cmd_fetch()
metasync.cmd_update()
root = metasync.get_root_blob()
cnt = 0
for i in root.walk():
cnt += 1
assert cnt == 7
def test_concurrent_upload(metasync, opts):
def _put(srv, path, remote_path):
with open(path, "rb") as f:
srv.put(remote_path, f.read())
# bump files
tmpdir = os.path.join(opts.tmpdir, "metasync-files")
sizes = [1024, 2048, 4192, 8192, 1*MB]
files = []
total_size = 1*MB
print tmpdir
util.mkdirs(tmpdir)
for size in sizes:
count = total_size / size
fl = []
for i in range(count):
fn = "file-%s-%s" % (size, i)
pn = os.path.join(tmpdir, fn)
if not os.path.exists(pn):
util.create_random_file(pn, size)
fl.append(fn)
files.append(fl)
from metasyncAPI import Worker, ThreadPool
from multiprocessing import cpu_count
pool = ThreadPool(cpu_count())
# try uploading each file
result = [["Services"] + files]
for cls in services.all_services:
if cls in [services.DiskAPI]:
continue
row = [services.slug(cls)]
srv = cls()
if srv.exists('/concurrent_upload'):
srv.rmdir('/concurrent_upload')
srv.putdir('/concurrent_upload')
print 'uploading:', row[0]
for fl in files:
beg = time.time()
for f in fl:
path = os.path.join(tmpdir, f)
remote_path = '/concurrent_upload/%s' % f
pool.submit(srv.copy, _put, path, remote_path)
pool.join()
end = time.time()
row.append(end - beg)
result.append(row)
# tabularize
for row in result:
for e in row:
print "%s\t" % e,
print
def cmd_test(metasync, args, opts):
"quick test (e.g., metasync test {%s})"
# invoke pdb when failed in testing
util.install_pdb()
tmpdir = tempfile.mkdtemp()
root = os.path.join(tmpdir, "repo")
util.mkdirs(root)
metasync = MetaSync(root)
# opts for sub test routines
opts.root = root
opts.tmpdir = tmpdir
opts.encrypt_key = "testkey" if opts.encrypt else ""
dbg.info("root: %s" % root)
dbg.info("args: %s" % args)
dbg.info("opts: ")
for (k, v) in vars(opts).iteritems():
dbg.info(" %-12s = %s" % (k, v))
alltests = dict(test.get_all_tests())
if any(case not in alltests for case in args):
dbg.err("no such a test case: %s" % args)
alltests["help"](metasync, opts)
exit(1)
# print help if no test case is provided
if len(args) == 0:
args = ["help"]
for case in args:
dbg.info("#R<testing %s#> (%s)" % (case, alltests[case].__doc__))
alltests[case](metasync, opts)
# poorman's tree
def tree(path):
for root, dirs, files in os.walk(path):
base = os.path.basename(root)
idnt = ' ' * (root.replace(path, '').count(os.sep))
print('%s%s/' % (idnt, base))
for f in files:
pn = os.path.join(root, f)
print(' %s%s [%s]' % (idnt, f, os.stat(pn).st_size))
# dump some content of blobs
if opts.dump and "objects" == base:
print(util.hexdump(util.read_file(pn, 32*2)))
print
# dump root
if not opts.notree:
tree(tmpdir)
# cleanup tmpdir
if not opts.keep:
shutil.rmtree(tmpdir)
def test_bench_upload(metasync, opts):
"bencmark upload speed of storage services"
# bump files
tmpdir = os.path.join(opts.tmpdir, "metasync-files")
sizes = [1024, 2048, 1*MB]
files = []
# for real bench
if opts.slow:
sizes = [10*MB, 100*MB]
util.mkdirs(tmpdir)
for size in sizes:
fn = "file-%s" % size
pn = os.path.join(tmpdir, fn)
if not os.path.exists(pn):
util.create_random_file(pn, size)
files.append(fn)
# try uploading each file
result = [["Services"] + files]
for cls in services.all_services:
if cls in [services.DiskAPI]:
continue
if opt.slow and cls in [services.BaiduAPI]:
continue
row = [services.slug(cls)]
srv = cls()
print 'uploading:', row[0]
if srv.exists('/upload_test'):
srv.rmdir('/upload_test')
srv.putdir('/upload_test')
for f in files:
#if row[0] == 'baidu' and f == 'file-104857600':
# continue
content = open(os.path.join(tmpdir, f), 'r').read()
beg = time.time()
srv.put('/upload_test/' + f, content)
end = time.time()
row.append(end - beg)
result.append(row)
# tabularize
for row in result:
for e in row:
print "%s\t" % e,
print
def append(self, path, msg):
import portalocker
import time
pn = self.get_path(path)
util.mkdirs(os.path.dirname(pn))
with open(pn, "a+") as log:
while True:
try:
portalocker.lock(log, portalocker.LOCK_EX)
break
except:
dbg.dbg("lock failed")
time.sleep(0.1)
log.write("%d\t%s\n" % (util.current_sec(), msg))
def test_checkin_dir(metasync, opts):
"test checkin with directory"
test_init(metasync, opts)
dst = os.path.join(metasync.path_root, "a/b")
util.mkdirs(dst)
pn = os.path.join(dst, "test-1024")
util.create_random_file(pn, 1024)
dst = os.path.join(metasync.path_root, "a")
metasync.cmd_checkin(dst)
metasync.cmd_push()
test_clone(metasync, opts, False)
def _file_create(blob, pn):
if(blob.thv == "D" or blob.thv == "M"):
util.mkdirs(pn)
for i in blob.entries:
_file_create(blob[i], os.path.join(pn, i))
elif(blob.thv == "F"):
content = blob.read()
util.write_file(pn, content.getvalue())
content.close()
# touch metadata blob (for cmd_status)
os.utime(os.path.join(self.path_objs, blob.hv), None)
elif(blob.thv == "m"):
content = blob.read()
util.write_file(pn, content)
# touch metadata blob (for cmd_status)
os.utime(os.path.join(self.path_objs, blob.hv), None)
else:
assert False
def test_blob_load(metasync, opts):
"test loading file/dir from a path"
_init_disk_metasync(metasync, opts)
bs = blobs.BlobStore2(metasync)
# /a/b/c
dirp = metasync.get_local_path("a", "b", "c")
util.mkdirs(dirp)
# /a/b/c/file
pn = os.path.join(dirp, "file")
util.create_random_file(pn, 5*KB)
blob = bs.load_dir(dirp)
blob.add("file", bs.load_file(pn))
# count how many blobs
root = bs.get_root_blob()
dbg.dbg("%-15s: %s" % ("/", root.hv))
cnt = 0
for (name, blob) in bs.walk():
dbg.dbg("%-15s: %s" % (name, blob.hv))
cnt += 1
assert cnt == len(["a", "b", "c", "file"])
# flush all new blobs
assert len(os.listdir(metasync.path_objs)) == 0
root.store()
assert len(os.listdir(metasync.path_objs)) == 6
# "." => root
test_blob = bs.load_dir(metasync.get_local_path("."))
assert test_blob == root
test_blob = bs.load_dir(metasync.get_local_path(""))
assert test_blob == root
def test_rm(metasync, opts):
"test rm file"
_init_disk_metasync(metasync, opts, 3, 2, opts.encrypt_key)
# create/commit some files
size = 512
for i in range(5):
pn = os.path.join(opts.root, "file-%s-%s" % (size, i))
util.create_random_file(pn, size)
metasync.cmd_checkin(pn)
pn = os.path.join(opts.root, "a/b")
util.mkdirs(pn)
metasync.cmd_checkin(pn)
metasync.cmd_push()
pn = os.path.join(opts.root, "a/b/e")
util.mkdirs(pn)
# try to remove non-exist directory
pn = os.path.join(opts.root, "a/b/c/d")
assert not metasync.cmd_rm(pn)
pn = os.path.join(opts.root, "a/b/e/f")
assert not metasync.cmd_rm(pn)
# try to remove non-exist file
for i in range(3):
pn = os.path.join(opts.root, "file-%s-%s" % (size, i))
metasync.cmd_rm(pn)
assert not os.path.exists(pn)
metasync.cmd_rm(os.path.join(opts.root,"a/b"))
metasync.cmd_push()
def process(imageDir, outputDir):
"""使用DBSACN裁剪图片"""
files = api.getFiles(imageDir)
total = len(files)
pointsDir = os.path.join(outputDir, "points")
cluserDir = os.path.join(outputDir, "cluster")
cropDir = os.path.join(outputDir, "crop")
util.mkdirs([cluserDir, cropDir, pointsDir])
marginInfo = {}
for i, f in enumerate(files):
basename = os.path.basename(f)
print("[info] crop image, process {} / {}: {}".format(
i + 1, total, basename))
img = cv2.imread(f)
cropImg = moveMargin(img, (45, -45, 45, -45)) # 删除边框特征
maxMargin = None
method = ["SIFT", "SURF", "ORB"]
mergePoints = []
for m in method:
img = copy.deepcopy(cropImg)
curpointsDir = os.path.join(pointsDir, m)
curcluserDir = os.path.join(cluserDir, m)
curcropDir = os.path.join(cropDir, m)
util.mkdirs([curcluserDir, curcropDir, curpointsDir])
points = getROIPoint(img, f=m) # 获得ROI点
# 保存ROI
pointsSavePath = os.path.join(curpointsDir, basename)
savePointsImg(pointsSavePath, points, img.shape)
try:
maxCluster = getMaxCluster(points, 40) # 获得最大聚类
except:
print("[error] skip: {}".format(basename))
continue
mergePoints.extend(maxCluster) # 融合特征点
# 保存关键点图像
clusterSavePath = os.path.join(curcluserDir, basename)
savePointsImg(clusterSavePath, maxCluster, img.shape)
margin = getMargin(maxCluster) # 获得边界点u, r, d, l
if not maxMargin:
maxMargin = margin
else:
u, r, d, l = margin
maxMargin[0] = min(maxMargin[0], u)
maxMargin[1] = max(maxMargin[1], r)
maxMargin[2] = max(maxMargin[2], d)
maxMargin[3] = min(maxMargin[3], l)
retImg = cropImage(img, margin) # 裁剪图片
# 保存裁剪结果
cropSavePath = os.path.join(curcropDir, basename)
cv2.imwrite(cropSavePath, retImg)
# 三种方法获得的最大边界
w, h = img.shape[0], img.shape[1]
if maxMargin[0] - up < 0:
maxMargin[0] = 0
else:
maxMargin[0] -= up
if maxMargin[1] + right > w:
maxMargin[1] = w - 1
else:
maxMargin[1] += right
# 图片下边缘不补偿
if maxMargin[3] - left < 0:
maxMargin[3] = 0
else:
maxMargin[3] -= left
marginInfo[basename] = maxMargin # 记录切边信息
retImg = cropImage(img, maxMargin) # 裁剪图片
# 保存不同特征点融合图像
mergeSavePath = os.path.join(pointsDir,
"mergePoints_{}".format(basename))
savePointsImg(mergeSavePath, mergePoints, img.shape)
# 保存裁剪结果
cropSavePath = os.path.join(cropDir, basename)
cv2.imwrite(cropSavePath, retImg)
# if i == 5: assert False, 'break'
writeMarginInfo(os.path.join(outputDir, "marginInfo.txt"),
marginInfo) # 保存到结果目录
os.startfile(outputDir)
def main():
#### options
parser = argparse.ArgumentParser()
parser.add_argument('-opt', type=str, help='Path to option YMAL file.')
parser.add_argument('--launcher', choices=['none', 'pytorch'], default='none',
help='job launcher')
parser.add_argument('--local_rank', type=int, default=0)
args = parser.parse_args()
opt = option.parse(args.opt, is_train=True)
#### distributed training settings
if args.launcher == 'none': # disabled distributed training
opt['dist'] = False
rank = -1
print('Disabled distributed training.')
else:
opt['dist'] = True
init_dist()
world_size = torch.distributed.get_world_size()
rank = torch.distributed.get_rank()
#### loading resume state if exists
if opt['path'].get('resume_state', None):
# distributed resuming: all load into default GPU
device_id = torch.cuda.current_device()
resume_state = torch.load(opt['path']['resume_state'],
map_location=lambda storage, loc: storage.cuda(device_id))
option.check_resume(opt, resume_state['iter']) # check resume options
else:
resume_state = None
#### mkdir and loggers
if rank <= 0: # normal training (rank -1) OR distributed training (rank 0)
if resume_state is None:
print(opt['path'])
util.mkdir_and_rename(
opt['path']['experiments_root']) # rename experiment folder if exists
util.mkdirs((path for key, path in opt['path'].items() if not key == 'experiments_root'
and 'pretrain_model' not in key and 'resume' not in key and path is not None))
# config loggers. Before it, the log will not work
util.setup_logger('base', opt['path']['log'], 'train_' + opt['name'], level=logging.INFO,
screen=True, tofile=True)
util.setup_logger('val', opt['path']['log'], 'val_' + opt['name'], level=logging.INFO,
screen=True, tofile=True)
logger = logging.getLogger('base')
logger.info(option.dict2str(opt))
# tensorboard logger
if opt['use_tb_logger'] and 'debug' not in opt['name']:
version = float(torch.__version__[0:3])
if version >= 1.1: # PyTorch 1.1
from torch.utils.tensorboard import SummaryWriter
else:
logger.info(
'You are using PyTorch {}. Tensorboard will use [tensorboardX]'.format(version))
from tensorboardX import SummaryWriter
trial = 0
while os.path.isdir('../Loggers/' + opt['name'] + '/' + str(trial)):
trial += 1
tb_logger = SummaryWriter(log_dir='../Loggers/' + opt['name'] + '/' + str(trial))
else:
util.setup_logger('base', opt['path']['log'], 'train', level=logging.INFO, screen=True)
logger = logging.getLogger('base')
# convert to NoneDict, which returns None for missing keys
opt = option.dict_to_nonedict(opt)
# -------------------------------------------- ADDED --------------------------------------------
l1_loss = torch.nn.L1Loss()
mse_loss = torch.nn.MSELoss()
calc_lpips = PerceptualLossLPIPS()
if torch.cuda.is_available():
l1_loss = l1_loss.cuda()
mse_loss = mse_loss.cuda()
# -----------------------------------------------------------------------------------------------
#### random seed
seed = opt['train']['manual_seed']
if seed is None:
seed = random.randint(1, 10000)
if rank <= 0:
logger.info('Random seed: {}'.format(seed))
util.set_random_seed(seed)
torch.backends.cudnn.benckmark = True
# torch.backends.cudnn.deterministic = True
#### create train and val dataloader
dataset_ratio = 200 # enlarge the size of each epoch
for phase, dataset_opt in opt['datasets'].items():
if phase == 'train':
train_set = create_dataset(dataset_opt)
train_size = int(math.ceil(len(train_set) / dataset_opt['batch_size']))
total_iters = int(opt['train']['niter'])
total_epochs = int(math.ceil(total_iters / train_size))
if opt['dist']:
train_sampler = DistIterSampler(train_set, world_size, rank, dataset_ratio)
total_epochs = int(math.ceil(total_iters / (train_size * dataset_ratio)))
else:
train_sampler = None
train_loader = create_dataloader(train_set, dataset_opt, opt, train_sampler)
if rank <= 0:
logger.info('Number of train images: {:,d}, iters: {:,d}'.format(
len(train_set), train_size))
logger.info('Total epochs needed: {:d} for iters {:,d}'.format(
total_epochs, total_iters))
elif phase == 'val':
val_set = create_dataset(dataset_opt)
val_loader = create_dataloader(val_set, dataset_opt, opt, None)
if rank <= 0:
logger.info('Number of val images in [{:s}]: {:d}'.format(
dataset_opt['name'], len(val_set)))
else:
raise NotImplementedError('Phase [{:s}] is not recognized.'.format(phase))
assert train_loader is not None
#### create model
model = Model(opt)
#### resume training
if resume_state:
logger.info('Resuming training from epoch: {}, iter: {}.'.format(
resume_state['epoch'], resume_state['iter']))
start_epoch = resume_state['epoch']
current_step = resume_state['iter']
model.resume_training(resume_state) # handle optimizers and schedulers
else:
current_step = 0
start_epoch = 0
#### training
logger.info('Start training from epoch: {:d}, iter: {:d}'.format(start_epoch, current_step))
for epoch in range(start_epoch, total_epochs + 1):
if opt['dist']:
train_sampler.set_epoch(epoch)
train_bar = tqdm(train_loader, desc='[%d/%d]' % (epoch, total_epochs))
for bus, train_data in enumerate(train_bar):
# validation
if epoch % opt['train']['val_freq'] == 0 and bus == 0 and rank <= 0:
avg_ssim = avg_psnr = avg_lpips = val_pix_err_f = val_pix_err_nf = val_mean_color_err = 0.0
print("into validation!")
idx = 0
val_bar = tqdm(val_loader, desc='[%d/%d]' % (epoch, total_epochs))
for val_data in val_bar:
idx += 1
img_name = os.path.splitext(os.path.basename(val_data['LQ_path'][0]))[0]
img_dir = os.path.join(opt['path']['val_images'], img_name)
util.mkdir(img_dir)
model.feed_data(val_data)
model.test()
visuals = model.get_current_visuals()
sr_img = util.tensor2img(visuals['SR']) # uint8
gt_img = util.tensor2img(visuals['GT']) # uint8
lq_img = util.tensor2img(visuals['LQ']) # uint8
#nr_img = util.tensor2img(visuals['NR']) # uint8
#nf_img = util.tensor2img(visuals['NF']) # uint8
#nh_img = util.tensor2img(visuals['NH']) # uint8
#print("Great! images got into here.")
# Save SR images for reference
save_sr_img_path = os.path.join(img_dir,
'{:s}_{:d}_sr.png'.format(img_name, current_step))
save_nr_img_path = os.path.join(img_dir,
'{:s}_{:d}_lq.png'.format(img_name, current_step))
#save_nf_img_path = os.path.join(img_dir,
# 'bs_{:s}_{:d}_nr.png'.format(img_name, current_step))
#save_nh_img_path = os.path.join(img_dir,
# 'bs_{:s}_{:d}_nh.png'.format(img_name, current_step))
util.save_img(sr_img, save_sr_img_path)
util.save_img(lq_img, save_nr_img_path)
#util.save_img(nf_img, save_nf_img_path)
#util.save_img(nh_img, save_nh_img_path)
#print("Saved")
# calculate PSNR
gt_img = gt_img / 255.
sr_img = sr_img / 255.
#nf_img = nf_img / 255.
lq_img = lq_img / 255.
#cropped_lq_img = lq_img[crop_size:-crop_size, crop_size:-crop_size, :]
#cropped_nr_img = nr_img[crop_size:-crop_size, crop_size:-crop_size, :]
avg_psnr += util.calculate_psnr(sr_img * 255, gt_img * 255)
avg_ssim += util.calculate_ssim(sr_img * 255, gt_img * 255)
avg_lpips += calc_lpips(visuals['SR'], visuals['GT'])
#avg_psnr_n += util.calculate_psnr(cropped_lq_img * 255, cropped_nr_img * 255)
# ----------------------------------------- ADDED -----------------------------------------
val_pix_err_nf += l1_loss(visuals['SR'], visuals['GT'])
val_mean_color_err += mse_loss(visuals['SR'].mean(2).mean(1), visuals['GT'].mean(2).mean(1))
# -----------------------------------------------------------------------------------------
avg_psnr = avg_psnr / idx
avg_ssim = avg_ssim / idx
avg_lpips = avg_lpips / idx
val_pix_err_f /= idx
val_pix_err_nf /= idx
val_mean_color_err /= idx
# log
logger.info('# Validation # PSNR: {:.4e},'.format(avg_psnr))
logger.info('# Validation # SSIM: {:.4e},'.format(avg_ssim))
logger.info('# Validation # LPIPS: {:.4e},'.format(avg_lpips))
logger_val = logging.getLogger('val') # validation logger
logger_val.info('<epoch:{:3d}, iter:{:8,d}> psnr: {:.4e} ssim: {:.4e} lpips: {:.4e}'.format(
epoch, current_step, avg_psnr, avg_ssim, avg_lpips))
# tensorboard logger
if opt['use_tb_logger'] and 'debug' not in opt['name']:
tb_logger.add_scalar('val_psnr', avg_psnr, current_step)
tb_logger.add_scalar('val_ssim', avg_ssim, current_step)
tb_logger.add_scalar('val_lpips', avg_lpips, current_step)
tb_logger.add_scalar('val_pix_err_nf', val_pix_err_nf, current_step)
tb_logger.add_scalar('val_mean_color_err', val_mean_color_err, current_step)
current_step += 1
if current_step > total_iters:
break
#### update learning rate
model.update_learning_rate(current_step, warmup_iter=opt['train']['warmup_iter'])
#### training
model.feed_data(train_data)
model.optimize_parameters(current_step)
model.clear_data()
#### tb_logger
if current_step % opt['logger']['tb_freq'] == 0:
logs = model.get_current_log()
if opt['use_tb_logger'] and 'debug' not in opt['name']:
for k, v in logs.items():
if rank <= 0:
tb_logger.add_scalar(k, v, current_step)
#### logger
if epoch % opt['logger']['print_freq'] == 0 and epoch != 0 and bus == 0:
logs = model.get_current_log()
message = '<epoch:{:3d}, iter:{:8,d}, lr:{:.3e}> '.format(
epoch, current_step, model.get_current_learning_rate())
for k, v in logs.items():
message += '{:s}: {:.4e} '.format(k, v)
if rank <= 0:
logger.info(message)
#### save models and training states
if epoch % opt['logger']['save_checkpoint_freq'] == 0 and epoch != 0 and bus == 0:
if rank <= 0:
logger.info('Saving models and training states.')
model.save(current_step)
model.save_training_state(epoch, current_step)
if rank <= 0:
logger.info('Saving the final model.')
model.save('latest')
logger.info('End of training.')
# UResNet if network_type == 'UResNet': model = UResNet() alpha_list = [0.445, 0.275, 0.13] beta_list = [0.15, 0., 0.] # RectNet elif network_type == 'RectNet': model = RectNet() alpha_list = [0.535, 0.272] beta_list = [0.134, 0.068,] else: assert True, 'Unsupported network type' # Make the checkpoint directory mkdirs(checkpoint_dir) # ------------------------------------------------------- # Set up the training routine network = nn.DataParallel( model.float(), device_ids=device_ids).to(device) #loading the model checkpoint = torch.load(checkpoint_path) util.load_partial_model(network, checkpoint['state_dict']) #runnig inference
consumer_key = 'CDq4zm9Gu5A07KakkYyIviMYn'
consumer_secret = 'PVUgtdj7FC6xkKPMRBZ1cjB8xQ86EcCWFLiQJP2UUaReFJSqA9'
access_token = '7939525151602774016-wmIukaGwjopWCz4ebAvGjmJD9I4YbJA'
access_secret = 'SNHNLDftRySr5gln7sbANjIJzF7wTEP2k4xqe71Z2CYZi'
auth = OAuthHandler(consumer_key=consumer_key, consumer_secret=consumer_secret)
auth.set_access_token(access_token, access_secret)
api = tweepy.API(auth)
#target_name = 'alibaba'
#extra_key_words = ['alibaba']
target_name = sys.argv[1]
extra_key_words = [sys.argv[1]]
file_path = target_name + '/' + time.strftime('%Y-%m-%d-%M',
time.localtime(time.time()))
mkdirs(file_path)
max_text_count = 500
current_text_count = 0
def process_full_text(full_text):
global current_text_count
# skip short text
size = len(full_text)
if size <= 2:
return
# clear url
full_text = clear_url(full_text)
from datetime import datetime
if __name__ == "__main__":
config = util.initialize_from_env()
report_frequency = config["report_frequency"]
eval_frequency = config["eval_frequency"]
# cluster_config = config["cluster"]
# util.set_gpus(*cluster_config["gpus"])
model = cm.CorefModel(config)
saver = tf.train.Saver()
TIMESTAMP = "{0:%Y-%m-%dT%H-%M-%S/}".format(datetime.now())
log_dir = util.mkdirs(os.path.join(config["log_dir"], "Global_warming"))
writer = tf.summary.FileWriter(log_dir, flush_secs=20)
max_f1 = 0
max_presission = 0
max_recall = 0
max_accuracy = 0
coord = tf.train.Coordinator()
with tf.Session() as session:
# session = tf_debug.LocalCLIDebugWrapperSession(session)
# session.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
#session.run(tf.initialize_all_variables())
session.run(tf.global_variables_initializer())
model.start_enqueue_thread(session)
def genDiffPicByThresholdValue(imgDir, outputDir): files = api.getFiles(imgDir) util.mkdirs(outputDir)
if shutil.which("gradle") is None:
print("Gradle not installed")
exit(1)
gradleLines = []
buildInfo = util.parse_build_info()
with open(args.template, 'r') as templateFile:
gradleLines = templateFile.readlines()
gradleLines = buildInfo.rewrite_variables(gradleLines)
with open(gradleBuildFile, "w+") as gradleFile:
gradleFile.writelines(gradleLines)
# Fake patches dir
util.mkdirs(os.path.join(args.gradle, 'patches'))
gradleDecompiledDir = os.path.join(args.gradle, "projects", "Clean", "src", "main", "java")
if args.force or not os.path.exists(os.path.join(gradleDecompiledDir, "net", "minecraft")):
print("Tricking ForgeGradle into decompiling minecraft")
gradle_log = open("gradle.log", "w+")
result = subprocess.Popen(["gradle", "clean", "setup"], universal_newlines=True,
stdout=gradle_log, stderr=gradle_log, cwd=args.gradle).wait()
if result != 0:
print("Unable to decompile minecraft")
print("See gradle.log for more info")
exit(1)
print("Successfully Decompiled minecraft")
else:
print("Using cached sources")
import coref_model as cm
import util
if __name__ == "__main__":
if "GPU" in os.environ:
util.set_gpus(int(os.environ["GPU"]))
else:
util.set_gpus()
if len(sys.argv) > 1:
name = sys.argv[1]
print("Running experiment: {} (from command-line argument).".format(name))
else:
name = os.environ["EXP"]
print("Running experiment: {} (from environment variable).".format(name))
config = util.get_config("experiments.conf")[name]
config["log_dir"] = util.mkdirs(os.path.join(config["log_root"], name))
util.print_config(config)
model = cm.CorefModel(config)
saver = tf.train.Saver()
log_dir = config["log_dir"]
with tf.Session() as session:
checkpoint_path = os.path.join(log_dir, "model.max.ckpt")
print("Evaluating {}".format(checkpoint_path))
saver.restore(session, checkpoint_path)
model.evaluate(session, official_stdout=True)
def test(options):
# print variable names
for v in tf.trainable_variables():
print(v.name)
print(v.get_shape())
print('Loading data ...')
data_provision = DataProvision(options)
batch_size = 400
split = 'test'
test_batch_generator = data_provision.iterate_batch(split, batch_size)
unique_anno_ids = data_provision.get_ids(split)
anchors = data_provision.get_anchors()
grounding = data_provision.get_grounding(split)
print('Start to predict ...')
t0 = time.time()
count = 0
# output data, for evaluation
out_data = {}
out_data['results'] = {}
results = {}
for batch_data in test_batch_generator:
video_feats = batch_data['video_feat']
video_feat_mask = batch_data['video_feat_mask']
feat_lens = np.sum(video_feat_mask, axis=-1)
this_batch_size = video_feat_mask.shape[0]
for sample_id in range(this_batch_size):
unique_anno_id = unique_anno_ids[count]
feat_len = feat_lens[sample_id]
# small gap (in seconds) due to feature resolution
gap = 0.5
print('%d-th video-query: %s, feat_len: %d' %
(count, unique_anno_id, feat_len))
result = []
scores = np.random.random(size=(feat_len, options['num_anchors']))
for i in range(feat_len):
for j in range(options['num_anchors']):
# calculate time stamp from feature id
end_feat = i + 0.5
start_feat = end_feat - anchors[j]
end_time = options['feature_to_second'] * end_feat
start_time = options['feature_to_second'] * start_feat
if start_time < 0. - gap:
continue
start_time = max(0., start_time)
result.append({
'timestamp': [start_time, end_time],
'score': scores[i, j]
})
print('Number of proposals (before post-processing): %d' %
len(result))
result = sorted(result, key=lambda x: x['score'], reverse=True)
# non-maximum suppresion
result = nms_detections(result, overlap=options['nms_threshold'])
print('Number of proposals (after nms): %d' % len(result))
result = sorted(result, key=lambda x: x['score'], reverse=True)
result = result[:10]
print('#{}, {}'.format(count, unique_anno_id))
print('sentence query:')
sentence_query = grounding[unique_anno_id]['raw_sentence']
print(sentence_query)
print('result (top 10):')
print(result)
print('groundtruth:')
print(grounding[unique_anno_id]['timestamp'])
results[unique_anno_id] = result
count = count + 1
out_data['results'] = results
out_json_file = 'results/random_anchor_predict_proposals_%s_nms_%.2f.json' % (
split, options['nms_threshold'])
mkdirs(os.path.dirname(out_json_file))
print('Writing result json file ...')
with open(out_json_file, 'w') as fid:
json.dump(out_data, fid)
print('Evaluating ...')
recall_at_k = get_recall_at_k(results, grounding, options['tiou_measure'],
options['max_proposal_num'])
print('Recall at {}: {}'.format(options['max_proposal_num'], recall_at_k))
print('Total running time: %f seconds.' % (time.time() - t0))
#!/usr/bin/env python
import sys
import subprocess as sp
import util
def screen(py_script, name, args):
bash = "source ~/.bashrc; python {} {}; exec bash".format(py_script, " ".join(str(a) for a in args))
command = ["screen", "-dmS", name, "bash", "-c", bash]
print(" ".join(command))
sp.call(command)
if __name__ == "__main__":
exp_name = sys.argv[1]
util.mkdirs("logs")
cluster_config = util.get_config("experiments.conf")[exp_name]["cluster"]
screen("parameter_server.py", "ps", [exp_name])
screen("evaluator.py", "eval", [exp_name])
for i, _ in enumerate(cluster_config["addresses"]["worker"]):
screen("worker.py", "w{}".format(i), [exp_name, i])
def putdir(self, path):
pn = self.get_path(path)
util.mkdirs(pn)
if __name__ == "__main__":
config = util.initialize_from_env()
report_frequency = config["report_frequency"]
eval_frequency = config["eval_frequency"]
# cluster_config = config["cluster"]
# util.set_gpus(*cluster_config["gpus"])
model = cm.CorefModel(config)
saver = tf.train.Saver()
TIMESTAMP = "{0:%Y-%m-%dT%H-%M-%S/}".format(datetime.now())
log_dir = util.mkdirs(
os.path.join(
config["log_dir"],
"v1_PublicKeyCryptography_lambda:" + str(config["lambda"])))
writer = tf.summary.FileWriter(log_dir, flush_secs=20)
max_f1 = 0
max_presission = 0
max_recall = 0
max_accuracy = 0
coord = tf.train.Coordinator()
with tf.Session() as session:
# session = tf_debug.LocalCLIDebugWrapperSession(session)
# session.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
#session.run(tf.initialize_all_variables())
session.run(tf.global_variables_initializer())
def update(self, path, content):
dbg.api('put:' + self.root + ',' + path)
pn = self.get_path(path)
util.mkdirs(os.path.dirname(pn))
return util.write_file(pn, content)
def putdir(self, path):
pn = self.get_path(path)
util.mkdirs(pn)
def __init__(self, root):
self.root = root
util.mkdirs(root)
self._sid = util.md5(self.root) % 10000
import os, sys, logging, logging.handlers
from util import mkdirs
#########################################
# Setup Logging
#########################################
LOG_FILENAME = sys.path[0] + '/log/change-merge-hook-output.log'
mkdirs(LOG_FILENAME)
formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s >> %(message)s")
# Add the log message handler to the logger
fileHandler = logging.handlers.RotatingFileHandler(
LOG_FILENAME, maxBytes=2400000, backupCount=5)
fileHandler.setFormatter(formatter)
consoleHandler = logging.StreamHandler()
consoleHandler.setLevel(logging.DEBUG)
consoleHandler.setFormatter(formatter)
# Set up a specific logger with our desired output level
def getLogger(loggerName, level=logging.DEBUG):
logger = logging.getLogger(loggerName)
logger.setLevel(level)
logger.addHandler(consoleHandler)
logger.addHandler(fileHandler)
return logger
#if "GPU" in os.environ:
# util.set_gpus(int(os.environ["GPU"]))
#else:
# util.set_gpus()
if len(sys.argv) > 1:
name = sys.argv[1]
print("Running experiment: {} (from command-line argument).".format(
name))
else:
name = os.environ["EXP"]
print(
"Running experiment: {} (from environment variable).".format(name))
config = util.get_config("experiments.conf")[name]
config["log_dir"] = util.mkdirs(os.path.join(config["log_root"], name))
config["batch_size"] = -1
config["max_tokens_per_batch"] = -1
# Use dev lm, if provided.
if config["lm_path"] and "lm_path_dev" in config and config["lm_path_dev"]:
config["lm_path"] = config["lm_path_dev"]
util.print_config(config)
data = LSGNData(config)
model = SRLModel(data, config)
evaluator = LSGNEvaluator(config)
variables_to_restore = []
for var in tf.global_variables():
def update(self, path, content):
dbg.api('put:' + self.root +',' + path)
pn = self.get_path(path)
util.mkdirs(os.path.dirname(pn))
return util.write_file(pn, content)
import sys
from os.path import abspath
from subprocess import check_call
from util import projectDir, rmpath, mkdirs
sys.platform == 'linux' or sys.exit('error: script only runs on Linux')
shutil.which('git') or sys.exit('error: git missing')
shutil.which('make') or sys.exit('error: make missing')
shutil.which('tar') or sys.exit('error: tar missing')
buildDir = os.path.join(projectDir, 'out', 'build-backend')
artifactDir = os.path.join(projectDir, 'out', 'artifact')
rmpath(buildDir)
mkdirs(buildDir)
mkdirs(artifactDir)
os.chdir(buildDir)
prebuiltUrl = 'https://github.com/rprichard/x86_64-linux-glibc2.15-4.8.git'
prebuiltPath = abspath('linux-prebuilt')
artifactPath = os.path.join(artifactDir, 'backend.tar.gz')
check_call(['git', 'clone', prebuiltUrl, prebuiltPath])
os.putenv('CXX', os.path.join(prebuiltPath, 'bin', 'x86_64-linux-g++'))
os.putenv('STRIP', os.path.join(prebuiltPath, 'bin', 'x86_64-linux-strip'))
check_call(['make', 'clean'], cwd=os.path.join(projectDir, 'backend'))
check_call(['make'], cwd=os.path.join(projectDir, 'backend'))
check_call(['tar', 'cfa', artifactPath, '-C', '..', 'wslbridge-backend'])
def __init__(self, root):
self.root = root
util.mkdirs(root)
self._sid = util.md5(self.root) % 10000
def process(imageDir, outputDir):
"""使用DBSACN裁剪图片"""
files = api.getFiles(imageDir)
total = len(files)
pointsDir = os.path.join(outputDir, "points")
cluserDir = os.path.join(outputDir, "cluster")
cropDir = os.path.join(outputDir, "crop")
util.mkdirs([cluserDir, cropDir, pointsDir])
marginInfo = {}
for i, f in enumerate(files):
basename = os.path.basename(f)
print("process {} / {}: {}".format(i+1, total, basename))
img = cv2.imread(f)
maxMargin = None
method = ["SIFT", "SURF", "ORB"]
for m in method:
curpointsDir = os.path.join(pointsDir, m)
curcluserDir = os.path.join(cluserDir, m)
curcropDir = os.path.join(cropDir, m)
util.mkdirs([curcluserDir, curcropDir, curpointsDir])
points = getROIPoint(img) # 获得ROI点
# 保存ROI
pointsSavePath = os.path.join(curpointsDir, basename)
savePointsImg(pointsSavePath, points, img.shape)
maxCluster = getMaxCluster(points, 40) # 获得最大聚类
# 保存关键点图像
clusterSavePath = os.path.join(curcluserDir, basename)
savePointsImg(clusterSavePath, maxCluster, img.shape)
margin = getMargin(maxCluster) # 获得边界点u, r, d, l
if not maxMargin:
maxMargin = margin
else:
u, r, d, l = margin
maxMargin[0] = min(maxMargin[0], u)
maxMargin[1] = max(maxMargin[1], r)
maxMargin[2] = max(maxMargin[2], d)
maxMargin[3] = min(maxMargin[3], l)
retImg = cropImage(img, margin) # 裁剪图片
# 保存裁剪结果
cropSavePath = os.path.join(curcropDir, basename)
cv2.imwrite(cropSavePath, retImg)
# 三种方法获得的最大边界
h, w = img.shape[0], img.shape[1]
print
if maxMargin[0] - up < 0:
maxMargin[0] = 0
else:
maxMargin[0] -= up
if maxMargin[1] + right > w:
maxMargin[1] = w - 1
else:
maxMargin[1] += right
if maxMargin[3] - left < 0:
maxMargin[3] = 0
else:
maxMargin[3] -= left
marginInfo[basename] = maxMargin # 记录切边信息
retImg = cropImage(img, maxMargin) # 裁剪图片
# 保存裁剪结果
cropSavePath = os.path.join(cropDir, basename)
cv2.imwrite(cropSavePath, retImg)
writeMarginInfo(os.path.join(outputDir, "marginInfo.txt"), marginInfo) # 保存到结果目录
os.startfile(outputDir)
