def download(url, out=None):
import wget
if out is not None:
mkdir_if_not(out)
logg.info('downloading... [{}] to [{}]'.format(url, out))
f = wget.download(url, out=out)
return f
def imsave(fname, *args, **kwargs):
from skimage import io
mkdir_if_not(fname)
res = io.imsave(fname, *args, **kwargs)
logg.info('image saved to [{}]'.format(fname))
return res
def get_cuda_nums():
from snipy.cuda import get_cuda_device_count
logg.info('flag.cuda: {}'.format(flag.cuda))
if not flag.cuda:
# resotre me
count = get_cuda_device_count()
# count = 8
flag.cuda = ','.join(str(i) for i in range(count))
return flag.cuda.split(',')
def untar(t, member=None):
import tarfile
path = os.path.dirname(t)
tar = tarfile.open(t)
if member is None:
tar.extractall(path)
logg.info('unzip [{}] to [{}]'.format(t, path))
else:
tar.extract(member, path)
logg.info('unzip [{}] to [{}]'.format(member, path))
tar.close()
def unzip(z, member=None):
import zipfile
path = os.path.dirname(z)
zip = zipfile.ZipFile(z, 'r')
if member is None:
zip.extractall(path)
logg.info('unzip [{}] to [{}]'.format(z, path))
else:
zip.extract(member, path)
logg.info('unzip [{}] to [{}]'.format(member, path))
zip.close()
def _run_gpu_test(cuda, script, *args, **kwargs):
import sys
params = ['--{}={}'.format(k, v) for k, v in kwargs.items()]
cmd = 'python {} {} {} --cuda={} '.format(script, ' '.join(args),
' '.join(params), cuda)
logg.info('cmd: [{}]'.format(cmd))
# res = os.system(cmd)
sys.stdout.flush()
return 0
def run_grid(script, *args, **kwargs):
"""
run script with each combination of parameters
:param str script: python script
:param args:
:param kwargs:
:return:
"""
from snipy.iterflow import iqueue
script = script[0]
test = kwargs.pop('test', False)
action = _run_gpu_test if test else None
logg.info('script == {}'.format(script))
logg.info('args== {}'.format(str(args)))
logg.info('flag.cuda: {}'.format(flag.cuda))
# flag.cuda = kwargs.pop('cuda', None)
args = list(map(_try_as_list, args))
logg.info('args== {}'.format(str(args)))
kwargs = {k: _try_as_list(v) for k, v in kwargs.items()}
# all parameter combinations
jobq = iqueue(_iter_combination(args, kwargs))
# cudajob_q foreach gpu
cudaq = [
CudaJobQ(jobq, str(i), script, action=action).start()
for i in get_cuda_nums()
]
while not jobq.empty():
sleep(1)
# wait for all job finished
for q in cudaq:
while not q.empty():
sleep(0.5)
q.stop()
q.join()
while not jobq.done:
continue
logg.info('done!')
return 0
def finish(self):
if self._first:
raise ValueError('setup not called')
self.moviewriter.finish()
logg.info('movie saved to [{}]'.format(self.outfile))