def build_rec_process(img_dir, train=False, num_thread=1):
rec_dir = os.path.abspath(os.path.join(img_dir, '../rec'))
mkdir(rec_dir)
prefix = 'train' if train else 'val'
print('Building ImageRecord file for ' + prefix + ' ...')
to_path = rec_dir
# download lst file and im2rec script
script_path = os.path.join(rec_dir, 'im2rec.py')
script_url = 'https://raw.githubusercontent.com/apache/incubator-encoding/master/tools/im2rec.py'
download(script_url, script_path)
lst_path = os.path.join(rec_dir, prefix + '.lst')
lst_url = 'http://data.encoding.io/models/imagenet/resnet/' + prefix + '.lst'
download(lst_url, lst_path)
# execution
import sys
cmd = [
sys.executable, script_path, rec_dir, img_dir, '--recursive',
'--pass-through', '--pack-label', '--num-thread',
str(num_thread)
]
subprocess.call(cmd)
os.remove(script_path)
os.remove(lst_path)
print('ImageRecord file for ' + prefix + ' has been built!')
def get_model_file(name, root='./pretrain_models'):
r"""Return location for the pretrained on local file system.
This function will download from online model zoo when model cannot be found or has mismatch.
The root directory will be created if it doesn't exist.
Parameters
----------
name : str
Name of the model.
root : str, default './pretrain_models'
Location for keeping the model parameters.
Returns
-------
file_path
Path to the requested pretrained model file.
"""
file_name = '{name}-{short_hash}'.format(name=name,
short_hash=short_hash(name))
root = os.path.expanduser(root)
file_path = os.path.join(root, file_name + '.pth')
sha1_hash = _model_sha1[name]
if os.path.exists(file_path):
if check_sha1(file_path, sha1_hash):
return file_path
else:
print(
'Mismatch in the content of model file detected. Downloading again.'
)
else:
print('Model file is not found. Downloading.')
if not os.path.exists(root):
os.makedirs(root)
zip_file_path = os.path.join(root, file_name + '.zip')
repo_url = os.environ.get('ENCODING_REPO', encoding_repo_url)
if repo_url[-1] != '/':
repo_url = repo_url + '/'
download(_url_format.format(repo_url=repo_url, file_name=file_name),
path=zip_file_path,
overwrite=True)
with zipfile.ZipFile(zip_file_path) as zf:
zf.extractall(root)
os.remove(zip_file_path)
if check_sha1(file_path, sha1_hash):
return file_path
else:
raise ValueError(
'Downloaded file has different hash. Please try again.')
def download_ade(path, overwrite=False):
_AUG_DOWNLOAD_URLS = [
('http://host.robots.ox.ac.uk/pascal/VOC/voc2010/VOCtrainval_03-May-2010.tar',
'bf9985e9f2b064752bf6bd654d89f017c76c395a'),
('https://codalabuser.blob.core.windows.net/public/trainval_merged.json',
'169325d9f7e9047537fedca7b04de4dddf10b881'),
# You can skip these if the network is slow, the dataset will automatically generate them.
('https://hangzh.s3.amazonaws.com/encoding/data/pcontext/train.pth',
'4bfb49e8c1cefe352df876c9b5434e655c9c1d07'),
('https://hangzh.s3.amazonaws.com/encoding/data/pcontext/val.pth',
'ebedc94247ec616c57b9a2df15091784826a7b0c'),
]
download_dir = os.path.join(path, 'downloads')
mkdir(download_dir)
for url, checksum in _AUG_DOWNLOAD_URLS:
filename = download(url,
path=download_dir,
overwrite=overwrite,
sha1_hash=checksum)
# extract
if os.path.splitext(filename)[1] == '.tar':
with tarfile.open(filename) as tar:
tar.extractall(path=path)
else:
shutil.move(
filename,
os.path.join(path, 'VOCdevkit/VOC2010/' +
os.path.basename(filename)))
def download_coco(path, overwrite=False):
_DOWNLOAD_URLS = [
('http://images.cocodataset.org/zips/train2017.zip',
'10ad623668ab00c62c096f0ed636d6aff41faca5'),
('http://images.cocodataset.org/zips/val2017.zip',
'4950dc9d00dbe1c933ee0170f5797584351d2a41'),
('http://images.cocodataset.org/annotations/annotations_trainval2017.zip',
'8551ee4bb5860311e79dace7e79cb91e432e78b3'),
#('http://images.cocodataset.org/annotations/stuff_annotations_trainval2017.zip',
# '46cdcf715b6b4f67e980b529534e79c2edffe084'),
#('http://images.cocodataset.org/zips/test2017.zip',
# '99813c02442f3c112d491ea6f30cecf421d0e6b3'),
('https://hangzh.s3.amazonaws.com/encoding/data/coco/train_ids.pth',
'12cd266f97c8d9ea86e15a11f11bcb5faba700b6'),
('https://hangzh.s3.amazonaws.com/encoding/data/coco/val_ids.pth',
'4ce037ac33cbf3712fd93280a1c5e92dae3136bb'),
]
mkdir(path)
for url, checksum in _DOWNLOAD_URLS:
filename = download(url,
path=path,
overwrite=overwrite,
sha1_hash=checksum)
# extract
if os.path.splitext(filename)[1] == '.zip':
with zipfile.ZipFile(filename) as zf:
zf.extractall(path=path)
else:
shutil.move(
filename,
os.path.join(path,
'annotations/' + os.path.basename(filename)))
def download_aug(path, overwrite=False):
_AUG_DOWNLOAD_URLS = [(
'http://www.eecs.berkeley.edu/Research/Projects/CS/vision/grouping/semantic_contours/benchmark.tgz',
'7129e0a480c2d6afb02b517bb18ac54283bfaa35')]
download_dir = os.path.join(path, 'downloads')
mkdir(download_dir)
for url, checksum in _AUG_DOWNLOAD_URLS:
filename = download(url,
path=download_dir,
overwrite=overwrite,
sha1_hash=checksum)
# extract
with tarfile.open(filename) as tar:
tar.extractall(path=path)
shutil.move(os.path.join(path, 'benchmark_RELEASE'),
os.path.join(path, 'VOCaug'))
filenames = ['VOCaug/dataset/train.txt', 'VOCaug/dataset/val.txt']
# generate trainval.txt
with open(os.path.join(path, 'VOCaug/dataset/trainval.txt'),
'w') as outfile:
for fname in filenames:
fname = os.path.join(path, fname)
with open(fname) as infile:
for line in infile:
outfile.write(line)
def download_minc(path, overwrite=False):
_AUG_DOWNLOAD_URLS = [
('http://opensurfaces.cs.cornell.edu/static/minc/minc-2500.tar.gz', 'bcccbb3b1ab396ef540f024a5ba23eff54f7fe31')]
download_dir = os.path.join(path, 'downloads')
mkdir(download_dir)
for url, checksum in _AUG_DOWNLOAD_URLS:
filename = download(url, path=download_dir, overwrite=overwrite, sha1_hash=checksum)
# extract
with tarfile.open(filename) as tar:
tar.extractall(path=path)
def download_voc(path, overwrite=False):
_DOWNLOAD_URLS = [
('http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar',
'4e443f8a2eca6b1dac8a6c57641b67dd40621a49')]
download_dir = os.path.join(path, 'downloads')
mkdir(download_dir)
for url, checksum in _DOWNLOAD_URLS:
filename = download(url, path=download_dir, overwrite=overwrite, sha1_hash=checksum)
# extract
with tarfile.open(filename) as tar:
tar.extractall(path=path)
def download_ade(path, overwrite=False):
_AUG_DOWNLOAD_URLS = [
('http://data.csail.mit.edu/places/ADEchallenge/ADEChallengeData2016.zip', '219e1696abb36c8ba3a3afe7fb2f4b4606a897c7'),
('http://data.csail.mit.edu/places/ADEchallenge/release_test.zip', 'e05747892219d10e9243933371a497e905a4860c'),]
download_dir = os.path.join(path, 'downloads')
mkdir(download_dir)
for url, checksum in _AUG_DOWNLOAD_URLS:
filename = download(url, path=download_dir, overwrite=overwrite, sha1_hash=checksum)
# extract
with zipfile.ZipFile(filename,"r") as zip_ref:
zip_ref.extractall(path=path)
def download_coco(path, overwrite=False):
_DOWNLOAD_URLS = [
('http://images.cocodataset.org/zips/train2017.zip',
'10ad623668ab00c62c096f0ed636d6aff41faca5'),
('http://images.cocodataset.org/annotations/annotations_trainval2017.zip',
'8551ee4bb5860311e79dace7e79cb91e432e78b3'),
('http://images.cocodataset.org/zips/val2017.zip',
'4950dc9d00dbe1c933ee0170f5797584351d2a41')
]
mkdir(path)
for url, checksum in _DOWNLOAD_URLS:
filename = download(url,
path=path,
overwrite=overwrite,
sha1_hash=checksum)
# extract
with zipfile.ZipFile(filename) as zf:
zf.extractall(path=path)
def download_coco(path, overwrite=False):
_DOWNLOAD_URLS = [
('http://images.cocodataset.org/zips/train2017.zip',
'10ad623668ab00c62c096f0ed636d6aff41faca5'),
('http://images.cocodataset.org/annotations/annotations_trainval2017.zip',
'8551ee4bb5860311e79dace7e79cb91e432e78b3'),
('http://images.cocodataset.org/zips/val2017.zip',
'4950dc9d00dbe1c933ee0170f5797584351d2a41'),
#('http://images.cocodataset.org/annotations/stuff_annotations_trainval2017.zip',
# '46cdcf715b6b4f67e980b529534e79c2edffe084'),
#('http://images.cocodataset.org/zips/test2017.zip',
# '99813c02442f3c112d491ea6f30cecf421d0e6b3'),
]
mkdir(path)
for url, checksum in _DOWNLOAD_URLS:
filename = download(url, path=path, overwrite=overwrite, sha1_hash=checksum)
# extract
with zipfile.ZipFile(filename) as zf:
zf.extractall(path=path)
def download_coco(path, overwrite=False):
download_dir = os.path.join(path, 'downloads')
# _DOWNLOAD_URLS = [
# ('http://images.cocodataset.org/zips/train2017.zip',
# '10ad623668ab00c62c096f0ed636d6aff41faca5'),
# ('http://images.cocodataset.org/annotations/annotations_trainval2017.zip',
# '8551ee4bb5860311e79dace7e79cb91e432e78b3'),
# ('http://images.cocodataset.org/zips/val2017.zip',
# '4950dc9d00dbe1c933ee0170f5797584351d2a41'),
# ('http://images.cocodataset.org/annotations/stuff_annotations_trainval2017.zip',
# 'e7aa0f7515c07e23873a9f71d9095b06bcea3e12'),
# ('http://images.cocodataset.org/zips/test2017.zip',
# '99813c02442f3c112d491ea6f30cecf421d0e6b3'),
# ]
# mkdir(download_dir)
# for url, checksum in _DOWNLOAD_URLS:
# filename = download(url, path=download_dir, overwrite=overwrite, sha1_hash=checksum)
# # extract
# with zipfile.ZipFile(filename) as zf:
# zf.extractall(path=path)
_DOWNLOAD_URLS = [
('http://images.cocodataset.org/zips/train2017.zip'),
('http://images.cocodataset.org/annotations/annotations_trainval2017.zip'
),
('http://images.cocodataset.org/zips/val2017.zip'),
('http://images.cocodataset.org/annotations/stuff_annotations_trainval2017.zip'
),
('http://images.cocodataset.org/zips/test2017.zip'),
]
mkdir(download_dir)
for url in _DOWNLOAD_URLS:
filename = download(url,
path=download_dir,
overwrite=overwrite,
sha1_hash=None)
# extract
with zipfile.ZipFile(filename) as zf:
zf.extractall(path=path)
def download_ade(path, overwrite=False):
_AUG_DOWNLOAD_URLS = [
('http://host.robots.ox.ac.uk/pascal/VOC/voc2010/VOCtrainval_03-May-2010.tar',
'bf9985e9f2b064752bf6bd654d89f017c76c395a'),
('https://codalabuser.blob.core.windows.net/public/trainval_merged.json',
'169325d9f7e9047537fedca7b04de4dddf10b881')
]
download_dir = os.path.join(path, 'downloads')
mkdir(download_dir)
for url, checksum in _AUG_DOWNLOAD_URLS:
filename = download(url,
path=download_dir,
overwrite=overwrite,
sha1_hash=checksum)
# extract
if os.path.splitext(filename)[1] == '.tar':
with tarfile.open(filename) as tar:
tar.extractall(path=path)
else:
shutil.move(
filename,
os.path.join(path, 'VOCdevkit/VOC2010/' +
os.path.basename(filename)))
