def check_file(file, s3_file, validate=True, **kwargs):
if validate:
base_location = os.path.dirname(file['model'])
version = base_location + '/version'
download = False
if os.path.isfile(version):
with open(version) as fopen:
if not file['version'] in fopen.read():
print(f'Found old version of {base_location}, deleting..')
_delete_folder(base_location)
print('Done.')
download = True
else:
for key, item in file.items():
if not os.path.exists(item):
download = True
break
else:
download = True
if download:
for key, item in file.items():
if 'version' in key:
continue
if not os.path.isfile(item):
print(f'downloading frozen {base_location} {key}')
download_file(s3_file[key], item)
with open(version, 'w') as fopen:
fopen.write(file['version'])
else:
if not check_available(file):
path = '/'.join(file['model'].split('/')[:-1])
raise Exception(
f'{path} is not available, please `validate = True`')
def check_file(
file,
s3_file,
validate=True,
quantized=False,
optimized=False,
**kwargs,
):
if quantized:
if 'quantized' not in file:
raise Exception(get_module(file['model'], optimized=optimized))
model = 'quantized'
if optimized:
logging.warning(
'We have no concrete proof optimized model will maintain accuracy as normal model.'
)
else:
logging.warning('Load quantized model will cause accuracy drop.')
else:
model = 'model'
if validate:
base_location = os.path.dirname(file[model])
version = base_location + '/version'
download = False
if os.path.isfile(version):
with open(version) as fopen:
if not file['version'] in fopen.read():
print(f'Found old version of {base_location}, deleting..')
_delete_folder(base_location)
print('Done.')
download = True
else:
for key, item in file.items():
if not os.path.exists(item):
download = True
break
else:
download = True
if download:
for key, item in file.items():
if 'version' in key:
continue
if model == 'quantized' and key == 'model':
continue
if model == 'model' and key == 'quantized':
continue
if not os.path.isfile(item):
print(f'downloading frozen {base_location} {key}')
download_file(s3_file[key], item)
with open(version, 'w') as fopen:
fopen.write(file['version'])
else:
if not check_available(file, quantized=quantized):
path = '/'.join(file[model].split('/')[:-1])
raise Exception(
f'{path} is not available, please `validate = True`')
def download_from_dict(file, s3_file, validate = True, quantized = False):
if quantized:
if 'quantized' not in file:
f = file.replace(home, '').split('/')
raise Exception(
f'Quantized model for {f[1].upper()} module is not available, please load normal model.'
)
model = 'quantized'
logging.warning('Load quantized model will cause accuracy drop.')
else:
model = 'model'
if validate:
base_location = os.path.dirname(file[model])
version = os.path.join(base_location, 'version')
download = False
if os.path.isfile(version):
with open(version) as fopen:
if not file['version'] in fopen.read():
print(f'Found old version of {base_location}, deleting..')
_delete_folder(base_location)
print('Done.')
download = True
else:
for key, item in file.items():
if not os.path.exists(item):
download = True
break
else:
download = True
if download:
for key, item in file.items():
if 'version' in key:
continue
if model == 'quantized' and key == 'model':
continue
if model == 'model' and key == 'quantized':
continue
if not os.path.isfile(item):
print(f'downloading frozen {key} to {item}')
download_file_cloud(s3_file[key], item)
with open(version, 'w') as fopen:
fopen.write(file['version'])
else:
if not check_files_local(file):
path = '/'.join(file[model].split('/')[:-1])
raise Exception(
f'{path} is not available, please `validate = True`'
)
def download_from_string(path, module, keys, validate=True, quantized=False):
model = path
keys = keys.copy()
keys['version'] = 'version'
if quantized:
path = os.path.join(module, f'{path}-quantized')
quantized_path = os.path.join(path, 'model.pb')
if not check_file_cloud(quantized_path)[0]:
f = quantized_path.replace(home, '').split('/')
raise Exception(
f'Quantized model for `{os.path.join(module, model)}` is not available, please load normal model.'
)
logging.warning('Load quantized model will cause accuracy drop.')
else:
path = os.path.join(module, path)
path_local = os.path.join(home, path)
files_local = {'version': os.path.join(path_local, 'version')}
files_cloud = {}
for key, value in keys.items():
# if absolute path, means, shared file in cloud, but duplicated for each models
if '/' in value:
f_local = os.path.join(path_local, value.split('/')[-1])
f_cloud = value
# combine using `module` parameter
else:
f_local = os.path.join(path_local, value)
f_cloud = os.path.join(path, value)
files_local[key] = f_local
files_cloud[key] = f_cloud
if validate:
download = False
version = files_local['version']
latest = str(
max([
check_file_cloud(item)[1] for key, item in files_cloud.items()
]))
if os.path.isfile(version):
with open(version) as fopen:
if not latest in fopen.read():
p = os.path.dirname(version)
print(f'Found old version in {p}, deleting..')
_delete_folder(p)
print('Done.')
download = True
else:
for key, item in files_local.items():
if not os.path.exists(item):
download = True
break
else:
download = True
if download:
versions = []
for key, item in files_local.items():
if 'version' in key:
continue
if not os.path.isfile(item):
print(f'downloading frozen {key} to {item}')
versions.append(download_file_cloud(
files_cloud[key], item))
latest = str(max(versions))
with open(version, 'w') as fopen:
fopen.write(latest)
else:
if not check_files_local(files_local):
path = '/'.join(files_local['model'].split('/')[:-1])
raise Exception(
f'{path} is not available, please `validate = True`')
return files_local
