def download_software(software, dst, arch):
"""
下载软件的其他资源
"""
formal_name, version = get_software_name_version(software)
others = get_software_other(formal_name, version)
download_dir = os.path.join(dst, "resources",
"{0}_{1}".format(formal_name, version))
if not os.path.exists(download_dir):
os.makedirs(download_dir, mode=0o750, exist_ok=True)
LOG.info('item:{} save dir: {}'.format(software,
os.path.basename(download_dir)))
results = []
if formal_name == "CANN":
if arch == "x86_64" or arch == "aarch64":
others = (item for item in others if arch in item['filename'])
try:
for item in others:
dest_file = os.path.join(download_dir, item['filename'])
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
if file_hash == item['sha256']:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
ret = CANN_DOWNLOAD_INST.download(item['url'], dest_file)
if ret:
print(item['filename'].ljust(60), 'download success')
results.append(ret)
finally:
CANN_DOWNLOAD_INST.quit()
else:
for item in others:
dest_file = os.path.join(download_dir, item['filename'])
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
if file_hash == item['sha256']:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
ret = DOWNLOAD_INST.download(item['url'], dest_file)
if ret:
print(item['filename'].ljust(60), 'download success')
results.append(ret)
return all(results)
def download_other_packages(dst=None):
"""
download_other_packages
:return:
"""
if dst is None:
base_dir = PROJECT_DIR
else:
base_dir = dst
resources_json = os.path.join(CUR_DIR, 'other_resources.json')
results = {'ok': [], 'failed': []}
with open(resources_json, 'r', encoding='utf-8') as json_file:
data = json.load(json_file)
for item in data:
dest_file = os.path.join(base_dir, item['dest'], item['filename'])
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
LOG.info('download[{0}] -> [{1}]'.format(
item['url'], os.path.basename(dest_file)))
if DOWNLOAD_INST.download(item['url'], dest_file):
results['ok'].append(item['filename'])
print(item['filename'].ljust(60), 'download success')
continue
results['failed'].append(item['filename'])
return results
def need_download_again(dst_file, url_with_hash):
"""
need_download_again
校验目的文件的hash值与url中的hash值是否相等，来决定是否重新下载
:param dst_file: 目的文件
:param url_with_hash: 带hash值的URL
:return:
"""
if url_with_hash is None or len(url_with_hash) == 0:
return True
if not os.path.exists(dst_file):
return True
key_word = ''
file_hash = ''
if 'sha256=' in url_with_hash:
key_word = 'sha256='
file_hash = calc_sha256(dst_file)
else:
return True
index_of_hash = str(url_with_hash).index(key_word) + len(key_word)
target_hash = url_with_hash[index_of_hash:]
if target_hash != file_hash:
LOG.info('hash {0} in url: {1} != file: {2}'.format(
key_word, target_hash, file_hash))
return target_hash != file_hash
def need_download_again(target_sha256, dst_file):
"""
need_download_again
:param target_sha256:
:param dst_file:
:return:
"""
if target_sha256 is None:
return True
if not os.path.exists(dst_file):
return True
file_sha256 = calc_sha256(dst_file)
if target_sha256 != file_sha256:
LOG.info('target sha256 : {}, exists file sha256 : {}'.format(
target_sha256, file_sha256))
return True
else:
return False
def download_specified_python(dst=None):
"""
download ascend_python_version=Python-3.7.5
:return:
"""
if dst is None:
base_dir = PROJECT_DIR
else:
base_dir = dst
specified_python = get_specified_python()
resources_json = os.path.join(CUR_DIR, 'python_version.json')
with open(resources_json, 'r', encoding='utf-8') as json_file:
data = json.load(json_file)
results = {'ok': [], 'failed': []}
for item in data:
if specified_python == item['filename'].rstrip('.tar.xz'):
dest_file = os.path.join(base_dir, item['dest'],
item['filename'])
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
break
LOG.info('download[{0}] -> [{1}]'.format(
item['url'], os.path.basename(dest_file)))
if DOWNLOAD_INST.download(item['url'], dest_file):
results['ok'].append(item['filename'])
print(item['filename'].ljust(60), 'download success')
break
results['failed'].append(item['filename'])
break
return results
def download_ms_whl(os_item, software, dst):
"""
下载软件的其他资源
"""
formal_name, version = get_software_name_version(software)
download_dir = os.path.join(dst, "resources", "{}".format(os_item))
results = []
if version in ["1.2.1", "1.1.1"]:
whl_list = get_software_mindspore(formal_name, os_item, version)
for item in whl_list:
dest_file = os.path.join(download_dir, item["dst_dir"],
os.path.basename(item['url']))
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
else:
LOG.info('{0} need download again'.format(
item['filename']))
ret = DOWNLOAD_INST.download(item['url'], dest_file)
if ret:
print(item['filename'].ljust(60), 'download success')
results.append(ret)
else:
os_item_split = os_item.split("_")
os_name, arch = "_".join(os_item_split[:2]), "_".join(
os_item_split[2:])
specified_python = get_specified_python()
implement_flag = "cp37"
if "Python-3.7" in specified_python:
implement_flag = "cp37"
if "Python-3.8" in specified_python:
implement_flag = "cp38"
if "Python-3.9" in specified_python:
implement_flag = "cp39"
if os_name == "Ubuntu_18.04":
whl_list = get_software_mindspore(formal_name,
"{}".format(os_item), version)
for item in whl_list:
if item.get('python', 'cp37') != implement_flag:
print(
"Try to get {} on {}, but it does not match {}".format(
item['filename'], item.get('python'),
implement_flag))
continue
dest_file = os.path.join(download_dir, "CPU",
os.path.basename(item['url']))
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
else:
LOG.info('{0} need download again'.format(
item['filename']))
ret = DOWNLOAD_INST.download(item['url'], dest_file)
if ret:
print(item['filename'].ljust(60), 'download success')
results.append(ret)
if os_name in [
"Ubuntu_18.04", "CentOS_7.6", "EulerOS_2.8",
"OpenEuler_20.03LTS", "Kylin_V10Tercel"
]:
whl_list = get_software_mindspore(formal_name,
"linux_{}".format(arch), version)
for item in whl_list:
if item.get('python', 'cp37') != implement_flag:
print(
"Try to get {} on {}, but it does not match {}".format(
item['filename'], item.get('python'),
implement_flag))
continue
dest_file = os.path.join(download_dir, "Ascend910",
os.path.basename(item['url']))
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
else:
LOG.info('{0} need download again'.format(
item['filename']))
ret = DOWNLOAD_INST.download(item['url'], dest_file)
if ret:
print(item['filename'].ljust(60), 'download success')
results.append(ret)
for item in whl_list:
if item.get('python', 'cp37') != implement_flag:
print(
"Try to get {} on {}, but it does not match {}".format(
item['filename'], item.get('python'),
implement_flag))
continue
A910_dest_file = os.path.join(download_dir, "Ascend910",
os.path.basename(item['url']))
if os.path.exists(A910_dest_file) and os_name in [
"Ubuntu_18.04", "CentOS_7.6", "EulerOS_2.8"
]:
dest_file = os.path.join(download_dir, "Ascend310",
os.path.basename(item['url']))
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
LOG.info('{0} exist, no need copy again'.format(
os.path.basename(dest_file)))
else:
LOG.info(
'{0} exist but not completed, need copy again'.
format(os.path.basename(dest_file)))
os.remove(dest_file)
shutil.copy(A910_dest_file, dest_file)
else:
parent_dir = os.path.dirname(dest_file)
if not os.path.exists(parent_dir):
os.makedirs(parent_dir, mode=0o750, exist_ok=True)
LOG.info('{0} not exist, copy from Ascend910'.format(
os.path.basename(dest_file)))
shutil.copy(A910_dest_file, dest_file)
return all(results)