def download_other_packages():
script = os.path.realpath(__file__)
script_dir = os.path.dirname(script)
base_dir = os.path.dirname(script_dir)
resources_json = os.path.join(script_dir, 'other_resources.json')
with open(resources_json, 'r', encoding='utf-8') as json_file:
data = json.load(json_file)
for item in data:
dest_file = os.path.join(base_dir, item['dest'], item['filename'])
print('download[{0}] -> [{1}]'.format(item['url'], dest_file))
DOWNLOAD_INST.download(item['url'], dest_file)
def file_download(url, dest):
"""
file_download
:param url: 待下载文件的url
:param dest: 下载时本地文件名,带路径
:return:
"""
if os.path.exists(dest):
LOG.info('[{0}] exist'.format(os.path.basename(dest)))
os.remove(dest)
LOG.info('[{0}] deleted'.format(os.path.basename(dest)))
DOWNLOAD_INST.download(url, dest)
def download_other_packages(dst=None):
"""
download_other_packages
:return:
"""
if dst is None:
base_dir = PROJECT_DIR
else:
base_dir = dst
resources_json = os.path.join(CUR_DIR, 'other_resources.json')
results = {'ok': [], 'failed': []}
with open(resources_json, 'r', encoding='utf-8') as json_file:
data = json.load(json_file)
for item in data:
dest_file = os.path.join(base_dir, item['dest'], item['filename'])
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
LOG.info('download[{0}] -> [{1}]'.format(
item['url'], os.path.basename(dest_file)))
if DOWNLOAD_INST.download(item['url'], dest_file):
results['ok'].append(item['filename'])
print(item['filename'].ljust(60), 'download success')
continue
results['failed'].append(item['filename'])
return results
def get_simple_index(self, distribution):
"""
get_simple_index
:param distribution:
:return:
"""
if distribution in self.cache.keys():
index = self.cache.get(distribution)
else:
url = '{0}/{1}/'.format(self.pypi_url, distribution.lower())
LOG.info('pypi URL = [{0}]'.format(url))
index = ''
for retry in [x + 1 for x in range(5)]:
success = False
try:
resp = DOWNLOAD_INST.urlopen(url)
index = resp.read()
success = True
except http.client.HTTPException as e:
print(e)
LOG.error(e)
time.sleep(2 * retry)
finally:
pass
if success:
break
self.cache[distribution] = index
parser = SimpleIndexParser()
parser.feed(index.decode())
idx = parser.get_index()
return idx
def download_by_url(self, pkg, dst_dir):
"""
download_by_url
:param pkg: package information
:return:
"""
download_dir = dst_dir
if 'dst_dir' in pkg:
download_dir = os.path.join(os.path.dirname(dst_dir), pkg['dst_dir'])
if not os.path.exists(download_dir):
os.makedirs(download_dir, mode=0o750, exist_ok=True)
url = pkg['url']
file_name = os.path.basename(url)
dst_file = os.path.join(download_dir, file_name)
checksum = pkg['sha256'] if 'sha256' in pkg else None
if checksum and not self.need_download_again(checksum, dst_file):
print(file_name.ljust(60), 'exists')
return True
try:
LOG.info('download from [%s]', url)
return DOWNLOAD_INST.download(url, dst_file)
except HTTPError as http_error:
print('[{0}]->{1}'.format(url, http_error))
LOG.error('[%s]->[%s]', url, http_error)
return False
finally:
pass
def get_simple_index(self, distribution):
if distribution in self.cache.keys():
index = self.cache.get(distribution)
print('get from cache')
else:
url = '{0}/{1}'.format(self.pypi_url, distribution.lower())
print('pypi URL = [{0}]'.format(url))
resp = DOWNLOAD_INST.urlopen(url)
index = resp.read()
self.cache[distribution] = index
dom_tree = xml.dom.minidom.parseString(index)
collection = dom_tree.documentElement
idx = collection.getElementsByTagName('a')
return idx
def fetch_package_index(packages_url):
"""
fetch_package_index
:param packages_url:
:return:
"""
tmp_file = DOWNLOAD_INST.download_to_tmp(packages_url)
if tmp_file is False:
LOG.error('download %s failed', packages_url)
return False
with gzip.open(tmp_file) as resp:
html = resp.read()
os.unlink(tmp_file)
return html.decode('utf-8')
def download_software(software, dst, arch):
"""
下载软件的其他资源
"""
formal_name, version = get_software_name_version(software)
others = get_software_other(formal_name, version)
download_dir = os.path.join(dst, "resources",
"{0}_{1}".format(formal_name, version))
if not os.path.exists(download_dir):
os.makedirs(download_dir, mode=0o750, exist_ok=True)
LOG.info('item:{} save dir: {}'.format(software,
os.path.basename(download_dir)))
results = []
if formal_name == "CANN":
if arch == "x86_64" or arch == "aarch64":
others = (item for item in others if arch in item['filename'])
try:
for item in others:
dest_file = os.path.join(download_dir, item['filename'])
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
if file_hash == item['sha256']:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
ret = CANN_DOWNLOAD_INST.download(item['url'], dest_file)
if ret:
print(item['filename'].ljust(60), 'download success')
results.append(ret)
finally:
CANN_DOWNLOAD_INST.quit()
else:
for item in others:
dest_file = os.path.join(download_dir, item['filename'])
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
if file_hash == item['sha256']:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
ret = DOWNLOAD_INST.download(item['url'], dest_file)
if ret:
print(item['filename'].ljust(60), 'download success')
results.append(ret)
return all(results)
def download_file(url, dst_file):
"""
download_file
:param url:
:param dst_file:
:return:
"""
try:
LOG.info('download from [{0}]'.format(url))
if DOWNLOAD_INST.download(url, dst_file):
return True
return False
except HTTPError as http_error:
print('[{0}]->{1}'.format(url, http_error))
LOG.error('[{0}]->{1}'.format(url, http_error))
return False
finally:
pass
def download_specified_python(dst=None):
"""
download ascend_python_version=Python-3.7.5
:return:
"""
if dst is None:
base_dir = PROJECT_DIR
else:
base_dir = dst
specified_python = get_specified_python()
resources_json = os.path.join(CUR_DIR, 'python_version.json')
with open(resources_json, 'r', encoding='utf-8') as json_file:
data = json.load(json_file)
results = {'ok': [], 'failed': []}
for item in data:
if specified_python == item['filename'].rstrip('.tar.xz'):
dest_file = os.path.join(base_dir, item['dest'],
item['filename'])
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
break
LOG.info('download[{0}] -> [{1}]'.format(
item['url'], os.path.basename(dest_file)))
if DOWNLOAD_INST.download(item['url'], dest_file):
results['ok'].append(item['filename'])
print(item['filename'].ljust(60), 'download success')
break
results['failed'].append(item['filename'])
break
return results
def download_file(self, url, dst_file):
try:
print('download from [{0}]'.format(url))
DOWNLOAD_INST.download(url, dst_file)
except HTTPError as http_error:
print('[{0}]->{1}'.format(url, http_error))
def download_ms_whl(os_item, software, dst):
"""
下载软件的其他资源
"""
formal_name, version = get_software_name_version(software)
download_dir = os.path.join(dst, "resources", "{}".format(os_item))
results = []
if version in ["1.2.1", "1.1.1"]:
whl_list = get_software_mindspore(formal_name, os_item, version)
for item in whl_list:
dest_file = os.path.join(download_dir, item["dst_dir"],
os.path.basename(item['url']))
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
else:
LOG.info('{0} need download again'.format(
item['filename']))
ret = DOWNLOAD_INST.download(item['url'], dest_file)
if ret:
print(item['filename'].ljust(60), 'download success')
results.append(ret)
else:
os_item_split = os_item.split("_")
os_name, arch = "_".join(os_item_split[:2]), "_".join(
os_item_split[2:])
specified_python = get_specified_python()
implement_flag = "cp37"
if "Python-3.7" in specified_python:
implement_flag = "cp37"
if "Python-3.8" in specified_python:
implement_flag = "cp38"
if "Python-3.9" in specified_python:
implement_flag = "cp39"
if os_name == "Ubuntu_18.04":
whl_list = get_software_mindspore(formal_name,
"{}".format(os_item), version)
for item in whl_list:
if item.get('python', 'cp37') != implement_flag:
print(
"Try to get {} on {}, but it does not match {}".format(
item['filename'], item.get('python'),
implement_flag))
continue
dest_file = os.path.join(download_dir, "CPU",
os.path.basename(item['url']))
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
else:
LOG.info('{0} need download again'.format(
item['filename']))
ret = DOWNLOAD_INST.download(item['url'], dest_file)
if ret:
print(item['filename'].ljust(60), 'download success')
results.append(ret)
if os_name in [
"Ubuntu_18.04", "CentOS_7.6", "EulerOS_2.8",
"OpenEuler_20.03LTS", "Kylin_V10Tercel"
]:
whl_list = get_software_mindspore(formal_name,
"linux_{}".format(arch), version)
for item in whl_list:
if item.get('python', 'cp37') != implement_flag:
print(
"Try to get {} on {}, but it does not match {}".format(
item['filename'], item.get('python'),
implement_flag))
continue
dest_file = os.path.join(download_dir, "Ascend910",
os.path.basename(item['url']))
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
print(item['filename'].ljust(60), 'exists')
LOG.info('{0} no need download again'.format(
item['filename']))
continue
else:
LOG.info('{0} need download again'.format(
item['filename']))
ret = DOWNLOAD_INST.download(item['url'], dest_file)
if ret:
print(item['filename'].ljust(60), 'download success')
results.append(ret)
for item in whl_list:
if item.get('python', 'cp37') != implement_flag:
print(
"Try to get {} on {}, but it does not match {}".format(
item['filename'], item.get('python'),
implement_flag))
continue
A910_dest_file = os.path.join(download_dir, "Ascend910",
os.path.basename(item['url']))
if os.path.exists(A910_dest_file) and os_name in [
"Ubuntu_18.04", "CentOS_7.6", "EulerOS_2.8"
]:
dest_file = os.path.join(download_dir, "Ascend310",
os.path.basename(item['url']))
if os.path.exists(dest_file) and 'sha256' in item:
file_hash = calc_sha256(dest_file)
url_hash = item['sha256']
if file_hash == url_hash:
LOG.info('{0} exist, no need copy again'.format(
os.path.basename(dest_file)))
else:
LOG.info(
'{0} exist but not completed, need copy again'.
format(os.path.basename(dest_file)))
os.remove(dest_file)
shutil.copy(A910_dest_file, dest_file)
else:
parent_dir = os.path.dirname(dest_file)
if not os.path.exists(parent_dir):
os.makedirs(parent_dir, mode=0o750, exist_ok=True)
LOG.info('{0} not exist, copy from Ascend910'.format(
os.path.basename(dest_file)))
shutil.copy(A910_dest_file, dest_file)
return all(results)
def download_by_name(self, pkg, dst_dir):
"""
download
:param name:
:param dst_dir:
:return:
"""
if 'name' not in pkg:
return False
if 'dst_dir' in pkg:
dst_dir = os.path.join(dst_dir, pkg['dst_dir'])
url = None
name = pkg['name']
cur = self.primary_connection.cursor()
sql = 'SELECT packages.version, packages.url, packages.sha256, \
packages.source, packages.repo \
FROM packages \
WHERE name=:name ORDER by packages.version;'
param = {'name': name}
cur.execute(sql, param)
results = cur.fetchall()
cur.close()
if len(results) == 0:
print("can't find package {0}".format(name))
LOG.error("can't find package %s", name)
return False
pkg_sha256 = ''
version = results[0][0]
url = results[0][3] + results[0][1]
pkg_sha256 = results[0][2]
for item in results:
[cur_ver, cur_url, cur_sha256, cur_source, cur_repo] = item
if not self.version_compare(version, cur_ver):
version = cur_ver
url = cur_source + cur_url
pkg_sha256 = cur_sha256
if 'version' in pkg and pkg['version'] in cur_ver:
url = cur_source + cur_url
pkg_sha256 = cur_sha256
break
try:
LOG.info('[%s] download from [%s]', name, url)
file_name = os.path.basename(url)
dst_file = os.path.join(dst_dir, file_name)
if not self.need_download_again(pkg_sha256, dst_file):
LOG.info("%s no need download again", name)
print(name.ljust(60), 'exists')
return True
if DOWNLOAD_INST.download(url, dst_file):
print(name.ljust(60), 'download success')
return True
print(name.ljust(60), 'download failed')
return False
except HTTPError as http_error:
print('[{0}]->{1}'.format(url, http_error))
LOG.error('[%s]->[%s]', url, http_error)
return False
finally:
pass
def file_download(url, dest):
if os.path.exists(dest):
print('[{0}] exist'.format(dest))
else:
DOWNLOAD_INST.download(url, dest)