def download_software(software, dst, arch): """ 下载软件的其他资源 """ formal_name, version = get_software_name_version(software) others = get_software_other(formal_name, version) download_dir = os.path.join(dst, "resources", "{0}_{1}".format(formal_name, version)) if not os.path.exists(download_dir): os.makedirs(download_dir, mode=0o750, exist_ok=True) LOG.info('item:{} save dir: {}'.format(software, os.path.basename(download_dir))) results = [] if formal_name == "CANN": if arch == "x86_64" or arch == "aarch64": others = (item for item in others if arch in item['filename']) try: for item in others: dest_file = os.path.join(download_dir, item['filename']) if os.path.exists(dest_file) and 'sha256' in item: file_hash = calc_sha256(dest_file) if file_hash == item['sha256']: print(item['filename'].ljust(60), 'exists') LOG.info('{0} no need download again'.format( item['filename'])) continue ret = CANN_DOWNLOAD_INST.download(item['url'], dest_file) if ret: print(item['filename'].ljust(60), 'download success') results.append(ret) finally: CANN_DOWNLOAD_INST.quit() else: for item in others: dest_file = os.path.join(download_dir, item['filename']) if os.path.exists(dest_file) and 'sha256' in item: file_hash = calc_sha256(dest_file) if file_hash == item['sha256']: print(item['filename'].ljust(60), 'exists') LOG.info('{0} no need download again'.format( item['filename'])) continue ret = DOWNLOAD_INST.download(item['url'], dest_file) if ret: print(item['filename'].ljust(60), 'download success') results.append(ret) return all(results)
def download_other_packages(dst=None): """ download_other_packages :return: """ if dst is None: base_dir = PROJECT_DIR else: base_dir = dst resources_json = os.path.join(CUR_DIR, 'other_resources.json') results = {'ok': [], 'failed': []} with open(resources_json, 'r', encoding='utf-8') as json_file: data = json.load(json_file) for item in data: dest_file = os.path.join(base_dir, item['dest'], item['filename']) if os.path.exists(dest_file) and 'sha256' in item: file_hash = calc_sha256(dest_file) url_hash = item['sha256'] if file_hash == url_hash: print(item['filename'].ljust(60), 'exists') LOG.info('{0} no need download again'.format( item['filename'])) continue LOG.info('download[{0}] -> [{1}]'.format( item['url'], os.path.basename(dest_file))) if DOWNLOAD_INST.download(item['url'], dest_file): results['ok'].append(item['filename']) print(item['filename'].ljust(60), 'download success') continue results['failed'].append(item['filename']) return results
def need_download_again(dst_file, url_with_hash): """ need_download_again 校验目的文件的hash值与url中的hash值是否相等,来决定是否重新下载 :param dst_file: 目的文件 :param url_with_hash: 带hash值的URL :return: """ if url_with_hash is None or len(url_with_hash) == 0: return True if not os.path.exists(dst_file): return True key_word = '' file_hash = '' if 'sha256=' in url_with_hash: key_word = 'sha256=' file_hash = calc_sha256(dst_file) else: return True index_of_hash = str(url_with_hash).index(key_word) + len(key_word) target_hash = url_with_hash[index_of_hash:] if target_hash != file_hash: LOG.info('hash {0} in url: {1} != file: {2}'.format( key_word, target_hash, file_hash)) return target_hash != file_hash
def need_download_again(target_sha256, dst_file): """ need_download_again :param target_sha256: :param dst_file: :return: """ if target_sha256 is None: return True if not os.path.exists(dst_file): return True file_sha256 = calc_sha256(dst_file) if target_sha256 != file_sha256: LOG.info('target sha256 : {}, exists file sha256 : {}'.format( target_sha256, file_sha256)) return True else: return False
def download_specified_python(dst=None): """ download ascend_python_version=Python-3.7.5 :return: """ if dst is None: base_dir = PROJECT_DIR else: base_dir = dst specified_python = get_specified_python() resources_json = os.path.join(CUR_DIR, 'python_version.json') with open(resources_json, 'r', encoding='utf-8') as json_file: data = json.load(json_file) results = {'ok': [], 'failed': []} for item in data: if specified_python == item['filename'].rstrip('.tar.xz'): dest_file = os.path.join(base_dir, item['dest'], item['filename']) if os.path.exists(dest_file) and 'sha256' in item: file_hash = calc_sha256(dest_file) url_hash = item['sha256'] if file_hash == url_hash: print(item['filename'].ljust(60), 'exists') LOG.info('{0} no need download again'.format( item['filename'])) break LOG.info('download[{0}] -> [{1}]'.format( item['url'], os.path.basename(dest_file))) if DOWNLOAD_INST.download(item['url'], dest_file): results['ok'].append(item['filename']) print(item['filename'].ljust(60), 'download success') break results['failed'].append(item['filename']) break return results
def download_ms_whl(os_item, software, dst): """ 下载软件的其他资源 """ formal_name, version = get_software_name_version(software) download_dir = os.path.join(dst, "resources", "{}".format(os_item)) results = [] if version in ["1.2.1", "1.1.1"]: whl_list = get_software_mindspore(formal_name, os_item, version) for item in whl_list: dest_file = os.path.join(download_dir, item["dst_dir"], os.path.basename(item['url'])) if os.path.exists(dest_file) and 'sha256' in item: file_hash = calc_sha256(dest_file) url_hash = item['sha256'] if file_hash == url_hash: print(item['filename'].ljust(60), 'exists') LOG.info('{0} no need download again'.format( item['filename'])) continue else: LOG.info('{0} need download again'.format( item['filename'])) ret = DOWNLOAD_INST.download(item['url'], dest_file) if ret: print(item['filename'].ljust(60), 'download success') results.append(ret) else: os_item_split = os_item.split("_") os_name, arch = "_".join(os_item_split[:2]), "_".join( os_item_split[2:]) specified_python = get_specified_python() implement_flag = "cp37" if "Python-3.7" in specified_python: implement_flag = "cp37" if "Python-3.8" in specified_python: implement_flag = "cp38" if "Python-3.9" in specified_python: implement_flag = "cp39" if os_name == "Ubuntu_18.04": whl_list = get_software_mindspore(formal_name, "{}".format(os_item), version) for item in whl_list: if item.get('python', 'cp37') != implement_flag: print( "Try to get {} on {}, but it does not match {}".format( item['filename'], item.get('python'), implement_flag)) continue dest_file = os.path.join(download_dir, "CPU", os.path.basename(item['url'])) if os.path.exists(dest_file) and 'sha256' in item: file_hash = calc_sha256(dest_file) url_hash = item['sha256'] if file_hash == url_hash: print(item['filename'].ljust(60), 'exists') LOG.info('{0} no need download again'.format( item['filename'])) continue else: LOG.info('{0} need download again'.format( item['filename'])) ret = DOWNLOAD_INST.download(item['url'], dest_file) if ret: print(item['filename'].ljust(60), 'download success') results.append(ret) if os_name in [ "Ubuntu_18.04", "CentOS_7.6", "EulerOS_2.8", "OpenEuler_20.03LTS", "Kylin_V10Tercel" ]: whl_list = get_software_mindspore(formal_name, "linux_{}".format(arch), version) for item in whl_list: if item.get('python', 'cp37') != implement_flag: print( "Try to get {} on {}, but it does not match {}".format( item['filename'], item.get('python'), implement_flag)) continue dest_file = os.path.join(download_dir, "Ascend910", os.path.basename(item['url'])) if os.path.exists(dest_file) and 'sha256' in item: file_hash = calc_sha256(dest_file) url_hash = item['sha256'] if file_hash == url_hash: print(item['filename'].ljust(60), 'exists') LOG.info('{0} no need download again'.format( item['filename'])) continue else: LOG.info('{0} need download again'.format( item['filename'])) ret = DOWNLOAD_INST.download(item['url'], dest_file) if ret: print(item['filename'].ljust(60), 'download success') results.append(ret) for item in whl_list: if item.get('python', 'cp37') != implement_flag: print( "Try to get {} on {}, but it does not match {}".format( item['filename'], item.get('python'), implement_flag)) continue A910_dest_file = os.path.join(download_dir, "Ascend910", os.path.basename(item['url'])) if os.path.exists(A910_dest_file) and os_name in [ "Ubuntu_18.04", "CentOS_7.6", "EulerOS_2.8" ]: dest_file = os.path.join(download_dir, "Ascend310", os.path.basename(item['url'])) if os.path.exists(dest_file) and 'sha256' in item: file_hash = calc_sha256(dest_file) url_hash = item['sha256'] if file_hash == url_hash: LOG.info('{0} exist, no need copy again'.format( os.path.basename(dest_file))) else: LOG.info( '{0} exist but not completed, need copy again'. format(os.path.basename(dest_file))) os.remove(dest_file) shutil.copy(A910_dest_file, dest_file) else: parent_dir = os.path.dirname(dest_file) if not os.path.exists(parent_dir): os.makedirs(parent_dir, mode=0o750, exist_ok=True) LOG.info('{0} not exist, copy from Ascend910'.format( os.path.basename(dest_file))) shutil.copy(A910_dest_file, dest_file) return all(results)