def media_ts_format(ts: str = None, media: str = None) -> str:
""" invoked by build_media_files_from_list() - check and format 'ts' """
ts_month = ts.split(" ")[0]
if len(ts_month) == 3: # if month is "Dec" (3 letters)"
try:
media_ts = (
datetime.strptime(ts, "%b %d, %Y").date().strftime("%Y%m%d")
)
return media_ts
except Exception as e:
logger.warning(
f'"{media}" ts format is not supported. Skipping it.'
)
logger.debug(f"message was: {e}")
return False
elif len(ts_month) > 3: # if month is "December"
try:
media_ts = (
datetime.strptime(ts, "%B %d, %Y").date().strftime("%Y%m%d")
)
return media_ts
except Exception as e:
logger.warning(
f'"{media}" ts format is not supported. Skipping it.'
)
logger.debug(f"message was: {e}")
return False
else:
return False
def exists(object_path):
if os.path.exists(object_path):
return True
else:
msg = 'File object %s failed to download' % (object_path)
logger.warning(msg)
stdout_message('%s: %s' % (inspect.stack()[0][3], msg))
return False
def get_s3_files(
bucket_name: str = BUCKET_NAME,
save_to_disk: bool = True,
files_list_path: str = FILES_LIST_PATH,
files_list_filename: str = FILES_LIST_FILENAME,
aws_region: str = AWS_REGION,
s3_prefix: str = S3_PREFIX,
) -> list:
""" Get S3 objects and creates list """
logger.info("Building media list from S3 objects...")
logger.debug(
f"Context Parameters: {get_s3_files.__name__} => {get_s3_files.__code__.co_varnames}"
)
data = []
# testing format: assets/20160823/img.jpg
pattern = re.compile(
"^[a-z-A-Z-0-9]+/[a-z-A-Z-0-9]+/[0-9]{8}/.+[.][a-z-A-Z-0-9]+$"
)
try:
s3 = boto3.client("s3", region_name=aws_region)
paginator = s3.get_paginator("list_objects_v2")
pages = paginator.paginate(Bucket=bucket_name, Prefix=s3_prefix)
for page in pages:
for obj in page["Contents"]:
if pattern.match(obj["Key"]):
data.append(obj["Key"])
else:
logger.warning(
f'Wrong filename format, object "{obj["Key"]}", not added to the list.'
)
statistics.append(["get_s3_files", len(data)])
logger.info("Media Objects list generated successfully.")
logger.debug(f"Media objects count: {len(data)}.")
if save_to_disk:
logger.info("Writing media list to disk...")
export_data = [f"{item}\n" for item in data]
with open(f"{files_list_path}/{files_list_filename}", "w") as w:
w.writelines(export_data)
logger.info(
f'List successfully saved to disk: "{files_list_path}/{files_list_filename}".'
)
else:
pass
except Exception as e:
logger.error(e)
raise
return data
def create_table(
table_name: str = TABLE_NAME,
ReadCapacityUnits: int = TABLE_READ_CAPACITY_UNITS,
WriteCapacityUnits: int = TABLE_WRITE_CAPACITY_UNITS,
aws_region: str = AWS_REGION,
) -> bool:
""" Creates DynamoB table """
try:
client = boto3.client("dynamodb", region_name=aws_region)
response = client.list_tables()
tables = [
table for table in response["TableNames"] if table == table_name
]
if len(tables) > 0:
logger.warning(
f'Table "{table_name}" already exists. Skipping table creation.'
)
return False
else:
logger.info(
f'Table "{table_name}" does not exist. Starting creation process...'
)
except Exception as e:
logger.error(e)
raise
logger.info("Creating DB table...")
logger.debug(
f"Context Parameters: {create_table.__name__} => {create_table.__code__.co_varnames}"
)
try:
dynamodb = boto3.resource("dynamodb", region_name=aws_region)
table = dynamodb.create_table(
TableName=table_name,
AttributeDefinitions=[
{"AttributeName": "ts", "AttributeType": "S"}
],
KeySchema=[{"AttributeName": "ts", "KeyType": "HASH"}],
ProvisionedThroughput={
"ReadCapacityUnits": int(ReadCapacityUnits),
"WriteCapacityUnits": int(WriteCapacityUnits),
},
)
logger.info("Table created successfully.")
logger.debug(table)
except dynamodb.exceptions.ResourceInUseException as e:
logger.warning(
f'Table "{table_name}" already exists. Skipping table creation.'
)
logger.debug(e)
return False
return True
def detect_cudnn_win():
if SysInfo.cuda == "8.0":
required_cndunn = {'6': 'cudnn64_6.dll', '7': 'cudnn64_7.dll'}
else:
required_cndunn = {'7': 'cudnn64_7.dll'}
cmd = r"C:\Windows\System32\where.exe"
for version, dll in required_cndunn.items():
args = [dll]
status, cudnn = _run_cmd(cmd, args, True)
if status and next(filter(os.path.isfile, cudnn.split('\n')), None):
SysInfo.cudnn = version
logger.info("cuDNN: {0}".format(version))
if not SysInfo.cudnn:
logger.warning("Not detect cuDNN! We recommand cuDNN 7, please download and install cuDNN 7 from https://developer.nvidia.com/rdp/cudnn-download.")
def detect_mpi_win():
target_version = "7.0.12437.6"
mpi_path = _registry_read(winreg.HKEY_LOCAL_MACHINE, r"Software\Microsoft\MPI", "InstallRoot")
if (mpi_path and os.path.isfile(os.path.sep.join([mpi_path, "bin", "mpiexec.exe"]))):
SysInfo.mpi = _registry_read(winreg.HKEY_LOCAL_MACHINE, r"Software\Microsoft\MPI", "Version")
if SysInfo.mpi:
logger.info("MPI: {0}".format(SysInfo.mpi))
if not _version_compare(target_version, SysInfo.mpi):
logger.warning("CNTK suggests MPI version to be {0}, please manually upgrade MPI.".format(target_version))
return False
return True
else:
logger.warning("Not detect MPI, please manually download and isntall MPI.")
return False
def detect_cuda():
if (SysInfo.os == TOOLSFORAI_OS_WIN or SysInfo.os == TOOLSFORAI_OS_LINUX):
# return detect_cuda_()
status, stdout = _run_cmd("nvcc", ["-V"], True)
if status and re.search(r"release\s*8.0,\s*V8.0", stdout):
SysInfo.cuda = "8.0"
logger.info("CUDA: {0}".format(SysInfo.cuda))
if SysInfo.cuda80:
logger.warning(
"Detect parameter '--cuda80', the installer script will be forced to install dependency package for CUDA 8.0.")
return True
else:
logger.warning("We recommend CUDA 9.0 (https://developer.nvidia.com/cuda-toolkit)."
"If you want to install dependency package for CUDA 8.0, please run the installer script with '--cuda80' again.")
return False
elif status and re.search(r"release\s*9.0,\s*V9.0", stdout):
SysInfo.cuda = "9.0"
logger.info("CUDA: {0}".format(SysInfo.cuda))
else:
SysInfo.cuda = "9.0"
logger.warning("Not detect CUDA! We recommend CUDA 9.0 (https://developer.nvidia.com/cuda-toolkit). "
"The installer script will install dependency package for CUDA 9.0 by default.")
if SysInfo.cuda80:
SysInfo.cuda = "8.0"
logger.warning(
"Detect parameter '--cuda80', the installer script will be forced to install dependency package for CUDA 8.0.")
return True
else:
return True
def detect_vs():
vs = []
vs_2015_path = _registry_read(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\WOW6432Node\Microsoft\VisualStudio\14.0",
"InstallDir")
if (vs_2015_path and os.path.isfile(os.path.join(vs_2015_path, "devenv.exe"))):
vs.append("VS2015")
vs_2017_path = _registry_read(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\WOW6432Node\Microsoft\VisualStudio\SxS\VS7",
"15.0")
if (vs_2017_path and os.path.isfile(os.path.sep.join([vs_2017_path, "Common7", "IDE", "devenv.exe"]))):
vs.append("VS2017")
if (len(vs) == 0):
logger.warning("Not detect Visual Studio 2017 or 2015! We recommend Visual Studio 2017, "
"please manually download and install Visual Studio 2017 form https://www.visualstudio.com/downloads/.")
else:
logger.info("Visual Studio: {0}".format(" ".join(vs)))
def pip_install_caffe2(pkg_info, options):
if not (SysInfo.os == TOOLSFORAI_OS_WIN):
logger.warning(
"Fail to install caffe2. In non-Windows OS, you should manually install caffe2 from source."
)
return
name = pkg_info["caffe2"]["name"]
version = pkg_info["caffe2"]["version"]
arch = "win_amd64"
wheel_ver = SysInfo.python
if SysInfo.gpu and SysInfo.cuda == "8.0":
pkg = "https://raw.githubusercontent.com/linmajia/ai-package/master/caffe2/{0}/caffe2_gpu-{0}-cp{1}-cp{1}m-{2}.whl".format(
version, wheel_ver, arch)
else:
pkg = "https://raw.githubusercontent.com/linmajia/ai-package/master/caffe2/{0}/caffe2-{0}-cp{1}-cp{1}m-{2}.whl".format(
version, wheel_ver, arch)
return pip_install_package(name, options, version, pkg)
def get_media_type(
file_name: str = None,
supported_pictures_formats: tuple = SUPPORTED_PICTURE_FORMATS,
supported_movies_formats: tuple = SUPPORTED_MOVIE_FORMATS,
) -> str:
""" Returns the media type of the input filename """
pictures = [(item, item.upper()) for item in supported_pictures_formats]
movies = [(item, item.upper()) for item in supported_movies_formats]
if list(filter(file_name.endswith, pictures)) != []:
media_type = "picture"
elif list(filter(file_name.endswith, movies)) != []:
media_type = "movie"
else:
media_type = "UNSUPPORTED"
logger.warning(f"Type unsupported: \"{file_name.split('.')[-1]}\".")
return media_type
def create_s3_bucket(
bucket_name: str = BUCKET_NAME, aws_region: str = AWS_REGION
) -> bool:
""" Create the S3 bucket of the project """
s3 = boto3.client("s3", region_name=aws_region)
bucket_exists = True
try:
response = response = s3.list_buckets()
buckets = [
bucket["Name"]
for bucket in response["Buckets"]
if bucket["Name"] == bucket_name
]
if len(buckets) > 0:
logger.warning(
"S3 bucket already exists. Skipping bucket creation."
)
bucket_exists = True
else:
bucket_exists = False
except Exception as e:
logger.error(e)
raise
if not bucket_exists:
try:
response = s3.create_bucket(
Bucket=bucket_name,
ACL="private",
CreateBucketConfiguration={"LocationConstraint": aws_region},
)
logger.info(f'Created S3 bucket "{bucket_name}" successfully.')
logger.debug(f"S3 client response: {response}")
except Exception as e:
logger.error(e)
raise
else:
return False
return True
def detect_visualcpp_runtime_win():
pattern = re.compile(
"(^Microsoft Visual C\+\+ 201(5|7) x64 Additional Runtime)|(^Microsoft Visual C\+\+ 201(5|7) x64 Minimum Runtime)")
items = [(winreg.HKEY_CURRENT_USER, r"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall"),
(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall"),
(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall")]
for hkey, keypath in items:
try:
current_key = winreg.OpenKey(hkey, keypath)
for subkey in _registry_subkeys(hkey, keypath):
display_name = _registry_read(current_key, subkey, "DisplayName")
if (display_name and pattern.match(display_name)):
logger.info("Detect Visual C++ runtime already installed.")
return True
winreg.CloseKey(current_key)
except WindowsError:
pass
logger.warning("Not detect Visual C++ runtime.")
return False
def medias_copy(
local_path: str = LOCAL_MEDIA_OUTPUT_PATH,
video_encode: bool = VIDEO_ENCODE,
media_encode_platform: str = MEDIA_ENCODE_PLATFORM,
) -> bool:
""" Copy media files to S3 """
logger.info("Starting copy...")
try:
medias = get_local_medias_files(path=local_path, save_to_disk=False)
logger.debug(medias)
for media in medias:
media_type = get_media_type(basename(media))
ts = media.split("/")[-2]
if media_type == "movie":
if video_encode == "True" and media_encode_platform == "cloud":
send_to_bucket(media, ts)
elif (
video_encode == "True" and media_encode_platform == "local"
):
logger.info(
f"Skipping copy of {media} for local re-encoding."
)
elif media_type == "picture":
send_to_bucket(media, ts)
else:
logger.warning(f"Media type is: {media_type} !")
logger.info(
f"{len(medias)} medias files have been successfully copied."
)
except Exception as e:
logger.error(e)
raise
statistics.append(["medias_copy", len(medias)])
logger.info("...done.")
return True
def install_cntk_win(cntk_root):
suc = True
try:
utils._update_pathenv_win(os.path.join(cntk_root, "cntk"), True)
if (not utils.detect_mpi_win()):
mpi_exe = os.path.sep.join(
[cntk_root, "prerequisites", "MSMpiSetup.exe"])
logger.debug("MPI exe path: %s" % mpi_exe)
logger.info("Begin to install MPI ...")
utils._run_cmd_admin(mpi_exe, "-unattend")
if (utils.detect_mpi_win()):
logger.info("Install MPI successfully.")
else:
suc = False
logger.error(
"Fail to install MPI. Please manually install MPI >= 7.0.12437.6"
)
if (not utils.detect_visualcpp_runtime_win()):
vc_redist_exe = os.path.sep.join(
[cntk_root, "prerequisites", "VS2015", "vc_redist.x64.exe"])
logger.debug("VC redist exe path: {0}".format(vc_redist_exe))
logger.info("Begin to install Visual C++ runtime ...")
utils._run_cmd_admin(vc_redist_exe, "/install /norestart /passive")
if (utils.detect_visualcpp_runtime_win()):
logger.info("Install Visual C++ runtime successfully.")
logger.warning(
" Please manually install Visual C++ Redistributable Package for Visual Studio 2015 or 2017."
)
else:
suc = False
logger.error("Fail to install Visual C++ runtime.")
except:
suc = False
logger.error(
"Fail to install CNTK(BrainScript). The error massage: {0}".format(
sys.exc_info()))
return suc
def detect_os():
os_name = platform.platform(terse=True)
os_bit = platform.architecture()[0]
is_64bit = (os_bit == "64bit")
logger.info("OS: {0}, {1}".format(os_name, os_bit))
if (os_name.startswith("Windows")):
SysInfo.os = TOOLSFORAI_OS_WIN
if not os_name.startswith("Windows-10"):
logger.warning(
"We recommend Windows 10 as the primary development OS, other Windows versions are not fully supported.")
elif (os_name.startswith("Linux")):
SysInfo.os = TOOLSFORAI_OS_LINUX
elif (os_name.startswith("Darwin")):
SysInfo.os = TOOLSFORAI_OS_MACOS
is_64bit = sys.maxsize > 2 ** 32
else:
logger.error("Your OS({0}-{1}) can't be supported! Only Windows, Linux and MacOS can be supported now.".format(os_name, os_bit))
return False
if not is_64bit:
logger.error("Your OS is not 64-bit OS. Now only 64-bit OS is supported.")
return False
return True
def pip_install_ml_software(pkg_info, options):
logger.info(
"Begin to install ml software(scikit-learn, xgboost and libsvm) ...")
#1 scikit-learn
name = pkg_info["ml_software"]["scikit-learn"]["name"]
version = pkg_info["ml_software"]["scikit-learn"]["version"]
pip_install_package(name, options, version)
#2 xgboost
name = pkg_info["ml_software"]["xgboost"]["name"]
version = pkg_info["ml_software"]["xgboost"]["version"]
if SysInfo.os != TOOLSFORAI_OS_WIN:
if not pip_install_package(name, options, version):
logger.warning(
"In order to install xgboost, C++ compiler is needed.")
else:
wheel_ver = SysInfo.python
arch = "win_amd64"
pkg = "https://raw.githubusercontent.com/linmajia/ai-package/master/xgboost/{0}/xgboost-{0}-cp{1}-cp{1}m-{2}.whl".format(
version, wheel_ver, arch)
pip_install_package(name, options, version, pkg)
#3 libsvm
name = pkg_info["ml_software"]["libsvm"]["name"]
version = pkg_info["ml_software"]["libsvm"]["version"]
if SysInfo.os != TOOLSFORAI_OS_WIN:
logger.warning(
"Fail to install libsvm. On Linux or Mac, in order to install {0}=={1}, please manually download source code and install it."
.format(name, version))
else:
wheel_ver = SysInfo.python
arch = "win_amd64"
pkg = "https://raw.githubusercontent.com/linmajia/ai-package/master/libsvm/{0}/libsvm-{0}-cp{1}-cp{1}m-{2}.whl".format(
version, wheel_ver, arch)
logger.debug("Pip install libsvm from {0}".format(pkg))
pip_install_package(name, options, version, pkg)
def pip_install_pytorch(pkg_info, options):
name = pkg_info["pytorch"]["torch"]["name"]
version = pkg_info["pytorch"]["torch"]["version"]
if SysInfo.os == TOOLSFORAI_OS_MACOS:
pip_install_package(name, options, version)
elif SysInfo.os == TOOLSFORAI_OS_WIN or SysInfo.os == TOOLSFORAI_OS_LINUX:
wheel_ver = SysInfo.python
arch = "win_amd64" if SysInfo.os == TOOLSFORAI_OS_WIN else "linux_x86_64"
if not SysInfo.gpu:
gpu_type = "cpu"
else:
gpu_type = "cu80" if SysInfo.cuda == "8.0" else "cu90"
pkg = "http://download.pytorch.org/whl/{0}/{1}-{2}-cp{3}-cp{3}m-{4}.whl".format(
gpu_type, name, version, wheel_ver, arch)
pip_install_package(name, options, version, pkg)
else:
logger.error("Fail to install pytorch.")
logger.warning(
"Pytorch installation can not be supported on your OS! We recommand 64-bit Windows-10, Linux and Macos."
)
name = pkg_info["pytorch"]["torchvision"]["name"]
version = pkg_info["pytorch"]["torchvision"]["version"]
pip_install_package(name, options, version)
def install_cntk_linux(cntk_root):
logger.warning(
"CNTK(BrainScript) V2 on Linux requires C++ Compiler and Open MPI. "
"Please refer to https://docs.microsoft.com/en-us/cognitive-toolkit/Setup-Linux-Binary-Manual"
)
bashrc_file_path = os.path.sep.join([os.path.expanduser('~'), '.bashrc'])
content = ''
with open(bashrc_file_path, 'r') as bashrc_file:
content = bashrc_file.read()
with open(bashrc_file_path, 'a+') as bashrc_file:
CNTK_PATH = '{0}/cntk/bin'.format(cntk_root)
CNTK_PATH_EXPORT = 'export PATH={0}:$PATH'.format(CNTK_PATH)
if not (CNTK_PATH_EXPORT in content):
bashrc_file.write('{0}\n'.format(CNTK_PATH_EXPORT))
CNTK_LD_LIBRARY_PATH = '{0}/cntk/lib:{0}/cntk/dependencies/lib'.format(
cntk_root)
CNTK_LD_LIBRARY_PATH_EXPORT = 'export LD_LIBRARY_PATH={0}:$LD_LIBRARY_PATH'.format(
CNTK_LD_LIBRARY_PATH)
if not (CNTK_LD_LIBRARY_PATH_EXPORT in content):
bashrc_file.write('{0}\n'.format(CNTK_LD_LIBRARY_PATH_EXPORT))
return True
def pip_install_chainer(pkg_info, options):
# cupy installation for GPU linux
logger.info("Begin to install chainer(cupy, chainer, chainermn) ...")
version = pkg_info["chainer"]["cupy"]["version"]
if (SysInfo.gpu and (SysInfo.os == TOOLSFORAI_OS_LINUX)):
if SysInfo.cuda == "8.0":
name = pkg_info["chainer"]["cupy"]["name"]["linux"]["cuda80"]
elif SysInfo.cuda == "9.0":
name = pkg_info["chainer"]["cupy"]["name"]["linux"]["cuda90"]
else:
name = pkg_info["chainer"]["cupy"]["name"]["linux"]["other"]
pip_install_package(name, options)
elif (SysInfo.gpu and (SysInfo.os == TOOLSFORAI_OS_WIN)):
try:
name = pkg_info["chainer"]["cupy"]["name"]["win"]
cupy = importlib.import_module(name)
if (not utils._version_compare(version, cupy.__version__)):
logger.warning(
"Cupy's version is too low, please manually upgrade cupy >= 2.0.0."
)
else:
logger.info("cupy is already installed.")
except ImportError:
logger.warning(
"On windows, please manully install cupy. You can reference this link https://github.com/Microsoft/vs-tools-for-ai/blob/master/docs/prepare-localmachine.md#chainer."
)
name = pkg_info["chainer"]["chainer"]["name"]
version = pkg_info["chainer"]["chainer"]["version"]
pip_install_package(name, options, version)
name = pkg_info["chainer"]["chainermn"]["name"]
version = pkg_info["chainer"]["chainermn"]["version"]
if not pip_install_package(name, options, version):
logger.warning(
"On Linux, in order to install chainermn, please first manually install libmpich-dev and then run installer script again."
)
def media_generate(
media: str = None,
output_path: str = None,
media_ts: str = None,
output_image_width: int = None,
output_image_height: int = None,
processed_files_count: int = None,
unprocessed_files: list = None,
video_encode: str = VIDEO_ENCODE,
log_path: str = LOG_PATH,
s3_prefix: str = S3_PREFIX,
media_encode_platform: str = MEDIA_ENCODE_PLATFORM,
) -> list:
""" invoked by build_media_files_from_list() - gemerates media files """
media_name = media.split("/")[-1]
media_type = get_media_type(media_name)
if not os.path.exists(f"{output_path}/{media_ts}"):
os.mkdir(f"{output_path}/{media_ts}")
logger.debug(f'Created directory: "{output_path}/{media_ts}".')
else:
pass
if media_type == "picture":
logger.info(
f"Picture type identified, starting generation of media...")
image = Image.open(media)
if image:
with image as im:
im.thumbnail((output_image_width, output_image_height))
im.save(
f"{output_path}/{media_ts}/{media_name}",
format="JPEG",
quality="web_high",
dpi=(72, 72),
)
processed_files_count += 1
logger.info(
f'Generated media: "{output_path}/{media_ts}/{media_name}".')
else:
logger.warning(
f'Impossible to open the image file: "{media_name}"! File identified format is : "{media_type}". Skipping it.'
)
elif media_type == "movie":
if video_encode:
logger.info(f"Movie type identified...")
if media_encode_platform == "local":
if os.path.exists(f"{log_path}/ffmpeg.log"):
with open(f"{log_path}/ffmpeg.log", "r+") as w:
w.truncate(0)
else:
pass
video_encoder(media, media_ts, output_path)
elif media_encode_platform == "cloud":
logger.info(f"Movie type identified, starting copy of file...")
shutil.copyfile(media,
f"{output_path}/{media_ts}/{media_name}")
logger.info(
f'File copied successfully: "{media}" => "{output_path}/{media_ts}/{media_name}"'
)
movie = f"{s3_prefix}/{media_ts}/{media_name}"
cloud_video_encoder_list.append({
"src": movie,
"ts": media_ts,
"delete_old": True
})
logger.info(
f"Added movie '{movie}' to queue for defered remote re-encoding."
)
else:
logger.critical(
'Wrong or missing value! Valid values for "media_encode_platform": local|cloud'
)
else:
logger.info(f"Movie type identified, starting copy of file...")
shutil.copyfile(media, f"{output_path}/{media_ts}/{media_name}")
logger.info(
f'File copied successfully: "{media}" => "{output_path}/{media_ts}/{media_name}"'
)
processed_files_count += 1
else:
unprocessed_files.append(media)
logger.warning(f'Impossible to process file: "{media}". Skipping it.')
return (processed_files_count, unprocessed_files)
def install_cntk(target_dir):
logger.info("Begin to install CNTK(BrainScript) ...")
if SysInfo.os != TOOLSFORAI_OS_WIN and SysInfo.os != TOOLSFORAI_OS_LINUX:
logger.warning(
"CNTK(BrainScript) is not supported on your OS, we recommend 64-bit Windows-10 OS or 64-bit Linux OS."
)
# fail_install.append("CNTK(BrainScript)")
return False
if SysInfo.cuda == "8.0":
ver = "2.3.1"
else:
ver = "2.5.1"
target_version = 'CNTK-{0}'.format(ver.replace('.', '-'))
logger.debug(
"In install_cntk(), target_version: {0}".format(target_version))
version = utils._get_cntk_version(target_dir)
if target_version == version:
logger.info('CNTK(BrainScript)-{0} is already installed.'.format(ver))
return True
logger.debug('In install_cntk(), target_dir: {0}'.format(target_dir))
cntk_root = os.path.join(target_dir, 'cntk')
if os.path.isdir(cntk_root):
try:
shutil.rmtree(cntk_root)
except:
logger.error(
'Fail to install CNTK(BrainScript), the error message: can not remove old version in directory {0}.'
'Please manually remove old version, and run the installer script again.'
.format(cntk_root))
# fail_install.append("CNTK(BrainScript)")
return False
if not os.path.isdir(target_dir):
try:
os.makedirs(target_dir)
except:
logger.error(
'Fail to install CNTK(BrainScript), the error message: can not create directory {0}.'
'Please check if there is permission for creating directory.'.
format(target_dir))
# fail_install.append("CNTK(BrainScript)")
return False
cntk_file_name = "{}-{}-64bit-{}.{}".format(
target_version,
"Windows" if SysInfo.os == TOOLSFORAI_OS_WIN else "Linux",
"GPU" if SysInfo.gpu else "CPU-Only",
"zip" if SysInfo.os == TOOLSFORAI_OS_WIN else "tar.gz")
logger.debug(
"In install_cntk(), cntk_file_name: {0}".format(cntk_file_name))
cntk_url = "https://cntk.ai/BinaryDrop/{0}".format(cntk_file_name)
logger.debug("In install_cntk(), cntk_url: {0}".format(cntk_url))
cntk_file_path = os.path.join(target_dir, cntk_file_name)
logger.debug(
"In install_cntk(), cntk_file_path: {0}".format(cntk_file_path))
if SysInfo.os == TOOLSFORAI_OS_WIN:
download_dir = cntk_file_path
elif SysInfo.os == TOOLSFORAI_OS_LINUX:
download_dir = os.path.join(r"/tmp", cntk_file_name)
skip_downloading = False
if not skip_downloading:
if not utils._download_file(cntk_url, download_dir):
logger.error(
'Fail to install CNTK(BrainScript), the error message: cannot download {0}.'
'Please check your network.'.format(cntk_url))
# fail_install.append("CNTK(BrainScript)")
return False
if (not (utils._unzip_file(download_dir, target_dir)
if SysInfo.os == TOOLSFORAI_OS_WIN else utils._extract_tar(
download_dir, target_dir))):
logger.error(
'Fail to install CNTK(BrainScript), the error message: cannot decompress the downloaded package.'
)
# fail_install.append("CNTK(BrainScript)")
return False
if not skip_downloading:
if os.path.isfile(download_dir):
os.remove(download_dir)
if (SysInfo.os == TOOLSFORAI_OS_WIN):
suc = install_cntk_win(cntk_root)
else:
suc = install_cntk_linux(cntk_root)
version = utils._get_cntk_version(target_dir)
if (suc and (target_version == version)):
logger.info("Install CNTK(BrainScript) successfully!")
logger.warning(
"Please open a new terminal to make the updated Path environment variable effective."
)
return True
else:
logger.error("Fail to install CNTK(BrainScript).")
logger.warning(
"Please manually install {0} and update PATH environment.".format(
target_version))
logger.warning(
"You can reference this link based on your OS: https://docs.microsoft.com/en-us/cognitive-toolkit/Setup-CNTK-on-your-machine"
)
# fail_install.append("CNTK(BrainScript)")
return False
return True
def pip_install_converter(pkg_info, options):
logger.info(
"Begin to install converter(coremltools, onnx, tf2onnx, onnxmltools and winmltools) ..."
)
try:
#1 coremltools
name = pkg_info["converter"]["coremltools"]["name"]
version = pkg_info["converter"]["coremltools"]["version"]
if SysInfo.os == TOOLSFORAI_OS_WIN:
if SysInfo.git:
pkg = "git+https://github.com/apple/[email protected]"
pip_install_package(name, options, version, pkg)
else:
SysInfo.fail_install.append("%s %s" % (name, version))
logger.warning(
"Fail to install {0}. Please manually install git and run installer script again."
.format(name))
else:
pip_install_package(name, options, version)
#2 onnx
name = pkg_info["converter"]["onnx"]["name"]
version = pkg_info["converter"]["onnx"]["version"]
pip_install_package(name, options, version)
#3 tf2onnx
name = pkg_info["converter"]["tf2onnx"]["name"]
version = pkg_info["converter"]["tf2onnx"]["version"]
if not SysInfo.git:
utils.fail_install.append("%s %s" % (name, version))
logger.warning(
"Fail to install {0}. Please manually install git and run installer script again."
.format(name))
else:
pkg = "git+https://github.com/onnx/[email protected]"
if utils.module_exists(name):
logger.info(
"{0} is already installed. We will uninstall it and upgrade to the latest version."
.format(name))
pip_uninstall_packge(name, options, version)
pip_install_package(name, options, version, pkg)
#4 onnxmltools
name = pkg_info["converter"]["onnxmltools"]["name"]
version = pkg_info["converter"]["onnxmltools"]["version"]
if utils.module_exists(name):
logger.info("{0} is already installed.".format(name))
else:
pip_install_package(name, options, version)
#5 winmltools
name = pkg_info["converter"]["winmltools"]["name"]
version = pkg_info["converter"]["winmltools"]["version"]
if utils.module_exists(name):
logger.info("{0} is already installed.".format(name))
else:
pip_install_package(name, options, version)
except Exception as e:
logger.error(
"Fail to install converter, unexpected error! Please run installer again!"
)
def s3_clean(
bucket_name: str = BUCKET_NAME, aws_region: str = AWS_REGION
) -> bool:
""" Delete imcomplete multi-part uploads """
logger.info("Getting list of incomplete uploads...")
try:
multipart_uploads_cmd = f"aws s3api list-multipart-uploads --bucket {bucket_name} --region {aws_region}"
proc = subprocess.run(
multipart_uploads_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
logger.debug(
f'"s3api list-multipart-uploads" returns code: {proc.returncode} => OK'
)
except Exception as e:
if "exit status 254" in str(e):
logger.warning(
f"Bucket {bucket_name} does not exist. Stopping here."
)
return False
else:
logger.error(e)
raise
if proc.returncode == 0 and proc.stdout:
multipart_uploads_list = proc.stdout.strip()
multipart_uploads_list = json.loads(multipart_uploads_list)["Uploads"]
logger.info("Delete in progess...")
try:
for item in multipart_uploads_list:
proc_cmd = f"aws s3api abort-multipart-upload --bucket {bucket_name} --region {aws_region} --key \"{item['Key']}\" --upload-id {item['UploadId']}"
proc = subprocess.run(
proc_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
statistics.append(["s3_clean", len(multipart_uploads_list)])
logger.debug(
f'"s3api abort-multipart-upload" returns code: {proc.returncode} => OK'
)
logger.info(f"Deleted incomplete upload: \"{item['Key']}\".")
logger.debug(
f"{len(multipart_uploads_list)} incomplete upload(s) deleted."
)
except Exception as e:
logger.error(e)
raise
return True
else:
logger.info("Nothing to clean.")
return True
def build_media_files_from_list(
local_files_list: list = None,
output_image_width: int = OUTPUT_IMAGE_WIDTH,
output_image_height: int = OUTPUT_IMAGE_HEIGHT,
output_path: str = LOCAL_MEDIA_OUTPUT_PATH,
log_path: str = LOG_PATH,
) -> bool:
""" Generates web friendly resized images and copy other media files """
logger.info("Generating web friendly images...")
processed_files_count = 0
unprocessed_files = []
path_pattern = re.compile("^.*?/[0-9]{8}/.*[.][a-z-A-Z-0-9]+$")
ts_pattern = re.compile("^[0-9]{8}$")
try:
for media in local_files_list:
ts = media.split("/")[-2]
if path_pattern.match(media):
media_ts = ts
elif not ts_pattern.match(ts):
media_ts = media_ts_format(ts, media)
else:
logger.warning(
f'The file path format should by like eg.: "path/ts/image.jpg".'
)
logger.critical(
f'Input file path format "{media}" is incorrect! Stopping here!'
)
return False
if not media_ts:
unprocessed_files.append(media)
logger.warning(
f"Could not identify the date format. Skipping."
)
else:
gen = media_generate(
media=media,
output_path=output_path,
media_ts=media_ts,
output_image_width=output_image_width,
output_image_height=output_image_height,
processed_files_count=processed_files_count,
unprocessed_files=unprocessed_files,
)
# processed_files_count = gen[0]
# unprocessed_files = gen[1]
processed_files_count, unprocessed_files = gen
statistics.append(
["build_media_files_from_list", processed_files_count]
)
logger.info(
f"{processed_files_count} images have been generated successfully."
)
log_file = f"{log_path}/unprocessed_files.log"
if len(unprocessed_files) > 0:
up_files = [item + "\n" for item in unprocessed_files]
with open(log_file, "w") as w:
w.writelines(up_files)
logger.warning(f"{len(unprocessed_files)} unprocessed file(s)!")
logger.debug(f"Unprocessed file(s): {unprocessed_files}")
elif os.path.exists(log_file):
with open(log_file, "r+") as t:
t.truncate(0)
else:
pass
logger.info("Image files tree generation done.")
if len(unprocessed_files) > 0:
logger.info(
f'Some files were not processed, please review the list: "{log_path}/unprocessed_files.log".'
)
else:
pass
except Exception as e:
logger.error(e)
raise
return True
def build_media_objects(
items: list = None,
aws_region: str = AWS_REGION,
bucket_name: str = BUCKET_NAME,
) -> list:
""" Build media objects """
mediaItems: list = []
ts = None
logger.info("Building media list dictionaries...")
logger.debug(
f"Context Parameters: {build_media_objects.__name__} => {build_media_objects.__code__.co_varnames}"
)
try:
for item in items:
key = item.split("/")
name = key[3]
ts = key[2]
ts = f"{ts[0:4]}-{ts[4:6]}-{ts[6:8]}"
path = f"{key[0]}/{key[1]}/{key[2]}"
url = f"https://s3-{aws_region}.amazonaws.com/{bucket_name}/{path}/{name}"
media_type = get_media_type(name)
if ts != "" and name != "":
media = {}
media["ts"] = ts
media["name"] = name
media["kind"] = media_type
media["path"] = path
media["url"] = url
mediaItems.append(media)
else:
logger.warning(f"ts = {ts} and name = {name}. Stopping here.")
return False
data = sorted(mediaItems, key=itemgetter("ts"), reverse=False)
nbr_data = len(data)
nbr_items = len(items)
statistics.append(["build_media_objects", len(data)])
logger.info("Media list dictionaries built successfully.")
logger.debug(f"{nbr_data} objects in media list.")
if nbr_data != nbr_items:
logger.critical(
"Inconsistency found between data input and output! Stopping here!"
)
logger.debug(
f"Input objects list count [{nbr_items}] and generated media objects count [{nbr_data}] are uneven. Stopping here."
)
return False
else:
pass
except Exception as e:
logger.error(e)
raise
return data