def delete_env(name):
if name in os.environ:
logger.debug("Delete environment variable: {0}".format(name))
return _registry_delete(winreg.HKEY_CURRENT_USER, "Environment", name)
else:
logger.debug("Environment variable {0} doesn't exist.".format(name))
return True
def media_ts_format(ts: str = None, media: str = None) -> str:
""" invoked by build_media_files_from_list() - check and format 'ts' """
ts_month = ts.split(" ")[0]
if len(ts_month) == 3: # if month is "Dec" (3 letters)"
try:
media_ts = (
datetime.strptime(ts, "%b %d, %Y").date().strftime("%Y%m%d")
)
return media_ts
except Exception as e:
logger.warning(
f'"{media}" ts format is not supported. Skipping it.'
)
logger.debug(f"message was: {e}")
return False
elif len(ts_month) > 3: # if month is "December"
try:
media_ts = (
datetime.strptime(ts, "%B %d, %Y").date().strftime("%Y%m%d")
)
return media_ts
except Exception as e:
logger.warning(
f'"{media}" ts format is not supported. Skipping it.'
)
logger.debug(f"message was: {e}")
return False
else:
return False
def seed_db_table(
db_objects: list = None,
table_name: str = TABLE_NAME,
aws_region: str = AWS_REGION,
) -> bool:
""" Insert DB objects into table """
logger.info("Inserting data into DB...")
logger.debug(
f"Context Parameters: {seed_db_table.__name__} => {seed_db_table.__code__.co_varnames}"
)
try:
dynamodb = boto3.resource("dynamodb", region_name=aws_region)
table = dynamodb.Table(table_name)
with table.batch_writer() as batch:
for item in db_objects:
batch.put_item(Item=item)
statistics.append(["seed_db_table", len(db_objects)])
logger.info(f"{len(db_objects)} item(s) were inserted in DB.")
except Exception as e:
logger.error(e)
raise
return True
def pip_install_package(name, options, version, pkg=None):
try:
if not pkg:
if version:
if version.strip()[0] == "<" or version.strip()[0] == ">":
pkg = "{0}{1}".format(name, version)
else:
pkg = "{0} == {1}".format(name, version)
else:
pkg = name
version = ""
if not version:
version = ""
logger.info("Begin to pip-install {0} {1} ...".format(name, version))
logger.debug("pkg : {0}".format(pkg))
res = -1
res = subprocess.check_call(
[sys.executable, '-m', 'pip', 'install', *options, "-q", pkg])
if res != 0:
logger.error("Fail to pip-install {0}.".format(name))
SysInfo.fail_install.append("%s %s" % (name, version))
else:
logger.info("Pip-install {0} {1} successfully!".format(
name, version))
return res == 0
except Exception as e:
# logger.error("Fail to pip-install {0}, unexpected error: {0}".format(name, e))
logger.error(
"Fail to pip-install {0}, unexpected error! Please try to run installer script again!"
.format(name))
SysInfo.fail_install.append("%s %s" % (name, version))
return False
def process_local_movie_medias(
local_media_output_path: str = LOCAL_MEDIA_OUTPUT_PATH,
files_list_path: str = FILES_LIST_PATH,
files_list_filename: str = FILES_LIST_FILENAME,
) -> bool:
""" Get movie files """
logger.info("Starting batch movie encoding...")
try:
with open(f"{files_list_path}/{files_list_filename}", "r") as r:
data = r.read().splitlines()
for item in data:
if get_media_type(item) == "movie":
ts = item.split("/")[-2]
ts = media_ts_format(ts, item)
logger.debug(ts)
if ts:
video_encoder(
media=item, ts=ts, output_path=local_media_output_path
)
else:
pass
except Exception as e:
logger.error(e)
raise
logger.info("Encoder done.")
return True
def build_card_objects(media_list: list = None) -> list:
""" Creates DB objects from S3 objects list """
logger.info("Crafting list of DB objects...")
logger.debug(
f"Context Parameters: {build_card_objects.__name__} => {build_card_objects.__code__.co_varnames}"
)
medias_list = defaultdict(list)
try:
for item in media_list:
medias_list[item["ts"]].append({
"name": item["name"],
"path": item["path"],
"url": item["url"],
"kind": item["kind"],
})
medias = [{"ts": k, "medias": v} for k, v in medias_list.items()]
statistics.append(["build_card_objects", len(medias)])
logger.info(f'{len(medias)} "card" objects generated successfully.')
except Exception as e:
logger.error(e)
raise
return medias
def video_encoder(
media: str = None,
ts: str = None,
output_path: str = None,
log_path: str = LOG_PATH,
video_preset_data: dict = video_preset_data,
media_presets: str = VIDEO_PRESETS,
encoder_threads: int = ENCODER_THREADS,
) -> bool:
""" Encode video media based on preset """
try:
i = 0
media_presets = media_presets.split(" ")
for media_preset in media_presets:
media_preset = media_preset.split(".")
preset_category = media_preset[0]
preset_format = media_preset[1]
settings = video_preset_data[preset_category][preset_format]
logger.debug(f"Settings: {settings}")
file_format = settings["format"]
vcodec = settings["vcodec"]
acodec = settings["acodec"]
video_bitrate = settings["video_bitrate"]
audio_bitrate = settings["audio_bitrate"]
encoder_threads = int(encoder_threads)
i += 1
logger.info(
f'Encoding media "{media}" using preset "{preset_category} -> {preset_format}" ...'
)
logger.info(
f'Processing task(s) for: "{preset_format}"... {i}/{len(media_presets)}.'
)
output_filename = (
f"{media.split('/')[-1].split('.')[-2]}.{preset_format}")
output_file = f"{output_path}/{ts}/{output_filename}"
cli_cmd = f"ffmpeg -i '{media}' -f {file_format} -vcodec {vcodec} -acodec {acodec} -vb {video_bitrate} -ab {audio_bitrate} -threads {encoder_threads} -y '{output_file}'"
logger.debug(f"cli command: {cli_cmd}")
with open(f"{log_path}/ffmpeg.log", "a") as w:
subprocess.run(
cli_cmd,
shell=True,
check=True,
stdout=w,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
except Exception as e:
logger.error(e)
raise
return True
def _wait_process(processHandle, timeout=-1):
try:
ret = ctypes.windll.kernel32.WaitForSingleObject(processHandle, timeout)
logger.debug("Wait process return value: %d" % ret)
except Exception as e:
logger.debug("Fail to wait process, unexpected error: {0}".format(e))
finally:
ctypes.windll.kernel32.CloseHandle(processHandle)
def _registry_write(hkey, keypath, name, value):
try:
registry_key = winreg.CreateKeyEx(hkey, keypath)
winreg.SetValueEx(registry_key, name, 0, winreg.REG_SZ, value)
winreg.CloseKey(registry_key)
return True
except Exception as e:
logger.debug("Fail to write registry key: {0}, name: {1}, value: {2}, unexpected error: {3}".format(keypath, name, value, e))
return False
def _registry_delete(hkey, keypath, name):
try:
registry_key = winreg.OpenKey(hkey, keypath, 0, winreg.KEY_SET_VALUE)
winreg.DeleteValue(registry_key, name)
winreg.CloseKey(registry_key)
return True
except Exception as e:
logger.debug("Fail to delete registry key: {0}, name: {1}, unexpected error: {2}".format(keypath, name, e))
return False
def _registry_read(hkey, keypath, value_name):
try:
registry_key = winreg.OpenKey(hkey, keypath)
value, _ = winreg.QueryValueEx(registry_key, value_name)
winreg.CloseKey(registry_key)
return value
except Exception as e:
logger.debug("Fail to read registry key: {0}, value: {1}, unexpected error: {2}".format(keypath, value_name, e))
return None
def get_s3_files(
bucket_name: str = BUCKET_NAME,
save_to_disk: bool = True,
files_list_path: str = FILES_LIST_PATH,
files_list_filename: str = FILES_LIST_FILENAME,
aws_region: str = AWS_REGION,
s3_prefix: str = S3_PREFIX,
) -> list:
""" Get S3 objects and creates list """
logger.info("Building media list from S3 objects...")
logger.debug(
f"Context Parameters: {get_s3_files.__name__} => {get_s3_files.__code__.co_varnames}"
)
data = []
# testing format: assets/20160823/img.jpg
pattern = re.compile(
"^[a-z-A-Z-0-9]+/[a-z-A-Z-0-9]+/[0-9]{8}/.+[.][a-z-A-Z-0-9]+$"
)
try:
s3 = boto3.client("s3", region_name=aws_region)
paginator = s3.get_paginator("list_objects_v2")
pages = paginator.paginate(Bucket=bucket_name, Prefix=s3_prefix)
for page in pages:
for obj in page["Contents"]:
if pattern.match(obj["Key"]):
data.append(obj["Key"])
else:
logger.warning(
f'Wrong filename format, object "{obj["Key"]}", not added to the list.'
)
statistics.append(["get_s3_files", len(data)])
logger.info("Media Objects list generated successfully.")
logger.debug(f"Media objects count: {len(data)}.")
if save_to_disk:
logger.info("Writing media list to disk...")
export_data = [f"{item}\n" for item in data]
with open(f"{files_list_path}/{files_list_filename}", "w") as w:
w.writelines(export_data)
logger.info(
f'List successfully saved to disk: "{files_list_path}/{files_list_filename}".'
)
else:
pass
except Exception as e:
logger.error(e)
raise
return data
def _get_cntk_version(cntk_root):
logger.debug("In _get_cntk_version(), cntk_root: {0}".format(cntk_root))
version = ''
version_file = os.path.join(cntk_root, "cntk", "version.txt")
if os.path.isfile(version_file):
with open(version_file) as fin:
version = fin.readline().strip()
logger.debug("In _get_cntk_version(), find cntk_version: {0}".format(version))
return version
def detect_python_version():
py_architecture = platform.architecture()[0]
py_version = ".".join(map(str, sys.version_info[0:2]))
py_full_version = ".".join(map(str, sys.version_info[0:3]))
SysInfo.python = py_version.replace('.', '')
logger.debug("In detect_python_version(), sys_info['python']: {0}".format(SysInfo.python))
logger.info("Python: {0}, {1}".format(py_full_version, py_architecture))
if not (_version_compare("3.5", py_version) and py_architecture == '64bit'):
logger.error("64-bit Python 3.5 or higher is required to run this installer."
" We recommend latest Python 3.5 (https://www.python.org/downloads/release/python-355/).")
return False
return True
def _update_pathenv_win(path, add):
path_value = _registry_read(winreg.HKEY_CURRENT_USER, "Environment", "PATH")
logger.debug("Before update, PATH: {0}".format(path_value))
if add:
if path in path_value:
return
path_value = path + ";" + path_value
os.environ["PATH"] = path + ";" + os.environ.get("PATH", "")
else:
path_value = path_value.replace(path + ";", "")
os.environ["PATH"] = os.environ["PATH"].replace(path + ";", "")
_registry_write(winreg.HKEY_CURRENT_USER, "Environment", "PATH", path_value)
def create_s3_bucket(
bucket_name: str = BUCKET_NAME, aws_region: str = AWS_REGION
) -> bool:
""" Create the S3 bucket of the project """
s3 = boto3.client("s3", region_name=aws_region)
bucket_exists = True
try:
response = response = s3.list_buckets()
buckets = [
bucket["Name"]
for bucket in response["Buckets"]
if bucket["Name"] == bucket_name
]
if len(buckets) > 0:
logger.warning(
"S3 bucket already exists. Skipping bucket creation."
)
bucket_exists = True
else:
bucket_exists = False
except Exception as e:
logger.error(e)
raise
if not bucket_exists:
try:
response = s3.create_bucket(
Bucket=bucket_name,
ACL="private",
CreateBucketConfiguration={"LocationConstraint": aws_region},
)
logger.info(f'Created S3 bucket "{bucket_name}" successfully.')
logger.debug(f"S3 client response: {response}")
except Exception as e:
logger.error(e)
raise
else:
return False
return True
def medias_copy(
local_path: str = LOCAL_MEDIA_OUTPUT_PATH,
video_encode: bool = VIDEO_ENCODE,
media_encode_platform: str = MEDIA_ENCODE_PLATFORM,
) -> bool:
""" Copy media files to S3 """
logger.info("Starting copy...")
try:
medias = get_local_medias_files(path=local_path, save_to_disk=False)
logger.debug(medias)
for media in medias:
media_type = get_media_type(basename(media))
ts = media.split("/")[-2]
if media_type == "movie":
if video_encode == "True" and media_encode_platform == "cloud":
send_to_bucket(media, ts)
elif (
video_encode == "True" and media_encode_platform == "local"
):
logger.info(
f"Skipping copy of {media} for local re-encoding."
)
elif media_type == "picture":
send_to_bucket(media, ts)
else:
logger.warning(f"Media type is: {media_type} !")
logger.info(
f"{len(medias)} medias files have been successfully copied."
)
except Exception as e:
logger.error(e)
raise
statistics.append(["medias_copy", len(medias)])
logger.info("...done.")
return True
def install_cntk_win(cntk_root):
suc = True
try:
utils._update_pathenv_win(os.path.join(cntk_root, "cntk"), True)
if (not utils.detect_mpi_win()):
mpi_exe = os.path.sep.join(
[cntk_root, "prerequisites", "MSMpiSetup.exe"])
logger.debug("MPI exe path: %s" % mpi_exe)
logger.info("Begin to install MPI ...")
utils._run_cmd_admin(mpi_exe, "-unattend")
if (utils.detect_mpi_win()):
logger.info("Install MPI successfully.")
else:
suc = False
logger.error(
"Fail to install MPI. Please manually install MPI >= 7.0.12437.6"
)
if (not utils.detect_visualcpp_runtime_win()):
vc_redist_exe = os.path.sep.join(
[cntk_root, "prerequisites", "VS2015", "vc_redist.x64.exe"])
logger.debug("VC redist exe path: {0}".format(vc_redist_exe))
logger.info("Begin to install Visual C++ runtime ...")
utils._run_cmd_admin(vc_redist_exe, "/install /norestart /passive")
if (utils.detect_visualcpp_runtime_win()):
logger.info("Install Visual C++ runtime successfully.")
logger.warning(
" Please manually install Visual C++ Redistributable Package for Visual Studio 2015 or 2017."
)
else:
suc = False
logger.error("Fail to install Visual C++ runtime.")
except:
suc = False
logger.error(
"Fail to install CNTK(BrainScript). The error massage: {0}".format(
sys.exc_info()))
return suc
def pip_install_ml_software(pkg_info, options):
logger.info(
"Begin to install ml software(scikit-learn, xgboost and libsvm) ...")
#1 scikit-learn
name = pkg_info["ml_software"]["scikit-learn"]["name"]
version = pkg_info["ml_software"]["scikit-learn"]["version"]
pip_install_package(name, options, version)
#2 xgboost
name = pkg_info["ml_software"]["xgboost"]["name"]
version = pkg_info["ml_software"]["xgboost"]["version"]
if SysInfo.os != TOOLSFORAI_OS_WIN:
if not pip_install_package(name, options, version):
logger.warning(
"In order to install xgboost, C++ compiler is needed.")
else:
wheel_ver = SysInfo.python
arch = "win_amd64"
pkg = "https://raw.githubusercontent.com/linmajia/ai-package/master/xgboost/{0}/xgboost-{0}-cp{1}-cp{1}m-{2}.whl".format(
version, wheel_ver, arch)
pip_install_package(name, options, version, pkg)
#3 libsvm
name = pkg_info["ml_software"]["libsvm"]["name"]
version = pkg_info["ml_software"]["libsvm"]["version"]
if SysInfo.os != TOOLSFORAI_OS_WIN:
logger.warning(
"Fail to install libsvm. On Linux or Mac, in order to install {0}=={1}, please manually download source code and install it."
.format(name, version))
else:
wheel_ver = SysInfo.python
arch = "win_amd64"
pkg = "https://raw.githubusercontent.com/linmajia/ai-package/master/libsvm/{0}/libsvm-{0}-cp{1}-cp{1}m-{2}.whl".format(
version, wheel_ver, arch)
logger.debug("Pip install libsvm from {0}".format(pkg))
pip_install_package(name, options, version, pkg)
def send_to_bucket(
media_file: str,
ts: str,
bucket_name: str = BUCKET_NAME,
s3_prefix: str = S3_PREFIX,
aws_region: str = AWS_REGION,
) -> bool:
""" Send file to S3 """
logger.info(f'Sending "{media_file}" to bucket "{bucket_name}"...')
try:
key = f"{s3_prefix}/{ts}/{basename(media_file)}"
s3 = boto3.client("s3", region_name=aws_region)
with open(media_file, "rb") as data:
s3.upload_fileobj(data, bucket_name, key)
logger.debug(f"media_file: {media_file} - key: {key}")
except Exception as e:
logger.error(e)
raise
logger.info(f'File "{media_file}" sent successfully to bucket: "{key}"')
return True
def remote_video_encoder(files_list_path: str = FILES_LIST_PATH) -> bool:
""" Send movies list to SQS -> lambda/ffmpeg """
logger.info("Starting remote movie re-encoding operations...")
data_path = f"{files_list_path}/defered_encode.json"
if os.path.exists(data_path):
try:
with open(data_path, "r") as r:
movies = r.read()
for movie in movies:
queue_message = send_to_queue(movies)
logger.info(f"Re-encoding process launched for '{movie}'.")
logger.debug(queue_message)
except Exception as e:
logger.error(e)
raise
else:
logger.critical(f"Path does not exist: '{data_path}'. Stopping here.")
return False
logger.info("...done.")
return True
def create_table(
table_name: str = TABLE_NAME,
ReadCapacityUnits: int = TABLE_READ_CAPACITY_UNITS,
WriteCapacityUnits: int = TABLE_WRITE_CAPACITY_UNITS,
aws_region: str = AWS_REGION,
) -> bool:
""" Creates DynamoB table """
try:
client = boto3.client("dynamodb", region_name=aws_region)
response = client.list_tables()
tables = [
table for table in response["TableNames"] if table == table_name
]
if len(tables) > 0:
logger.warning(
f'Table "{table_name}" already exists. Skipping table creation.'
)
return False
else:
logger.info(
f'Table "{table_name}" does not exist. Starting creation process...'
)
except Exception as e:
logger.error(e)
raise
logger.info("Creating DB table...")
logger.debug(
f"Context Parameters: {create_table.__name__} => {create_table.__code__.co_varnames}"
)
try:
dynamodb = boto3.resource("dynamodb", region_name=aws_region)
table = dynamodb.create_table(
TableName=table_name,
AttributeDefinitions=[
{"AttributeName": "ts", "AttributeType": "S"}
],
KeySchema=[{"AttributeName": "ts", "KeyType": "HASH"}],
ProvisionedThroughput={
"ReadCapacityUnits": int(ReadCapacityUnits),
"WriteCapacityUnits": int(WriteCapacityUnits),
},
)
logger.info("Table created successfully.")
logger.debug(table)
except dynamodb.exceptions.ResourceInUseException as e:
logger.warning(
f'Table "{table_name}" already exists. Skipping table creation.'
)
logger.debug(e)
return False
return True
def main(display_env: str = LOG_DISPLAY_ENV_VARS) -> None:
""" Fetch, build and store S3 media files into DynamoDB """
if display_env == "True":
logger.debug("## Environment variables")
logger.debug(os.environ)
else:
pass
logger.debug("- Start of execution -")
prepare_local_resources()
setup_cloud_resources()
hydrate_cloud_resources()
logger.debug("- End of execution -")
print(tabulate(statistics))
monitor_remote_ops()
logger.info("- All tasks executed successfully -")
def get_local_medias_files(
path: str = LOCAL_MEDIA_PATH,
save_to_disk: bool = True,
files_list_path: str = FILES_LIST_PATH,
files_list_filename: str = FILES_LIST_FILENAME,
config_path: str = CONFIG_PATH,
) -> list:
""" Generates a list of local media files """
if os.path.exists(path):
local_medias = []
filtered_files = []
try:
logger.info("Generating list of local files...")
for dirpath, _, files in os.walk(path):
for filename in files:
fname = os.path.join(dirpath, filename)
if is_filtered(filename):
filtered_files.append(fname)
else:
local_medias.append(fname)
if len(local_medias) > 0:
statistics.append(
["get_local_medias_files", len(local_medias)]
)
logger.info("List successfully generated.")
logger.debug(f"Count: {len(local_medias)} local files.")
else:
logger.critical(
f'No files found in source directory: "{path}".'
)
return False
if save_to_disk:
logger.info("Writing local files list to disk...")
data_to_write = [item + "\n" for item in local_medias]
with open(
f"{files_list_path}/{files_list_filename}", "w"
) as w:
w.writelines(data_to_write)
logger.info(
f'The list has been saved successfully: "{files_list_path}/{files_list_filename}".'
)
else:
pass
if len(filtered_files) > 0:
logger.info(
f'Number of file(s) excluded by filter specified in "{config_path}/exclude_local.txt": {len(filtered_files)}.'
)
logger.debug(f"excluded by filter: {filtered_files}")
else:
pass
except Exception as e:
logger.error(e)
raise
return local_medias
else:
logger.critical(f'Missing input "path"! Stopping here!')
return False
def _run_cmd(cmd, args=[], return_stdout=False):
try:
p = subprocess.run([cmd, *args], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
stdout = p.stdout.strip()
stderr = p.stderr.strip()
status = p.returncode == 0
logger.debug("========== {:^30} ==========".format("%s : stdout" % cmd))
for line in filter(lambda x: x.strip(), p.stdout.split('\n')):
logger.debug(line)
logger.debug("========== {:^30} ==========".format("%s : stdout end" % cmd))
logger.debug("========== {:^30} ==========".format("%s : stderr" % cmd))
for line in filter(lambda x: x.strip(), p.stderr.split('\n')):
logger.debug(line)
logger.debug("========== {:^30} ==========".format("%s : stderr end" % cmd))
except Exception as e:
logger.debug("Fail to execute command: {0}, unexpected error: {1}".format(cmd, e))
status = False
stdout = ""
if return_stdout:
return status, stdout
else:
return status
def media_generate(
media: str = None,
output_path: str = None,
media_ts: str = None,
output_image_width: int = None,
output_image_height: int = None,
processed_files_count: int = None,
unprocessed_files: list = None,
video_encode: str = VIDEO_ENCODE,
log_path: str = LOG_PATH,
s3_prefix: str = S3_PREFIX,
media_encode_platform: str = MEDIA_ENCODE_PLATFORM,
) -> list:
""" invoked by build_media_files_from_list() - gemerates media files """
media_name = media.split("/")[-1]
media_type = get_media_type(media_name)
if not os.path.exists(f"{output_path}/{media_ts}"):
os.mkdir(f"{output_path}/{media_ts}")
logger.debug(f'Created directory: "{output_path}/{media_ts}".')
else:
pass
if media_type == "picture":
logger.info(
f"Picture type identified, starting generation of media...")
image = Image.open(media)
if image:
with image as im:
im.thumbnail((output_image_width, output_image_height))
im.save(
f"{output_path}/{media_ts}/{media_name}",
format="JPEG",
quality="web_high",
dpi=(72, 72),
)
processed_files_count += 1
logger.info(
f'Generated media: "{output_path}/{media_ts}/{media_name}".')
else:
logger.warning(
f'Impossible to open the image file: "{media_name}"! File identified format is : "{media_type}". Skipping it.'
)
elif media_type == "movie":
if video_encode:
logger.info(f"Movie type identified...")
if media_encode_platform == "local":
if os.path.exists(f"{log_path}/ffmpeg.log"):
with open(f"{log_path}/ffmpeg.log", "r+") as w:
w.truncate(0)
else:
pass
video_encoder(media, media_ts, output_path)
elif media_encode_platform == "cloud":
logger.info(f"Movie type identified, starting copy of file...")
shutil.copyfile(media,
f"{output_path}/{media_ts}/{media_name}")
logger.info(
f'File copied successfully: "{media}" => "{output_path}/{media_ts}/{media_name}"'
)
movie = f"{s3_prefix}/{media_ts}/{media_name}"
cloud_video_encoder_list.append({
"src": movie,
"ts": media_ts,
"delete_old": True
})
logger.info(
f"Added movie '{movie}' to queue for defered remote re-encoding."
)
else:
logger.critical(
'Wrong or missing value! Valid values for "media_encode_platform": local|cloud'
)
else:
logger.info(f"Movie type identified, starting copy of file...")
shutil.copyfile(media, f"{output_path}/{media_ts}/{media_name}")
logger.info(
f'File copied successfully: "{media}" => "{output_path}/{media_ts}/{media_name}"'
)
processed_files_count += 1
else:
unprocessed_files.append(media)
logger.warning(f'Impossible to process file: "{media}". Skipping it.')
return (processed_files_count, unprocessed_files)
def media_sync(
local_path: str = LOCAL_MEDIA_OUTPUT_PATH,
bucket_name: str = BUCKET_NAME,
remote_path_prefix: str = S3_PREFIX,
log_path: str = LOG_PATH,
aws_region: str = AWS_REGION,
config_path: str = CONFIG_PATH,
) -> bool:
""" Synchronize local/S3 media files tree """
exclude_s3_file_path = f"{config_path}/exclude_s3.txt"
if os.path.exists(exclude_s3_file_path):
with open(exclude_s3_file_path, "r") as r:
common_oses_filter = r.read().splitlines()
cli_filter_args = ""
cli_filter_args = cli_filter_args.join(
[
f' --exclude "{item}"'
for item in common_oses_filter
if not item.startswith("#") or item != "\n"
]
)
else:
cli_filter_args = ""
logger.info("Starting sync...")
logger.info(f"S3 sync task log => tail -F {log_path}/s3_sync.log")
try:
cli_cmd = f"aws s3 sync {local_path}/ s3://{bucket_name}/{remote_path_prefix}/ --delete --region {aws_region} {cli_filter_args}"
logger.debug(f"cli command: {cli_cmd}")
with open(f"{log_path}/s3_sync.log", "w") as w:
proc = subprocess.run(
cli_cmd,
shell=True,
check=True,
stdout=w,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
if proc.returncode == 0:
with open(f"{log_path}/s3_sync.log", "r") as r:
processed_objects = r.read().splitlines()
processed_objects = [
item for item in processed_objects if "upload" in item
]
statistics.append(["media_sync", len(processed_objects)])
logger.info("Sync completed successfully.")
logger.debug(
f"{len(processed_objects)} files have been synchronized successfully."
)
logger.debug(f"S3 CLI returned code: {proc.returncode} => OK")
else:
logger.critical("Something wrong happened during sync operation!")
return False
except Exception as e:
logger.error(e)
raise
return True
def install_cntk(target_dir):
logger.info("Begin to install CNTK(BrainScript) ...")
if SysInfo.os != TOOLSFORAI_OS_WIN and SysInfo.os != TOOLSFORAI_OS_LINUX:
logger.warning(
"CNTK(BrainScript) is not supported on your OS, we recommend 64-bit Windows-10 OS or 64-bit Linux OS."
)
# fail_install.append("CNTK(BrainScript)")
return False
if SysInfo.cuda == "8.0":
ver = "2.3.1"
else:
ver = "2.5.1"
target_version = 'CNTK-{0}'.format(ver.replace('.', '-'))
logger.debug(
"In install_cntk(), target_version: {0}".format(target_version))
version = utils._get_cntk_version(target_dir)
if target_version == version:
logger.info('CNTK(BrainScript)-{0} is already installed.'.format(ver))
return True
logger.debug('In install_cntk(), target_dir: {0}'.format(target_dir))
cntk_root = os.path.join(target_dir, 'cntk')
if os.path.isdir(cntk_root):
try:
shutil.rmtree(cntk_root)
except:
logger.error(
'Fail to install CNTK(BrainScript), the error message: can not remove old version in directory {0}.'
'Please manually remove old version, and run the installer script again.'
.format(cntk_root))
# fail_install.append("CNTK(BrainScript)")
return False
if not os.path.isdir(target_dir):
try:
os.makedirs(target_dir)
except:
logger.error(
'Fail to install CNTK(BrainScript), the error message: can not create directory {0}.'
'Please check if there is permission for creating directory.'.
format(target_dir))
# fail_install.append("CNTK(BrainScript)")
return False
cntk_file_name = "{}-{}-64bit-{}.{}".format(
target_version,
"Windows" if SysInfo.os == TOOLSFORAI_OS_WIN else "Linux",
"GPU" if SysInfo.gpu else "CPU-Only",
"zip" if SysInfo.os == TOOLSFORAI_OS_WIN else "tar.gz")
logger.debug(
"In install_cntk(), cntk_file_name: {0}".format(cntk_file_name))
cntk_url = "https://cntk.ai/BinaryDrop/{0}".format(cntk_file_name)
logger.debug("In install_cntk(), cntk_url: {0}".format(cntk_url))
cntk_file_path = os.path.join(target_dir, cntk_file_name)
logger.debug(
"In install_cntk(), cntk_file_path: {0}".format(cntk_file_path))
if SysInfo.os == TOOLSFORAI_OS_WIN:
download_dir = cntk_file_path
elif SysInfo.os == TOOLSFORAI_OS_LINUX:
download_dir = os.path.join(r"/tmp", cntk_file_name)
skip_downloading = False
if not skip_downloading:
if not utils._download_file(cntk_url, download_dir):
logger.error(
'Fail to install CNTK(BrainScript), the error message: cannot download {0}.'
'Please check your network.'.format(cntk_url))
# fail_install.append("CNTK(BrainScript)")
return False
if (not (utils._unzip_file(download_dir, target_dir)
if SysInfo.os == TOOLSFORAI_OS_WIN else utils._extract_tar(
download_dir, target_dir))):
logger.error(
'Fail to install CNTK(BrainScript), the error message: cannot decompress the downloaded package.'
)
# fail_install.append("CNTK(BrainScript)")
return False
if not skip_downloading:
if os.path.isfile(download_dir):
os.remove(download_dir)
if (SysInfo.os == TOOLSFORAI_OS_WIN):
suc = install_cntk_win(cntk_root)
else:
suc = install_cntk_linux(cntk_root)
version = utils._get_cntk_version(target_dir)
if (suc and (target_version == version)):
logger.info("Install CNTK(BrainScript) successfully!")
logger.warning(
"Please open a new terminal to make the updated Path environment variable effective."
)
return True
else:
logger.error("Fail to install CNTK(BrainScript).")
logger.warning(
"Please manually install {0} and update PATH environment.".format(
target_version))
logger.warning(
"You can reference this link based on your OS: https://docs.microsoft.com/en-us/cognitive-toolkit/Setup-CNTK-on-your-machine"
)
# fail_install.append("CNTK(BrainScript)")
return False
return True
def build_media_files_from_list(
local_files_list: list = None,
output_image_width: int = OUTPUT_IMAGE_WIDTH,
output_image_height: int = OUTPUT_IMAGE_HEIGHT,
output_path: str = LOCAL_MEDIA_OUTPUT_PATH,
log_path: str = LOG_PATH,
) -> bool:
""" Generates web friendly resized images and copy other media files """
logger.info("Generating web friendly images...")
processed_files_count = 0
unprocessed_files = []
path_pattern = re.compile("^.*?/[0-9]{8}/.*[.][a-z-A-Z-0-9]+$")
ts_pattern = re.compile("^[0-9]{8}$")
try:
for media in local_files_list:
ts = media.split("/")[-2]
if path_pattern.match(media):
media_ts = ts
elif not ts_pattern.match(ts):
media_ts = media_ts_format(ts, media)
else:
logger.warning(
f'The file path format should by like eg.: "path/ts/image.jpg".'
)
logger.critical(
f'Input file path format "{media}" is incorrect! Stopping here!'
)
return False
if not media_ts:
unprocessed_files.append(media)
logger.warning(
f"Could not identify the date format. Skipping."
)
else:
gen = media_generate(
media=media,
output_path=output_path,
media_ts=media_ts,
output_image_width=output_image_width,
output_image_height=output_image_height,
processed_files_count=processed_files_count,
unprocessed_files=unprocessed_files,
)
# processed_files_count = gen[0]
# unprocessed_files = gen[1]
processed_files_count, unprocessed_files = gen
statistics.append(
["build_media_files_from_list", processed_files_count]
)
logger.info(
f"{processed_files_count} images have been generated successfully."
)
log_file = f"{log_path}/unprocessed_files.log"
if len(unprocessed_files) > 0:
up_files = [item + "\n" for item in unprocessed_files]
with open(log_file, "w") as w:
w.writelines(up_files)
logger.warning(f"{len(unprocessed_files)} unprocessed file(s)!")
logger.debug(f"Unprocessed file(s): {unprocessed_files}")
elif os.path.exists(log_file):
with open(log_file, "r+") as t:
t.truncate(0)
else:
pass
logger.info("Image files tree generation done.")
if len(unprocessed_files) > 0:
logger.info(
f'Some files were not processed, please review the list: "{log_path}/unprocessed_files.log".'
)
else:
pass
except Exception as e:
logger.error(e)
raise
return True
def s3_clean(
bucket_name: str = BUCKET_NAME, aws_region: str = AWS_REGION
) -> bool:
""" Delete imcomplete multi-part uploads """
logger.info("Getting list of incomplete uploads...")
try:
multipart_uploads_cmd = f"aws s3api list-multipart-uploads --bucket {bucket_name} --region {aws_region}"
proc = subprocess.run(
multipart_uploads_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
logger.debug(
f'"s3api list-multipart-uploads" returns code: {proc.returncode} => OK'
)
except Exception as e:
if "exit status 254" in str(e):
logger.warning(
f"Bucket {bucket_name} does not exist. Stopping here."
)
return False
else:
logger.error(e)
raise
if proc.returncode == 0 and proc.stdout:
multipart_uploads_list = proc.stdout.strip()
multipart_uploads_list = json.loads(multipart_uploads_list)["Uploads"]
logger.info("Delete in progess...")
try:
for item in multipart_uploads_list:
proc_cmd = f"aws s3api abort-multipart-upload --bucket {bucket_name} --region {aws_region} --key \"{item['Key']}\" --upload-id {item['UploadId']}"
proc = subprocess.run(
proc_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
statistics.append(["s3_clean", len(multipart_uploads_list)])
logger.debug(
f'"s3api abort-multipart-upload" returns code: {proc.returncode} => OK'
)
logger.info(f"Deleted incomplete upload: \"{item['Key']}\".")
logger.debug(
f"{len(multipart_uploads_list)} incomplete upload(s) deleted."
)
except Exception as e:
logger.error(e)
raise
return True
else:
logger.info("Nothing to clean.")
return True
