def delete_env(name): if name in os.environ: logger.debug("Delete environment variable: {0}".format(name)) return _registry_delete(winreg.HKEY_CURRENT_USER, "Environment", name) else: logger.debug("Environment variable {0} doesn't exist.".format(name)) return True
def media_ts_format(ts: str = None, media: str = None) -> str: """ invoked by build_media_files_from_list() - check and format 'ts' """ ts_month = ts.split(" ")[0] if len(ts_month) == 3: # if month is "Dec" (3 letters)" try: media_ts = ( datetime.strptime(ts, "%b %d, %Y").date().strftime("%Y%m%d") ) return media_ts except Exception as e: logger.warning( f'"{media}" ts format is not supported. Skipping it.' ) logger.debug(f"message was: {e}") return False elif len(ts_month) > 3: # if month is "December" try: media_ts = ( datetime.strptime(ts, "%B %d, %Y").date().strftime("%Y%m%d") ) return media_ts except Exception as e: logger.warning( f'"{media}" ts format is not supported. Skipping it.' ) logger.debug(f"message was: {e}") return False else: return False
def seed_db_table( db_objects: list = None, table_name: str = TABLE_NAME, aws_region: str = AWS_REGION, ) -> bool: """ Insert DB objects into table """ logger.info("Inserting data into DB...") logger.debug( f"Context Parameters: {seed_db_table.__name__} => {seed_db_table.__code__.co_varnames}" ) try: dynamodb = boto3.resource("dynamodb", region_name=aws_region) table = dynamodb.Table(table_name) with table.batch_writer() as batch: for item in db_objects: batch.put_item(Item=item) statistics.append(["seed_db_table", len(db_objects)]) logger.info(f"{len(db_objects)} item(s) were inserted in DB.") except Exception as e: logger.error(e) raise return True
def pip_install_package(name, options, version, pkg=None): try: if not pkg: if version: if version.strip()[0] == "<" or version.strip()[0] == ">": pkg = "{0}{1}".format(name, version) else: pkg = "{0} == {1}".format(name, version) else: pkg = name version = "" if not version: version = "" logger.info("Begin to pip-install {0} {1} ...".format(name, version)) logger.debug("pkg : {0}".format(pkg)) res = -1 res = subprocess.check_call( [sys.executable, '-m', 'pip', 'install', *options, "-q", pkg]) if res != 0: logger.error("Fail to pip-install {0}.".format(name)) SysInfo.fail_install.append("%s %s" % (name, version)) else: logger.info("Pip-install {0} {1} successfully!".format( name, version)) return res == 0 except Exception as e: # logger.error("Fail to pip-install {0}, unexpected error: {0}".format(name, e)) logger.error( "Fail to pip-install {0}, unexpected error! Please try to run installer script again!" .format(name)) SysInfo.fail_install.append("%s %s" % (name, version)) return False
def process_local_movie_medias( local_media_output_path: str = LOCAL_MEDIA_OUTPUT_PATH, files_list_path: str = FILES_LIST_PATH, files_list_filename: str = FILES_LIST_FILENAME, ) -> bool: """ Get movie files """ logger.info("Starting batch movie encoding...") try: with open(f"{files_list_path}/{files_list_filename}", "r") as r: data = r.read().splitlines() for item in data: if get_media_type(item) == "movie": ts = item.split("/")[-2] ts = media_ts_format(ts, item) logger.debug(ts) if ts: video_encoder( media=item, ts=ts, output_path=local_media_output_path ) else: pass except Exception as e: logger.error(e) raise logger.info("Encoder done.") return True
def build_card_objects(media_list: list = None) -> list: """ Creates DB objects from S3 objects list """ logger.info("Crafting list of DB objects...") logger.debug( f"Context Parameters: {build_card_objects.__name__} => {build_card_objects.__code__.co_varnames}" ) medias_list = defaultdict(list) try: for item in media_list: medias_list[item["ts"]].append({ "name": item["name"], "path": item["path"], "url": item["url"], "kind": item["kind"], }) medias = [{"ts": k, "medias": v} for k, v in medias_list.items()] statistics.append(["build_card_objects", len(medias)]) logger.info(f'{len(medias)} "card" objects generated successfully.') except Exception as e: logger.error(e) raise return medias
def video_encoder( media: str = None, ts: str = None, output_path: str = None, log_path: str = LOG_PATH, video_preset_data: dict = video_preset_data, media_presets: str = VIDEO_PRESETS, encoder_threads: int = ENCODER_THREADS, ) -> bool: """ Encode video media based on preset """ try: i = 0 media_presets = media_presets.split(" ") for media_preset in media_presets: media_preset = media_preset.split(".") preset_category = media_preset[0] preset_format = media_preset[1] settings = video_preset_data[preset_category][preset_format] logger.debug(f"Settings: {settings}") file_format = settings["format"] vcodec = settings["vcodec"] acodec = settings["acodec"] video_bitrate = settings["video_bitrate"] audio_bitrate = settings["audio_bitrate"] encoder_threads = int(encoder_threads) i += 1 logger.info( f'Encoding media "{media}" using preset "{preset_category} -> {preset_format}" ...' ) logger.info( f'Processing task(s) for: "{preset_format}"... {i}/{len(media_presets)}.' ) output_filename = ( f"{media.split('/')[-1].split('.')[-2]}.{preset_format}") output_file = f"{output_path}/{ts}/{output_filename}" cli_cmd = f"ffmpeg -i '{media}' -f {file_format} -vcodec {vcodec} -acodec {acodec} -vb {video_bitrate} -ab {audio_bitrate} -threads {encoder_threads} -y '{output_file}'" logger.debug(f"cli command: {cli_cmd}") with open(f"{log_path}/ffmpeg.log", "a") as w: subprocess.run( cli_cmd, shell=True, check=True, stdout=w, stderr=subprocess.STDOUT, universal_newlines=True, ) except Exception as e: logger.error(e) raise return True
def _wait_process(processHandle, timeout=-1): try: ret = ctypes.windll.kernel32.WaitForSingleObject(processHandle, timeout) logger.debug("Wait process return value: %d" % ret) except Exception as e: logger.debug("Fail to wait process, unexpected error: {0}".format(e)) finally: ctypes.windll.kernel32.CloseHandle(processHandle)
def _registry_write(hkey, keypath, name, value): try: registry_key = winreg.CreateKeyEx(hkey, keypath) winreg.SetValueEx(registry_key, name, 0, winreg.REG_SZ, value) winreg.CloseKey(registry_key) return True except Exception as e: logger.debug("Fail to write registry key: {0}, name: {1}, value: {2}, unexpected error: {3}".format(keypath, name, value, e)) return False
def _registry_delete(hkey, keypath, name): try: registry_key = winreg.OpenKey(hkey, keypath, 0, winreg.KEY_SET_VALUE) winreg.DeleteValue(registry_key, name) winreg.CloseKey(registry_key) return True except Exception as e: logger.debug("Fail to delete registry key: {0}, name: {1}, unexpected error: {2}".format(keypath, name, e)) return False
def _registry_read(hkey, keypath, value_name): try: registry_key = winreg.OpenKey(hkey, keypath) value, _ = winreg.QueryValueEx(registry_key, value_name) winreg.CloseKey(registry_key) return value except Exception as e: logger.debug("Fail to read registry key: {0}, value: {1}, unexpected error: {2}".format(keypath, value_name, e)) return None
def get_s3_files( bucket_name: str = BUCKET_NAME, save_to_disk: bool = True, files_list_path: str = FILES_LIST_PATH, files_list_filename: str = FILES_LIST_FILENAME, aws_region: str = AWS_REGION, s3_prefix: str = S3_PREFIX, ) -> list: """ Get S3 objects and creates list """ logger.info("Building media list from S3 objects...") logger.debug( f"Context Parameters: {get_s3_files.__name__} => {get_s3_files.__code__.co_varnames}" ) data = [] # testing format: assets/20160823/img.jpg pattern = re.compile( "^[a-z-A-Z-0-9]+/[a-z-A-Z-0-9]+/[0-9]{8}/.+[.][a-z-A-Z-0-9]+$" ) try: s3 = boto3.client("s3", region_name=aws_region) paginator = s3.get_paginator("list_objects_v2") pages = paginator.paginate(Bucket=bucket_name, Prefix=s3_prefix) for page in pages: for obj in page["Contents"]: if pattern.match(obj["Key"]): data.append(obj["Key"]) else: logger.warning( f'Wrong filename format, object "{obj["Key"]}", not added to the list.' ) statistics.append(["get_s3_files", len(data)]) logger.info("Media Objects list generated successfully.") logger.debug(f"Media objects count: {len(data)}.") if save_to_disk: logger.info("Writing media list to disk...") export_data = [f"{item}\n" for item in data] with open(f"{files_list_path}/{files_list_filename}", "w") as w: w.writelines(export_data) logger.info( f'List successfully saved to disk: "{files_list_path}/{files_list_filename}".' ) else: pass except Exception as e: logger.error(e) raise return data
def _get_cntk_version(cntk_root): logger.debug("In _get_cntk_version(), cntk_root: {0}".format(cntk_root)) version = '' version_file = os.path.join(cntk_root, "cntk", "version.txt") if os.path.isfile(version_file): with open(version_file) as fin: version = fin.readline().strip() logger.debug("In _get_cntk_version(), find cntk_version: {0}".format(version)) return version
def detect_python_version(): py_architecture = platform.architecture()[0] py_version = ".".join(map(str, sys.version_info[0:2])) py_full_version = ".".join(map(str, sys.version_info[0:3])) SysInfo.python = py_version.replace('.', '') logger.debug("In detect_python_version(), sys_info['python']: {0}".format(SysInfo.python)) logger.info("Python: {0}, {1}".format(py_full_version, py_architecture)) if not (_version_compare("3.5", py_version) and py_architecture == '64bit'): logger.error("64-bit Python 3.5 or higher is required to run this installer." " We recommend latest Python 3.5 (https://www.python.org/downloads/release/python-355/).") return False return True
def _update_pathenv_win(path, add): path_value = _registry_read(winreg.HKEY_CURRENT_USER, "Environment", "PATH") logger.debug("Before update, PATH: {0}".format(path_value)) if add: if path in path_value: return path_value = path + ";" + path_value os.environ["PATH"] = path + ";" + os.environ.get("PATH", "") else: path_value = path_value.replace(path + ";", "") os.environ["PATH"] = os.environ["PATH"].replace(path + ";", "") _registry_write(winreg.HKEY_CURRENT_USER, "Environment", "PATH", path_value)
def create_s3_bucket( bucket_name: str = BUCKET_NAME, aws_region: str = AWS_REGION ) -> bool: """ Create the S3 bucket of the project """ s3 = boto3.client("s3", region_name=aws_region) bucket_exists = True try: response = response = s3.list_buckets() buckets = [ bucket["Name"] for bucket in response["Buckets"] if bucket["Name"] == bucket_name ] if len(buckets) > 0: logger.warning( "S3 bucket already exists. Skipping bucket creation." ) bucket_exists = True else: bucket_exists = False except Exception as e: logger.error(e) raise if not bucket_exists: try: response = s3.create_bucket( Bucket=bucket_name, ACL="private", CreateBucketConfiguration={"LocationConstraint": aws_region}, ) logger.info(f'Created S3 bucket "{bucket_name}" successfully.') logger.debug(f"S3 client response: {response}") except Exception as e: logger.error(e) raise else: return False return True
def medias_copy( local_path: str = LOCAL_MEDIA_OUTPUT_PATH, video_encode: bool = VIDEO_ENCODE, media_encode_platform: str = MEDIA_ENCODE_PLATFORM, ) -> bool: """ Copy media files to S3 """ logger.info("Starting copy...") try: medias = get_local_medias_files(path=local_path, save_to_disk=False) logger.debug(medias) for media in medias: media_type = get_media_type(basename(media)) ts = media.split("/")[-2] if media_type == "movie": if video_encode == "True" and media_encode_platform == "cloud": send_to_bucket(media, ts) elif ( video_encode == "True" and media_encode_platform == "local" ): logger.info( f"Skipping copy of {media} for local re-encoding." ) elif media_type == "picture": send_to_bucket(media, ts) else: logger.warning(f"Media type is: {media_type} !") logger.info( f"{len(medias)} medias files have been successfully copied." ) except Exception as e: logger.error(e) raise statistics.append(["medias_copy", len(medias)]) logger.info("...done.") return True
def install_cntk_win(cntk_root): suc = True try: utils._update_pathenv_win(os.path.join(cntk_root, "cntk"), True) if (not utils.detect_mpi_win()): mpi_exe = os.path.sep.join( [cntk_root, "prerequisites", "MSMpiSetup.exe"]) logger.debug("MPI exe path: %s" % mpi_exe) logger.info("Begin to install MPI ...") utils._run_cmd_admin(mpi_exe, "-unattend") if (utils.detect_mpi_win()): logger.info("Install MPI successfully.") else: suc = False logger.error( "Fail to install MPI. Please manually install MPI >= 7.0.12437.6" ) if (not utils.detect_visualcpp_runtime_win()): vc_redist_exe = os.path.sep.join( [cntk_root, "prerequisites", "VS2015", "vc_redist.x64.exe"]) logger.debug("VC redist exe path: {0}".format(vc_redist_exe)) logger.info("Begin to install Visual C++ runtime ...") utils._run_cmd_admin(vc_redist_exe, "/install /norestart /passive") if (utils.detect_visualcpp_runtime_win()): logger.info("Install Visual C++ runtime successfully.") logger.warning( " Please manually install Visual C++ Redistributable Package for Visual Studio 2015 or 2017." ) else: suc = False logger.error("Fail to install Visual C++ runtime.") except: suc = False logger.error( "Fail to install CNTK(BrainScript). The error massage: {0}".format( sys.exc_info())) return suc
def pip_install_ml_software(pkg_info, options): logger.info( "Begin to install ml software(scikit-learn, xgboost and libsvm) ...") #1 scikit-learn name = pkg_info["ml_software"]["scikit-learn"]["name"] version = pkg_info["ml_software"]["scikit-learn"]["version"] pip_install_package(name, options, version) #2 xgboost name = pkg_info["ml_software"]["xgboost"]["name"] version = pkg_info["ml_software"]["xgboost"]["version"] if SysInfo.os != TOOLSFORAI_OS_WIN: if not pip_install_package(name, options, version): logger.warning( "In order to install xgboost, C++ compiler is needed.") else: wheel_ver = SysInfo.python arch = "win_amd64" pkg = "https://raw.githubusercontent.com/linmajia/ai-package/master/xgboost/{0}/xgboost-{0}-cp{1}-cp{1}m-{2}.whl".format( version, wheel_ver, arch) pip_install_package(name, options, version, pkg) #3 libsvm name = pkg_info["ml_software"]["libsvm"]["name"] version = pkg_info["ml_software"]["libsvm"]["version"] if SysInfo.os != TOOLSFORAI_OS_WIN: logger.warning( "Fail to install libsvm. On Linux or Mac, in order to install {0}=={1}, please manually download source code and install it." .format(name, version)) else: wheel_ver = SysInfo.python arch = "win_amd64" pkg = "https://raw.githubusercontent.com/linmajia/ai-package/master/libsvm/{0}/libsvm-{0}-cp{1}-cp{1}m-{2}.whl".format( version, wheel_ver, arch) logger.debug("Pip install libsvm from {0}".format(pkg)) pip_install_package(name, options, version, pkg)
def send_to_bucket( media_file: str, ts: str, bucket_name: str = BUCKET_NAME, s3_prefix: str = S3_PREFIX, aws_region: str = AWS_REGION, ) -> bool: """ Send file to S3 """ logger.info(f'Sending "{media_file}" to bucket "{bucket_name}"...') try: key = f"{s3_prefix}/{ts}/{basename(media_file)}" s3 = boto3.client("s3", region_name=aws_region) with open(media_file, "rb") as data: s3.upload_fileobj(data, bucket_name, key) logger.debug(f"media_file: {media_file} - key: {key}") except Exception as e: logger.error(e) raise logger.info(f'File "{media_file}" sent successfully to bucket: "{key}"') return True
def remote_video_encoder(files_list_path: str = FILES_LIST_PATH) -> bool: """ Send movies list to SQS -> lambda/ffmpeg """ logger.info("Starting remote movie re-encoding operations...") data_path = f"{files_list_path}/defered_encode.json" if os.path.exists(data_path): try: with open(data_path, "r") as r: movies = r.read() for movie in movies: queue_message = send_to_queue(movies) logger.info(f"Re-encoding process launched for '{movie}'.") logger.debug(queue_message) except Exception as e: logger.error(e) raise else: logger.critical(f"Path does not exist: '{data_path}'. Stopping here.") return False logger.info("...done.") return True
def create_table( table_name: str = TABLE_NAME, ReadCapacityUnits: int = TABLE_READ_CAPACITY_UNITS, WriteCapacityUnits: int = TABLE_WRITE_CAPACITY_UNITS, aws_region: str = AWS_REGION, ) -> bool: """ Creates DynamoB table """ try: client = boto3.client("dynamodb", region_name=aws_region) response = client.list_tables() tables = [ table for table in response["TableNames"] if table == table_name ] if len(tables) > 0: logger.warning( f'Table "{table_name}" already exists. Skipping table creation.' ) return False else: logger.info( f'Table "{table_name}" does not exist. Starting creation process...' ) except Exception as e: logger.error(e) raise logger.info("Creating DB table...") logger.debug( f"Context Parameters: {create_table.__name__} => {create_table.__code__.co_varnames}" ) try: dynamodb = boto3.resource("dynamodb", region_name=aws_region) table = dynamodb.create_table( TableName=table_name, AttributeDefinitions=[ {"AttributeName": "ts", "AttributeType": "S"} ], KeySchema=[{"AttributeName": "ts", "KeyType": "HASH"}], ProvisionedThroughput={ "ReadCapacityUnits": int(ReadCapacityUnits), "WriteCapacityUnits": int(WriteCapacityUnits), }, ) logger.info("Table created successfully.") logger.debug(table) except dynamodb.exceptions.ResourceInUseException as e: logger.warning( f'Table "{table_name}" already exists. Skipping table creation.' ) logger.debug(e) return False return True
def main(display_env: str = LOG_DISPLAY_ENV_VARS) -> None: """ Fetch, build and store S3 media files into DynamoDB """ if display_env == "True": logger.debug("## Environment variables") logger.debug(os.environ) else: pass logger.debug("- Start of execution -") prepare_local_resources() setup_cloud_resources() hydrate_cloud_resources() logger.debug("- End of execution -") print(tabulate(statistics)) monitor_remote_ops() logger.info("- All tasks executed successfully -")
def get_local_medias_files( path: str = LOCAL_MEDIA_PATH, save_to_disk: bool = True, files_list_path: str = FILES_LIST_PATH, files_list_filename: str = FILES_LIST_FILENAME, config_path: str = CONFIG_PATH, ) -> list: """ Generates a list of local media files """ if os.path.exists(path): local_medias = [] filtered_files = [] try: logger.info("Generating list of local files...") for dirpath, _, files in os.walk(path): for filename in files: fname = os.path.join(dirpath, filename) if is_filtered(filename): filtered_files.append(fname) else: local_medias.append(fname) if len(local_medias) > 0: statistics.append( ["get_local_medias_files", len(local_medias)] ) logger.info("List successfully generated.") logger.debug(f"Count: {len(local_medias)} local files.") else: logger.critical( f'No files found in source directory: "{path}".' ) return False if save_to_disk: logger.info("Writing local files list to disk...") data_to_write = [item + "\n" for item in local_medias] with open( f"{files_list_path}/{files_list_filename}", "w" ) as w: w.writelines(data_to_write) logger.info( f'The list has been saved successfully: "{files_list_path}/{files_list_filename}".' ) else: pass if len(filtered_files) > 0: logger.info( f'Number of file(s) excluded by filter specified in "{config_path}/exclude_local.txt": {len(filtered_files)}.' ) logger.debug(f"excluded by filter: {filtered_files}") else: pass except Exception as e: logger.error(e) raise return local_medias else: logger.critical(f'Missing input "path"! Stopping here!') return False
def _run_cmd(cmd, args=[], return_stdout=False): try: p = subprocess.run([cmd, *args], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) stdout = p.stdout.strip() stderr = p.stderr.strip() status = p.returncode == 0 logger.debug("========== {:^30} ==========".format("%s : stdout" % cmd)) for line in filter(lambda x: x.strip(), p.stdout.split('\n')): logger.debug(line) logger.debug("========== {:^30} ==========".format("%s : stdout end" % cmd)) logger.debug("========== {:^30} ==========".format("%s : stderr" % cmd)) for line in filter(lambda x: x.strip(), p.stderr.split('\n')): logger.debug(line) logger.debug("========== {:^30} ==========".format("%s : stderr end" % cmd)) except Exception as e: logger.debug("Fail to execute command: {0}, unexpected error: {1}".format(cmd, e)) status = False stdout = "" if return_stdout: return status, stdout else: return status
def media_generate( media: str = None, output_path: str = None, media_ts: str = None, output_image_width: int = None, output_image_height: int = None, processed_files_count: int = None, unprocessed_files: list = None, video_encode: str = VIDEO_ENCODE, log_path: str = LOG_PATH, s3_prefix: str = S3_PREFIX, media_encode_platform: str = MEDIA_ENCODE_PLATFORM, ) -> list: """ invoked by build_media_files_from_list() - gemerates media files """ media_name = media.split("/")[-1] media_type = get_media_type(media_name) if not os.path.exists(f"{output_path}/{media_ts}"): os.mkdir(f"{output_path}/{media_ts}") logger.debug(f'Created directory: "{output_path}/{media_ts}".') else: pass if media_type == "picture": logger.info( f"Picture type identified, starting generation of media...") image = Image.open(media) if image: with image as im: im.thumbnail((output_image_width, output_image_height)) im.save( f"{output_path}/{media_ts}/{media_name}", format="JPEG", quality="web_high", dpi=(72, 72), ) processed_files_count += 1 logger.info( f'Generated media: "{output_path}/{media_ts}/{media_name}".') else: logger.warning( f'Impossible to open the image file: "{media_name}"! File identified format is : "{media_type}". Skipping it.' ) elif media_type == "movie": if video_encode: logger.info(f"Movie type identified...") if media_encode_platform == "local": if os.path.exists(f"{log_path}/ffmpeg.log"): with open(f"{log_path}/ffmpeg.log", "r+") as w: w.truncate(0) else: pass video_encoder(media, media_ts, output_path) elif media_encode_platform == "cloud": logger.info(f"Movie type identified, starting copy of file...") shutil.copyfile(media, f"{output_path}/{media_ts}/{media_name}") logger.info( f'File copied successfully: "{media}" => "{output_path}/{media_ts}/{media_name}"' ) movie = f"{s3_prefix}/{media_ts}/{media_name}" cloud_video_encoder_list.append({ "src": movie, "ts": media_ts, "delete_old": True }) logger.info( f"Added movie '{movie}' to queue for defered remote re-encoding." ) else: logger.critical( 'Wrong or missing value! Valid values for "media_encode_platform": local|cloud' ) else: logger.info(f"Movie type identified, starting copy of file...") shutil.copyfile(media, f"{output_path}/{media_ts}/{media_name}") logger.info( f'File copied successfully: "{media}" => "{output_path}/{media_ts}/{media_name}"' ) processed_files_count += 1 else: unprocessed_files.append(media) logger.warning(f'Impossible to process file: "{media}". Skipping it.') return (processed_files_count, unprocessed_files)
def media_sync( local_path: str = LOCAL_MEDIA_OUTPUT_PATH, bucket_name: str = BUCKET_NAME, remote_path_prefix: str = S3_PREFIX, log_path: str = LOG_PATH, aws_region: str = AWS_REGION, config_path: str = CONFIG_PATH, ) -> bool: """ Synchronize local/S3 media files tree """ exclude_s3_file_path = f"{config_path}/exclude_s3.txt" if os.path.exists(exclude_s3_file_path): with open(exclude_s3_file_path, "r") as r: common_oses_filter = r.read().splitlines() cli_filter_args = "" cli_filter_args = cli_filter_args.join( [ f' --exclude "{item}"' for item in common_oses_filter if not item.startswith("#") or item != "\n" ] ) else: cli_filter_args = "" logger.info("Starting sync...") logger.info(f"S3 sync task log => tail -F {log_path}/s3_sync.log") try: cli_cmd = f"aws s3 sync {local_path}/ s3://{bucket_name}/{remote_path_prefix}/ --delete --region {aws_region} {cli_filter_args}" logger.debug(f"cli command: {cli_cmd}") with open(f"{log_path}/s3_sync.log", "w") as w: proc = subprocess.run( cli_cmd, shell=True, check=True, stdout=w, stderr=subprocess.STDOUT, universal_newlines=True, ) if proc.returncode == 0: with open(f"{log_path}/s3_sync.log", "r") as r: processed_objects = r.read().splitlines() processed_objects = [ item for item in processed_objects if "upload" in item ] statistics.append(["media_sync", len(processed_objects)]) logger.info("Sync completed successfully.") logger.debug( f"{len(processed_objects)} files have been synchronized successfully." ) logger.debug(f"S3 CLI returned code: {proc.returncode} => OK") else: logger.critical("Something wrong happened during sync operation!") return False except Exception as e: logger.error(e) raise return True
def install_cntk(target_dir): logger.info("Begin to install CNTK(BrainScript) ...") if SysInfo.os != TOOLSFORAI_OS_WIN and SysInfo.os != TOOLSFORAI_OS_LINUX: logger.warning( "CNTK(BrainScript) is not supported on your OS, we recommend 64-bit Windows-10 OS or 64-bit Linux OS." ) # fail_install.append("CNTK(BrainScript)") return False if SysInfo.cuda == "8.0": ver = "2.3.1" else: ver = "2.5.1" target_version = 'CNTK-{0}'.format(ver.replace('.', '-')) logger.debug( "In install_cntk(), target_version: {0}".format(target_version)) version = utils._get_cntk_version(target_dir) if target_version == version: logger.info('CNTK(BrainScript)-{0} is already installed.'.format(ver)) return True logger.debug('In install_cntk(), target_dir: {0}'.format(target_dir)) cntk_root = os.path.join(target_dir, 'cntk') if os.path.isdir(cntk_root): try: shutil.rmtree(cntk_root) except: logger.error( 'Fail to install CNTK(BrainScript), the error message: can not remove old version in directory {0}.' 'Please manually remove old version, and run the installer script again.' .format(cntk_root)) # fail_install.append("CNTK(BrainScript)") return False if not os.path.isdir(target_dir): try: os.makedirs(target_dir) except: logger.error( 'Fail to install CNTK(BrainScript), the error message: can not create directory {0}.' 'Please check if there is permission for creating directory.'. format(target_dir)) # fail_install.append("CNTK(BrainScript)") return False cntk_file_name = "{}-{}-64bit-{}.{}".format( target_version, "Windows" if SysInfo.os == TOOLSFORAI_OS_WIN else "Linux", "GPU" if SysInfo.gpu else "CPU-Only", "zip" if SysInfo.os == TOOLSFORAI_OS_WIN else "tar.gz") logger.debug( "In install_cntk(), cntk_file_name: {0}".format(cntk_file_name)) cntk_url = "https://cntk.ai/BinaryDrop/{0}".format(cntk_file_name) logger.debug("In install_cntk(), cntk_url: {0}".format(cntk_url)) cntk_file_path = os.path.join(target_dir, cntk_file_name) logger.debug( "In install_cntk(), cntk_file_path: {0}".format(cntk_file_path)) if SysInfo.os == TOOLSFORAI_OS_WIN: download_dir = cntk_file_path elif SysInfo.os == TOOLSFORAI_OS_LINUX: download_dir = os.path.join(r"/tmp", cntk_file_name) skip_downloading = False if not skip_downloading: if not utils._download_file(cntk_url, download_dir): logger.error( 'Fail to install CNTK(BrainScript), the error message: cannot download {0}.' 'Please check your network.'.format(cntk_url)) # fail_install.append("CNTK(BrainScript)") return False if (not (utils._unzip_file(download_dir, target_dir) if SysInfo.os == TOOLSFORAI_OS_WIN else utils._extract_tar( download_dir, target_dir))): logger.error( 'Fail to install CNTK(BrainScript), the error message: cannot decompress the downloaded package.' ) # fail_install.append("CNTK(BrainScript)") return False if not skip_downloading: if os.path.isfile(download_dir): os.remove(download_dir) if (SysInfo.os == TOOLSFORAI_OS_WIN): suc = install_cntk_win(cntk_root) else: suc = install_cntk_linux(cntk_root) version = utils._get_cntk_version(target_dir) if (suc and (target_version == version)): logger.info("Install CNTK(BrainScript) successfully!") logger.warning( "Please open a new terminal to make the updated Path environment variable effective." ) return True else: logger.error("Fail to install CNTK(BrainScript).") logger.warning( "Please manually install {0} and update PATH environment.".format( target_version)) logger.warning( "You can reference this link based on your OS: https://docs.microsoft.com/en-us/cognitive-toolkit/Setup-CNTK-on-your-machine" ) # fail_install.append("CNTK(BrainScript)") return False return True
def build_media_files_from_list( local_files_list: list = None, output_image_width: int = OUTPUT_IMAGE_WIDTH, output_image_height: int = OUTPUT_IMAGE_HEIGHT, output_path: str = LOCAL_MEDIA_OUTPUT_PATH, log_path: str = LOG_PATH, ) -> bool: """ Generates web friendly resized images and copy other media files """ logger.info("Generating web friendly images...") processed_files_count = 0 unprocessed_files = [] path_pattern = re.compile("^.*?/[0-9]{8}/.*[.][a-z-A-Z-0-9]+$") ts_pattern = re.compile("^[0-9]{8}$") try: for media in local_files_list: ts = media.split("/")[-2] if path_pattern.match(media): media_ts = ts elif not ts_pattern.match(ts): media_ts = media_ts_format(ts, media) else: logger.warning( f'The file path format should by like eg.: "path/ts/image.jpg".' ) logger.critical( f'Input file path format "{media}" is incorrect! Stopping here!' ) return False if not media_ts: unprocessed_files.append(media) logger.warning( f"Could not identify the date format. Skipping." ) else: gen = media_generate( media=media, output_path=output_path, media_ts=media_ts, output_image_width=output_image_width, output_image_height=output_image_height, processed_files_count=processed_files_count, unprocessed_files=unprocessed_files, ) # processed_files_count = gen[0] # unprocessed_files = gen[1] processed_files_count, unprocessed_files = gen statistics.append( ["build_media_files_from_list", processed_files_count] ) logger.info( f"{processed_files_count} images have been generated successfully." ) log_file = f"{log_path}/unprocessed_files.log" if len(unprocessed_files) > 0: up_files = [item + "\n" for item in unprocessed_files] with open(log_file, "w") as w: w.writelines(up_files) logger.warning(f"{len(unprocessed_files)} unprocessed file(s)!") logger.debug(f"Unprocessed file(s): {unprocessed_files}") elif os.path.exists(log_file): with open(log_file, "r+") as t: t.truncate(0) else: pass logger.info("Image files tree generation done.") if len(unprocessed_files) > 0: logger.info( f'Some files were not processed, please review the list: "{log_path}/unprocessed_files.log".' ) else: pass except Exception as e: logger.error(e) raise return True
def s3_clean( bucket_name: str = BUCKET_NAME, aws_region: str = AWS_REGION ) -> bool: """ Delete imcomplete multi-part uploads """ logger.info("Getting list of incomplete uploads...") try: multipart_uploads_cmd = f"aws s3api list-multipart-uploads --bucket {bucket_name} --region {aws_region}" proc = subprocess.run( multipart_uploads_cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, ) logger.debug( f'"s3api list-multipart-uploads" returns code: {proc.returncode} => OK' ) except Exception as e: if "exit status 254" in str(e): logger.warning( f"Bucket {bucket_name} does not exist. Stopping here." ) return False else: logger.error(e) raise if proc.returncode == 0 and proc.stdout: multipart_uploads_list = proc.stdout.strip() multipart_uploads_list = json.loads(multipart_uploads_list)["Uploads"] logger.info("Delete in progess...") try: for item in multipart_uploads_list: proc_cmd = f"aws s3api abort-multipart-upload --bucket {bucket_name} --region {aws_region} --key \"{item['Key']}\" --upload-id {item['UploadId']}" proc = subprocess.run( proc_cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, ) statistics.append(["s3_clean", len(multipart_uploads_list)]) logger.debug( f'"s3api abort-multipart-upload" returns code: {proc.returncode} => OK' ) logger.info(f"Deleted incomplete upload: \"{item['Key']}\".") logger.debug( f"{len(multipart_uploads_list)} incomplete upload(s) deleted." ) except Exception as e: logger.error(e) raise return True else: logger.info("Nothing to clean.") return True