def read_config():
""" Parses configuration
This function parses the configuration file and
load the configurations into the program
TODO: Do something about KeyError
"""
if not os.path.isfile(args.config): # Configuration not found
Avalon.error('SCUTUM configuration file not found! Please re-install SCUTUM!')
Avalon.warning('Please run \"scutum --install\" before using it for the first time')
raise FileNotFoundError(args.config)
# Initialize python confparser and read config
with open(args.config, 'r') as raw_config:
config = json.load(raw_config)
# Get controlled interfaces
interfaces = []
for interface in config['Interfaces']['interfaces']:
if os.path.isdir(f'/sys/class/net/{interface}'):
# Check if interface is connected
interfaces.append(interface)
# Get controlled network controllers
network_controllers = config['NetworkControllers']['controllers']
# Check if we should handle ufw
ufw_handled = config['Ufw']['handled']
# Get ARP Controller driver
arp_driver = config['ArpController']['driver']
return interfaces, network_controllers, ufw_handled, arp_driver
def run(self):
global successes
self.running = True
while self.running and (self.limit is None or successes < self.limit):
self.send_request()
time.sleep(self.interval)
Avalon.warning(f"Thread {self.name} exiting")
def enroll_settings():
settings = {}
settings['waifu2x_path'] = get_path('waifu2x-caffe-cui.exe path: ')
settings['ffmpeg_path'] = get_path('ffmpeg binaries directory: ')
settings['ffmpeg_arguments'] = []
while True:
argument = Avalon.gets(
'Extra arguments passed to ffmpeg (empty=none): ')
if argument:
settings['ffmpeg_arguments'].append(argument)
else:
break
settings['ffmpeg_hwaccel'] = Avalon.gets(
'ffmpeg hardware acceleration method (empty=auto): ')
if settings['ffmpeg_hwaccel'] == '':
settings['ffmpeg_hwaccel'] = 'auto'
settings['video2x_cache_folder'] = Avalon.gets(
'Video2X cache folder (empty=system default): ')
if settings['video2x_cache_folder'] == '':
settings['video2x_cache_folder'] = False
settings['preserve_frames'] = Avalon.ask(
'Preserve extracted or upscaled frames')
return settings
def install(self):
""" Install SCUTUM
This method will install all SCUTUM files and components
"""
Avalon.info('Starting installation procedure')
# Initialize configuration containers
self.config = {}
self.config['Interfaces'] = {}
self.config['NetworkControllers'] = {}
self.config['Ufw'] = {}
self.config['ArpController'] = {}
self._install_scutum_files()
self._install_service()
self._get_arp_controller_driver()
self._install_arp_controller_driver()
self._get_controlled_interfaces()
self._get_controlled_nm()
self._setup_ufw()
self._install_scutum_gui()
# Export the configuration into configuration file
with open(self.CONFPATH, 'w') as configfile:
json.dump(self.config, configfile, indent=2)
configfile.close()
print('\n' + Avalon.FM.BD, end='')
Avalon.info('SCUTUM installation completed')
Avalon.info('SCUTUM service is now enabled on system startup')
Avalon.info('You can now control it with systemd')
Avalon.info('You can also control it manually via the \"scutum\" command')
def install_wicd_scripts(self):
""" Write scutum scripts for WICD
"""
print(Avalon.FG.G + '[+] INFO: Installing for WICD' + Avalon.FM.RST + '.....', end='')
if not os.path.isdir('/etc/wicd/'):
print(Avalon.FG.R + Avalon.FM.BD + 'ERROR' + Avalon.FM.RST)
Avalon.warning('WICD folder not found! WICD does not appear to be installed!')
if Avalon.ask('Continue anyway? (Create Directories)', False):
Utilities.execute(['mkdir', '-p', '/etc/wicd/scripts/postconnect/'])
Utilities.execute(['mkdir', '-p', '/etc/wicd/scripts/postdisconnect/'])
else:
Avalon.warning('Aborting installation for WICD')
return False
with open('/etc/wicd/scripts/postconnect/scutum_connect', 'w') as postconnect:
postconnect.write('#!/bin/bash\n')
postconnect.write('scutum')
postconnect.close()
with open('/etc/wicd/scripts/postdisconnect/scutum_disconnect', 'w') as postdisconnect:
postdisconnect.write('#!/bin/bash\n')
postdisconnect.write('scutum --reset')
postdisconnect.close()
Utilities.execute(['chown', 'root:', '/etc/wicd/scripts/postconnect/scutum_connect'])
Utilities.execute(['chmod', '755', '/etc/wicd/scripts/postconnect/scutum_connect'])
Utilities.execute(['chown', 'root:', '/etc/wicd/scripts/postdisconnect/scutum_disconnect'])
Utilities.execute(['chmod', '755', '/etc/wicd/scripts/postdisconnect/scutum_disconnect'])
print(Avalon.FG.G + Avalon.FM.BD + 'SUCCEED' + Avalon.FM.RST)
return True
def _setup_ufw(self):
""" Enable UFW to controll the firewall
"""
print(Avalon.FM.BD + '\nEnable UFW firewall?' + Avalon.FM.RST)
print('Do you want SCUTUM to help configuring and enabling UFW firewall?')
print('This may help preventing a lot of scanning and attacks')
if Avalon.ask('Enable?', True):
# If ufw is not installed
if shutil.which('ufw') is None:
if Avalon.ask('UFW is not installed. Install?', True):
Utilities.install_packages(['ufw'])
else:
Avalon.warning('UFW package not available, disabling UFW')
self.config['Ufw']['handled'] = False
return
ufwctrl = Ufw()
print('SCUTUM can configure UFW Firewall for you')
print('However this will reset your current UFW configurations')
print('It is recommended to do so the first time you install SCUTUM')
if Avalon.ask('Let SCUTUM configure UFW for you?', True):
ufwctrl.initialize(True)
else:
Avalon.info('Okay. Then we will simply enable it for you')
ufwctrl.enable()
print('If you let SCUTUM handle UFW, then UFW will be activated and deactivated with SCUTUM')
if Avalon.ask('Let SCUTUM handle UFW?', True):
self.config['Ufw']['handled'] = True
else:
self.config['Ufw']['handled'] = False
else:
self.config['Ufw']['handled'] = False
Avalon.info('You can turn it on whenever you change your mind')
def upscale(self, input_directory, output_directory):
""" start upscaling process
"""
# overwrite config file settings
self.driver_settings['input_path'] = input_directory
self.driver_settings['output_path'] = output_directory
# list to be executed
# initialize the list with waifu2x binary path as the first element
execute = [self.driver_settings['path']]
for key in self.driver_settings.keys():
value = self.driver_settings[key]
# null or None means that leave this option out (keep default)
if key == 'path' or value is None or value is False:
continue
else:
if len(key) == 1:
execute.append(f'-{key}')
else:
execute.append(f'--{key}')
# true means key is an option
if value is not True:
execute.append(str(value))
# return the Popen object of the new process created
self.print_lock.acquire()
Avalon.debug_info(f'[upscaler] Subprocess {os.getpid()} executing: {shlex.join(execute)}')
self.print_lock.release()
return subprocess.Popen(execute)
def uninstall(self):
""" Remove SCUTUM completely
This method will remove all SCUTUM files from
the system. It is yet to be tested completely.
"""
self.remove_wicd_scripts()
self.remove_nm_scripts()
Utilities.execute(['ufw', '--force', 'reset']) # Reset ufw configurations
Utilities.execute(['rm', '-f', '/etc/ufw/*.*.*']) # Delete automatic backups
# A list of files, directories and links to remove
rmlist = ['/usr/bin/scutum',
self.INSTALL_DIR,
self.CONFPATH,
'/var/log/scutum.log',
'/usr/lib/systemd/system/scutum.service', '/etc/init.d/scutum',
'/etc/systemd/system/multi-user.target.wants/scutum.service'
]
# Remove all files in rmlist
for path in rmlist:
if os.path.isfile(path) or os.path.islink(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
Avalon.info('SCUTUM removed successfully!')
exit(0)
def convDataToPerFloor(self):
dbGot = json.loads(self.__db.checkExistPage(1)[1])
totalPage = int(dbGot['page']['total_page'])
for pageNum in range(totalPage):
gotData = self.__db.checkExistPage(pageNum + 1)
if not gotData:
Avalon.error('Can\'t Get Page %s,Skip' % pageNum)
continue
gotData = json.loads(gotData[1])
self.__writePageUserNameToDatabase(gotData)
for i in gotData['post_list']:
replyID = int(i['id'])
replyNum = int(i.get('sub_post_number', 0))
floorNumber = int(i['floor'])
publishTime = int(i['time'])
userID = int(i['author_id'])
userName = self.__getUserName(userID)
context = str(json.dumps(i))
self.__db.writeFloor(floorNumber, replyID, replyNum,
publishTime, userName, context)
Avalon.debug_info(
'[%s]Floor Info at Page %s Finished.Database Changed %s Record'
% (self.__tid, pageNum + 1, self.__dbTotalChange))
def enroll_settings():
settings = {}
settings['waifu2x_path'] = get_path('waifu2x-caffe-cui.exe path: ')
settings['ffmpeg_path'] = get_path('ffmpeg binaries directory: ')
settings['ffmpeg_arguments'] = []
while True:
argument = Avalon.gets('Extra arguments passed to ffmpeg (empty when done): ')
if argument:
settings['ffmpeg_arguments'].append(argument)
else:
break
settings['ffmpeg_hwaccel'] = Avalon.gets('ffmpeg hardware acceleration method (cuda): ')
if settings['ffmpeg_hwaccel'] == '':
settings['ffmpeg_hwaccel'] = 'cuda'
settings['extracted_frames'] = Avalon.gets('Temporary directory for extracted frames (empty for mkdtemp): ')
if settings['extracted_frames'] == '':
settings['extracted_frames'] = False
settings['upscaled_frames'] = Avalon.gets('Temporary directory for upscaled frames (empty for mkdtemp): ')
if settings['upscaled_frames'] == '':
settings['upscaled_frames'] = False
settings['preserve_frames'] = Avalon.ask('Preserve extracted or upscaled frames')
return settings
def get_video_info(self, input_video):
""" Gets input video information
This method reads input video information
using ffprobe in dictionary
Arguments:
input_video {string} -- input video file path
Returns:
dictionary -- JSON text of input video information
"""
# this execution command needs to be hard-coded
# since video2x only strictly recignizes this one format
execute = [
self.ffmpeg_probe_binary, '-v', 'quiet', '-print_format', 'json',
'-show_format', '-show_streams', '-i', input_video
]
# turn elements into str
execute = [str(e) for e in execute]
Avalon.debug_info(f'Executing: {" ".join(execute)}')
json_str = subprocess.run(execute, check=True,
stdout=subprocess.PIPE).stdout
return json.loads(json_str.decode('utf-8'))
def get_parameters(self):
""" Get common name and Org name from user
"""
while self.common_name == '':
self.common_name = Avalon.gets('CA Common Name: ')
while self.organization == '':
self.organization = Avalon.gets('Organization Name: ')
def get_pixel_formats(self):
""" Get a dictionary of supported pixel formats
List all supported pixel formats and their
corresponding bit depth.
Returns:
dictionary -- JSON dict of all pixel formats to bit depth
"""
execute = [self.ffmpeg_probe_binary, '-v', 'quiet', '-pix_fmts']
# turn elements into str
execute = [str(e) for e in execute]
Avalon.debug_info(f'Executing: {" ".join(execute)}')
# initialize dictionary to store pixel formats
pixel_formats = {}
# record all pixel formats into dictionary
for line in subprocess.run(
execute, check=True,
stdout=subprocess.PIPE).stdout.decode().split('\n'):
try:
pixel_formats[' '.join(line.split()).split()[1]] = int(
' '.join(line.split()).split()[3])
except (IndexError, ValueError):
pass
# print pixel formats for debugging
Avalon.debug_info(pixel_formats)
return pixel_formats
def create_temp_directories(self):
"""create temporary directory
"""
self.extracted_frames = tempfile.mkdtemp(dir=self.video2x_cache_directory)
Avalon.debug_info(f'Extracted frames are being saved to: {self.extracted_frames}')
self.upscaled_frames = tempfile.mkdtemp(dir=self.video2x_cache_directory)
Avalon.debug_info(f'Upscaled frames are being saved to: {self.upscaled_frames}')
def upscale(self, folderin, folderout, width, height):
"""This is the core function for WAIFU2X class
Arguments:
folderin {string} -- source folder path
folderout {string} -- output folder path
width {int} -- output video width
height {int} -- output video height
"""
# Print thread start message
self.print_lock.acquire()
Avalon.debug_info('[upscaler] Thread {} started'.format(
threading.current_thread().name))
self.print_lock.release()
# Create string for execution
# execute = ['{}'.format(self.waifu2x_path), '-p', self.method, '-I', 'png', '-i', '\"{}\"'.format(folderin), '-e', 'png', '-o', folderout, '-w', str(width), '-h', str(height), '-n', '3', '-m', 'noise_scale', '-y', self.model_type]
execute = '\"{}\" -p {} -I png -i \"{}\" -e png -o {} -w {} -h {} -n 3 -m noise_scale -y {}'.format(
self.waifu2x_path, self.method, folderin, folderout, width, height,
self.model_type)
print('Executing: {}'.format(execute))
subprocess.run(execute, shell=True, check=True)
# Print thread exiting message
self.print_lock.acquire()
Avalon.debug_info('[upscaler] Thread {} exiting'.format(
threading.current_thread().name))
self.print_lock.release()
def create_temp_directories(self):
"""create temporary directories
"""
# if cache directory unspecified, use %TEMP%\video2x
if self.video2x_cache_directory is None:
self.video2x_cache_directory = pathlib.Path(tempfile.gettempdir()) / 'video2x'
# if specified cache path exists and isn't a directory
if self.video2x_cache_directory.exists() and not self.video2x_cache_directory.is_dir():
Avalon.error(_('Specified or default cache directory is a file/link'))
raise FileExistsError('Specified or default cache directory is a file/link')
# if cache directory doesn't exist, try creating it
if not self.video2x_cache_directory.exists():
try:
Avalon.debug_info(_('Creating cache directory {}').format(self.video2x_cache_directory))
self.video2x_cache_directory.mkdir(parents=True, exist_ok=True)
except Exception as exception:
Avalon.error(_('Unable to create {}').format(self.video2x_cache_directory))
raise exception
# create temp directories for extracted frames and upscaled frames
self.extracted_frames = pathlib.Path(tempfile.mkdtemp(dir=self.video2x_cache_directory))
Avalon.debug_info(_('Extracted frames are being saved to: {}').format(self.extracted_frames))
self.upscaled_frames = pathlib.Path(tempfile.mkdtemp(dir=self.video2x_cache_directory))
Avalon.debug_info(_('Upscaled frames are being saved to: {}').format(self.upscaled_frames))
def upscale(self, folderin, folderout, width, height):
"""This is the core function for WAIFU2X class
Arguments:
folderin {string} -- source folder path
folderout {string} -- output folder path
width {int} -- output video width
height {int} -- output video height
"""
# Print thread start message
self.print_lock.acquire()
Avalon.debug_info('[upscaler] Thread {} started'.format(
threading.current_thread().name))
self.print_lock.release()
# Create string for execution
execute = '\"{}\" -p {} -I png -i \"{}\" -e png -o {} -w {} -h {} -n 3 -m noise_scale -y {}'.format(
self.waifu2x_path, self.method, folderin, folderout, width, height,
self.model_type)
subprocess.call(execute)
# Print thread exiting message
self.print_lock.acquire()
Avalon.debug_info('[upscaler] Thread {} exiting'.format(
threading.current_thread().name))
self.print_lock.release()
def install(self):
""" Start the installation
Start the installation and install all packages and
components.
"""
# A list of packages to be installed
# by system package manager
self.pm_installation_list = []
self._install_defense_matrix()
# self._install_passwdcmplx()
# Packages to be installed by pm
self._install_tigher()
self._install_rkhunter()
# Commit installation
if len(self.pm_installation_list) > 0:
Utilities.install_packages(self.pm_installation_list)
# Install SCUTUM separately since it's not a
# standard package
self._install_scutum()
print('\n' + Avalon.FM.BD, end='')
Avalon.info('Defense Matrix installation completed')
Avalon.info(
'You can now control it via the \"defense-matrix\" command')
def get_number_of_frames(self, input_file: str,
video_stream_index: int) -> int:
"""Count the number of frames in a video
Args:
input_file (str): input file path
video_stream_index (int): index number of the video stream
Returns:
int: number of frames in the video
"""
execute = [
self.ffmpeg_probe_binary,
"-v",
"quiet",
"-count_frames",
"-select_streams",
f"v:{video_stream_index}",
"-show_entries",
"stream=nb_read_frames",
"-of",
"default=nokey=1:noprint_wrappers=1",
input_file,
]
# turn elements into str
execute = [str(e) for e in execute]
Avalon.debug_info(f'Executing: {" ".join(execute)}')
return int(
subprocess.run(execute, check=True,
stdout=subprocess.PIPE).stdout.decode().strip())
def get_pixel_formats(self):
""" Get a dictionary of supported pixel formats
List all supported pixel formats and their corresponding
bit depth.
Returns:
dictionary -- JSON dict of all pixel formats to bit depth
"""
execute = [
self.ffmpeg_probe_binary,
'-v',
'quiet',
'-pix_fmts'
]
Avalon.debug_info(f'Executing: {" ".join(execute)}')
# initialize dictionary to store pixel formats
pixel_formats = {}
# record all pixel formats into dictionary
for line in subprocess.run(execute, check=True, stdout=subprocess.PIPE).stdout.decode().split('\n'):
try:
pixel_formats[' '.join(line.split()).split()[1]] = int(' '.join(line.split()).split()[3])
except (IndexError, ValueError):
pass
# print pixel formats for debugging
# this avalon command returns a code if it's not run in windows
# so we'll catch to skip in linux until avalon is patched
if sys.platform == 'win32':
Avalon.debug_info(pixel_formats)
return pixel_formats
def upscale(self, folderin, folderout, scale_ratio, threads):
""" Waifu2x Converter Driver Upscaler
This method executes the upscaling of extracted frames.
Arguments:
folderin {string} -- source folder path
folderout {string} -- output folder path
scale_ratio {int} -- frames' scale ratio
threads {int} -- number of threads
"""
# Print thread start message
self.print_lock.acquire()
Avalon.debug_info('[upscaler] Thread {} started'.format(
threading.current_thread().name))
self.print_lock.release()
# Create string for execution
execute = '\"{}\\waifu2x-converter-cpp.exe\" -i \"{}\" -o {} --scale_ratio {} --noise_level 3 -m noise_scale -j {} --model_dir {}\\models_rgb'.format(
self.waifu2x_path, folderin, folderout, scale_ratio, threads,
self.waifu2x_path)
subprocess.call(execute)
# Print thread exiting message
self.print_lock.acquire()
Avalon.debug_info('[upscaler] Thread {} exiting'.format(
threading.current_thread().name))
self.print_lock.release()
def _execute(self, execute: list) -> subprocess.Popen:
# turn all list elements into string to avoid errors
execute = [str(e) for e in execute]
Avalon.debug_info(f'Executing: {execute}')
return subprocess.Popen(execute)
def upscale(self, input_directory, output_directory, scale_ratio, upscaler_exceptions, push_strength=None, push_grad_strength=None):
""" Anime4K wrapper
Arguments:
file_in {string} -- input file path
file_out {string} -- output file path
Keyword Arguments:
scale {int} -- scale ratio (default: {None})
push_strength {int} -- residual push strength (default: {None})
push_grad_strength {int} -- residual gradient push strength (default: {None})
Returns:
subprocess.Popen.returncode -- command line return value of execution
"""
try:
# return value is the sum of all execution return codes
return_value = 0
# get a list lof all image files in input_directory
extracted_frame_files = [f for f in input_directory.iterdir() if str(f).lower().endswith('.png') or str(f).lower().endswith('.jpg')]
# upscale each image in input_directory
for image in extracted_frame_files:
execute = [
self.driver_settings['java_path'],
'-jar',
self.driver_settings['path'],
str(image.absolute()),
str(output_directory / image.name),
str(scale_ratio)
]
# optional arguments
kwargs = [
'push_strength',
'push_grad_strength'
]
# if optional argument specified, append value to execution list
for arg in kwargs:
if locals()[arg] is not None:
execute.extend([locals([arg])])
self.print_lock.acquire()
Avalon.debug_info(f'Executing: {execute}', )
self.print_lock.release()
return_value += subprocess.run(execute, check=True).returncode
# print thread exiting message
self.print_lock.acquire()
Avalon.debug_info(f'[upscaler] Thread {threading.current_thread().name} exiting')
self.print_lock.release()
# return command execution return code
return return_value
except Exception as e:
upscaler_exceptions.append(e)
def __init__(self,
input_video,
output_video,
method,
waifu2x_path,
ffmpeg_path,
waifu2x_driver='waifu2x_caffe',
ffmpeg_arguments=[],
ffmpeg_hwaccel='auto',
output_width=False,
output_height=False,
ratio=False,
model_type='anime_style_art_rgb',
threads=3,
video2x_cache_folder='{}\\video2x'.format(
tempfile.gettempdir()),
preserve_frames=False):
# Mandatory arguments
self.input_video = input_video
self.output_video = output_video
self.method = method
self.waifu2x_path = waifu2x_path
self.ffmpeg_path = ffmpeg_path
self.waifu2x_driver = waifu2x_driver
# Check sanity of waifu2x_driver option
if waifu2x_driver != 'waifu2x_caffe' and waifu2x_driver != 'waifu2x_converter':
raise Exception(
'Unrecognized waifu2x driver: {}'.format(waifu2x_driver))
# Optional arguments
self.ffmpeg_arguments = ffmpeg_arguments
self.ffmpeg_hwaccel = ffmpeg_hwaccel
self.output_width = output_width
self.output_height = output_height
self.ratio = ratio
self.model_type = model_type
self.threads = threads
# Create temporary folder/directories
self.video2x_cache_folder = video2x_cache_folder
self.extracted_frames_object = tempfile.TemporaryDirectory(
dir=self.video2x_cache_folder)
self.extracted_frames = self.extracted_frames_object.name
Avalon.debug_info('Extracted frames are being saved to: {}'.format(
self.extracted_frames))
self.upscaled_frames_object = tempfile.TemporaryDirectory(
dir=self.video2x_cache_folder)
self.upscaled_frames = self.upscaled_frames_object.name
Avalon.debug_info('Upscaled frames are being saved to: {}'.format(
self.upscaled_frames))
self.preserve_frames = preserve_frames
# If hardware acceleration enabled, append arguments
if self.ffmpeg_hwaccel:
self.ffmpeg_arguments.append('-hwaccel {}'.format(
self.ffmpeg_hwaccel))
def dpkg_purge_residual_configs(packages: list):
""" use dpkg to remove residual configs of the given packages
Args:
packages (list): a list of names of packages
"""
Avalon.debug_info("Purging residual configuration files")
subprocess.run(["/usr/bin/dpkg", "--purge"] + packages)
def get_path(text):
""" Get path and validate
"""
while True:
path = Avalon.gets(text)
if os.path.isdir(path):
return path
Avalon.error('{} id not a directory/folder'.format(path))
def __init__(self, driver='nftables'):
self.driver = driver
if self.driver != 'nftables' and self.driver != 'arptables':
Avalon.warning('Unrecognized ARP controller driver {}'.format(
self.driver))
Avalon.warning('Falling back to nftables')
self.driver = 'nftables'
def allow(self, port):
"""
Accept all traffic from one address
Arguments:
port {int} -- Port number
"""
Avalon.info(f'[UFW]: Allowing port {str(port)}\n\n')
Utilities.execute(['ufw', 'allow', f'{str(port)}/tcp'])
def resolve(self):
Avalon.info(f"正在对 {self._domain} 进行全球解析……")
self._session.cookies.clear()
self._get_src()
self._get_token()
self._get_dns_id()
self._global_query()
self._extend_query()
Avalon.info(f"{self._domain} 的全球解析已完成")
def expire(self, port):
"""
Disallows all traffic from one address
Arguments:
port {int} -- Port number
"""
Avalon.info(f'[UFW]: Expiring port {str(port)}\n\n')
Utilities.execute(['ufw', '--force', 'delete', 'allow', f'{str(port)}/tcp'])
