import os
import argparse
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from argparse import ArgumentDefaultsHelpFormatter
from logConfig import get_logger
logger = get_logger(__name__)
class PyDriveUtil(object):
KEY_MIME_TYPE = 'mimeType'
FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
def __init__(self, settings=None):
if settings is None:
settings = {"settings_file": None, "local_cred_file": "mycreds.txt"}
if os.path.exists(settings['local_cred_file']):
gauth = self.get_gauth(settings)
self.drive = GoogleDrive(gauth)
else:
raise Exception(
"Your current working dir doesn't include the client certificate file [%s]. Please make sure firefox/chrome profile creation is turned off, online mode is disabled, and GSuite and FB cases are not test targets!" % settings['local_cred_file'])
def get_file_object(self, folder_uri, file_name):
file_list = self.get_file_list(folder_uri=folder_uri)
for file_obj in file_list:
if file_name in file_obj['title']:
return file_obj
return None
import os
import time
import json
import platform
import logConfig
from urlparse import urljoin
logger = logConfig.get_logger(__name__)
class Environment(object):
DEFAULT_HASAL_DIR = os.getcwd()
DEFAULT_THIRDPARTY_DIR = os.path.join(DEFAULT_HASAL_DIR, "thirdParty")
DEFAULT_EXTENSIONS_DIR = os.path.join(DEFAULT_THIRDPARTY_DIR, "extensions")
DEFAULT_GECKODRIVER_DIR = os.path.join(DEFAULT_THIRDPARTY_DIR, "geckodriver")
DEFAULT_OUTPUT_DIR = os.path.join(DEFAULT_HASAL_DIR, "output")
DEFAULT_PROFILE_DIR = os.path.join(DEFAULT_HASAL_DIR, "resource")
DEFAULT_UPLOAD_DIR = os.path.join(DEFAULT_HASAL_DIR, "upload")
DEFAULT_FLOW_DIR = os.path.join(DEFAULT_HASAL_DIR, "flows")
DEFAULT_VIDEO_OUTPUT_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "videos")
DEFAULT_PROFILE_OUTPUT_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "profiles")
DEFAULT_WAVEFORM_OUTPUT_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "waveform")
DEFAULT_IMAGE_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "images")
DEFAULT_IMAGE_OUTPUT_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "images", "output")
DEFAULT_IMAGE_SAMPLE_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "images", "sample")
DEFAULT_GECKO_PROFILER_PATH = os.path.join(DEFAULT_THIRDPARTY_DIR, "geckoprofiler-signed.xpi")
DEFAULT_CHROME_DRIVER_PATH = os.path.join(DEFAULT_THIRDPARTY_DIR, "chromedriver")
DEFAULT_SIKULI_PATH = os.path.join(DEFAULT_THIRDPARTY_DIR, "sikulix") if os.path.isfile(os.path.join(DEFAULT_THIRDPARTY_DIR, "sikulix", "runsikulix")) else os.path.join(DEFAULT_THIRDPARTY_DIR)
DEFAULT_TEST_RESULT = os.path.join(DEFAULT_HASAL_DIR, "result.json")
DEFAULT_STAT_RESULT = os.path.join(DEFAULT_HASAL_DIR, "stat.json")
class CommonUtil(object):
RECORDER_LIST = [Environment.PROFILER_FLAG_AVCONV, ]
logger = get_logger(__file__)
@staticmethod
def atoi(text):
return int(text) if text.isdigit() else text
@classmethod
def natural_keys(cls, text):
return [cls.atoi(c) for c in re.split('(\d+)', text)]
@staticmethod
def _find_recorder_stat(total_recorder_settings):
recorder_name_list = []
total_recorder_flags = dict((item, False) for item in CommonUtil.RECORDER_LIST)
for k in total_recorder_flags.keys():
v = total_recorder_settings.get(k, {})
if v:
is_enabled = v.get('enable', False)
total_recorder_flags[k] = is_enabled
if is_enabled:
recorder_name_list.append(k)
CommonUtil.logger.debug('Enabled Recorders: {}'.format(recorder_name_list))
return recorder_name_list, total_recorder_flags
@staticmethod
def is_video_recording(settings):
"""
Checking the Video Recording settings.
@return: True if there is one Recorder, False if there is no Recorder. Raise exception when there are multiple Recorders.
"""
# TODO: load recorder settings from firefox_config now, it might be changed...
total_recorder_settings = settings.get('extensions', {})
recorder_name_list, total_recorder_flags = CommonUtil._find_recorder_stat(total_recorder_settings)
recording_enable_amount = sum(map(bool, total_recorder_flags.values()))
if recording_enable_amount == 0:
CommonUtil.logger.info('No Recorder.')
return False
elif recording_enable_amount == 1:
CommonUtil.logger.info('Enabled Recorder: {}'.format(recorder_name_list[0]))
return True
elif recording_enable_amount > 1:
CommonUtil.logger.error('More than one Recorder.')
raise Exception('More than one video recorders are enabled.\n{}'.format(total_recorder_flags))
@staticmethod
def is_validate_fps(settings):
"""
Checking the FPS validation settings.
@return: True if FPS validation is enabled, False if FPS validation is disabled
"""
try:
if CommonUtil.is_video_recording(settings):
total_recorder_settings = settings.get('extensions', {})
recorder_name_list, _ = CommonUtil._find_recorder_stat(total_recorder_settings)
if len(recorder_name_list) == 1:
recorder_name = recorder_name_list[0]
recorder_setting = total_recorder_settings.get(recorder_name, {})
is_validate_fps = recorder_setting.get('validate-fps', False)
CommonUtil.logger.info('The validate-fps of {} Recorder: {}.'.format(recorder_name, is_validate_fps))
return is_validate_fps
else:
return False
else:
return False
except:
return False
class CommonUtil(object):
RECORDER_LIST = [
Environment.PROFILER_FLAG_AVCONV, Environment.PROFILER_FLAG_OBS
]
logger = get_logger(__file__)
@staticmethod
def get_username():
"""
Get current username.
The UNIX platform will return the value of `pwd.getpwuid(os.getuid()).pw_name`.
The Windows platform will return 'USERNAME' of environ, or default username `user`.
@return: current username.
"""
if platform.system().lower() == 'windows':
username = os.environ.get('USERNAME')
if not username:
# default user name
username = '******'
else:
import pwd
username = pwd.getpwuid(os.getuid()).pw_name
return username
@staticmethod
def get_appdata_dir():
"""
Get current user's AppData folder.
Availability: Windows.
@return: current user's AppData folder.
"""
if platform.system().lower() == 'windows':
appdata_path = os.environ.get('APPDATA')
if not appdata_path:
# default user APPDATA folder
appdata_path = r'C:\Users\{username}\AppData\Roaming'.format(
username=CommonUtil.get_username())
else:
logger.error('Doesn\'t support get APPDATA at platform {}.'.format(
platform.system()))
appdata_path = ''
return appdata_path
@staticmethod
def get_user_dir():
"""
Get current user's home folder.
The UNIX platform will return the value of `pwd.getpwuid(os.getuid()).pw_dir`.
The Windows platform will return 'USERPROFILE' of environ.
@return: current user's home folder.
"""
if platform.system().lower() == 'windows':
user_dir = os.environ.get('USERPROFILE')
if not user_dir:
# default user dir
user_dir = r'C:\Users\{username}'.format(
username=CommonUtil.get_username())
else:
import pwd
user_dir = pwd.getpwuid(os.getuid()).pw_dir
return user_dir
@staticmethod
def execute_runipy_cmd(input_template_fp, output_ipynb_fp, **kwargs):
default_runipy_cmd = 'runipy'
ipynb_env = os.environ.copy()
for variable_name in kwargs.keys():
ipynb_env[variable_name] = str(kwargs[variable_name])
cmd_list = [default_runipy_cmd, input_template_fp, output_ipynb_fp]
return subprocess.call(cmd_list, env=ipynb_env)
@staticmethod
def get_mac_os_display_channel():
if platform.system().lower() == "darwin":
proc = subprocess.Popen([
'ffmpeg', '-f', 'avfoundation', '-list_devices', 'true', '-i',
'""'
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout = proc.stdout.read()
if "Capture screen" in stdout:
display_channel = stdout[stdout.index('Capture screen') - 3]
if display_channel in [str(i) for i in range(10)]:
return display_channel
return "1"
else:
return ":0.0"
@staticmethod
def load_json_string(input_string):
try:
json_object = json.loads(input_string)
except ValueError:
return {}
return json_object
@staticmethod
def load_json_file(fp):
if os.path.exists(fp):
try:
with open(fp) as fh:
json_obj = json.load(fh)
return json_obj
except Exception as e:
print e
return {}
else:
return {}
@staticmethod
def atoi(text):
return int(text) if text.isdigit() else text
@classmethod
def natural_keys(cls, text):
return [cls.atoi(c) for c in re.split('(\d+)', text)]
@staticmethod
def _find_recorder_stat(total_recorder_settings):
recorder_name_list = []
total_recorder_flags = dict(
(item, False) for item in CommonUtil.RECORDER_LIST)
for k in total_recorder_flags.keys():
v = total_recorder_settings.get(k, {})
if v:
is_enabled = v.get('enable', False)
total_recorder_flags[k] = is_enabled
if is_enabled:
recorder_name_list.append(k)
CommonUtil.logger.debug(
'Enabled Recorders: {}'.format(recorder_name_list))
return recorder_name_list, total_recorder_flags
@staticmethod
def is_video_recording(settings):
"""
Checking the Video Recording settings.
@return: True if there is one Recorder, False if there is no Recorder. Raise exception when there are multiple Recorders.
"""
# TODO: load recorder settings from firefox_config now, it might be changed...
total_recorder_settings = settings.get('extensions', {})
recorder_name_list, total_recorder_flags = CommonUtil._find_recorder_stat(
total_recorder_settings)
recording_enable_amount = sum(map(bool, total_recorder_flags.values()))
if recording_enable_amount == 0:
CommonUtil.logger.info('No Recorder.')
return False
elif recording_enable_amount == 1:
CommonUtil.logger.info('Enabled Recorder: {}'.format(
recorder_name_list[0]))
return True
elif recording_enable_amount > 1:
CommonUtil.logger.error('More than one Recorder.')
raise Exception(
'More than one video recorders are enabled.\n{}'.format(
total_recorder_flags))
@staticmethod
def is_validate_fps(settings):
"""
Checking the FPS validation settings.
@return: True if FPS validation is enabled, False if FPS validation is disabled
"""
try:
if CommonUtil.is_video_recording(settings):
total_recorder_settings = settings.get('extensions', {})
recorder_name_list, _ = CommonUtil._find_recorder_stat(
total_recorder_settings)
if len(recorder_name_list) == 1:
recorder_name = recorder_name_list[0]
recorder_setting = total_recorder_settings.get(
recorder_name, {})
is_validate_fps = recorder_setting.get(
'validate-fps', False)
CommonUtil.logger.info(
'The validate-fps of {} Recorder: {}.'.format(
recorder_name, is_validate_fps))
return is_validate_fps
else:
return False
else:
return False
except:
return False
@staticmethod
def get_value_from_config(config, key):
value = config.get(key)
if value is None:
logger.warn(
'There is no {key} in {config} config file (or the value is None).'
.format(key=key, config=config))
return value
@staticmethod
def update_json(input_data, filename, mode="r+"):
with open(filename, mode) as fh:
origin_data = json.load(fh)
origin_data.update(input_data)
fh.seek(0)
fh.write(json.dumps(origin_data))
@staticmethod
def subprocess_checkoutput_wrapper(input_cmd_list, **kwargs):
try:
output = subprocess.check_output(input_cmd_list, **kwargs)
return 0, output
except subprocess.CalledProcessError as e:
return e.returncode, e.message
@staticmethod
def deep_merge_dict(source_dict, destination_dict):
"""
>>> a = { 'first' : { 'all_rows' : { 'pass' : 'dog', 'number' : '1' } } }
>>> b = { 'first' : { 'all_rows' : { 'fail' : 'cat', 'number' : '5' } } }
>>> merge(b, a) == { 'first' : { 'all_rows' : { 'pass' : 'dog', 'fail' : 'cat', 'number' : '5' } } }
@param source_dict:
@param destination_dict:
@return:
"""
for key, value in source_dict.items():
if isinstance(value, dict):
# get node or create one
node = destination_dict.setdefault(key, {})
CommonUtil.deep_merge_dict(value, node)
else:
destination_dict[key] = value
return destination_dict
import os
import time
import json
import platform
import logConfig
from urlparse import urljoin
logger = logConfig.get_logger(__name__)
class Environment(object):
DEFAULT_HASAL_DIR = os.getcwd()
DEFAULT_THIRDPARTY_DIR = os.path.join(DEFAULT_HASAL_DIR, "thirdParty")
DEFAULT_EXTENSIONS_DIR = os.path.join(DEFAULT_THIRDPARTY_DIR, "extensions")
DEFAULT_GECKODRIVER_DIR = os.path.join(DEFAULT_THIRDPARTY_DIR,
"geckodriver")
DEFAULT_OUTPUT_DIR = os.path.join(DEFAULT_HASAL_DIR, "output")
DEFAULT_PROFILE_DIR = os.path.join(DEFAULT_HASAL_DIR, "resource")
DEFAULT_UPLOAD_DIR = os.path.join(DEFAULT_HASAL_DIR, "upload")
DEFAULT_FLOW_DIR = os.path.join(DEFAULT_HASAL_DIR, "flows")
DEFAULT_VIDEO_OUTPUT_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "videos")
DEFAULT_PROFILE_OUTPUT_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "profiles")
DEFAULT_WAVEFORM_OUTPUT_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "waveform")
DEFAULT_IMAGE_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "images")
DEFAULT_IMAGE_OUTPUT_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "images",
"output")
DEFAULT_IMAGE_SAMPLE_DIR = os.path.join(DEFAULT_OUTPUT_DIR, "images",
"sample")
DEFAULT_GECKO_PROFILER_PATH = os.path.join(DEFAULT_THIRDPARTY_DIR,
"geckoprofiler-signed.xpi")
