def check_new_ssh_connection():
"""Checks for a new ssh connection in the system
every new ssh session detected in the system, this funcion will sent a email
notification.
"""
server = 'bifrost.intel.com'
remote_user_ip = os.environ['SSH_CLIENT'].split()[0]
remote_username = bash.get_output('whoami')
machine_logged = bash.get_output(
'nslookup -timeout=10 {0}'.format(remote_user_ip)).split()[-1]
banner = '''
=======================================================
- session started from : {ip}
- logging as : {username}
- machine logged : {machine_logged}
=======================================================
'''.format(server=server,
ip=remote_user_ip,
username=remote_username,
machine_logged=machine_logged)
current_user_group = bash.get_output(
'groups {0} | grep sys-admin'.format(remote_username))
if 'sys-admin' in current_user_group:
print(banner_sys_admin())
print(banner)
send_email(remote_username, machine_logged, remote_user_ip)
def check_kernel_updates(self, target_folder):
"""Check for updates in the kernel tree.
:param target_folder, which is the kernel target folder to build.
:return
latest_commit_information: which is the full information from the
latest commit on the kernel tree.
latest_commit: which is the latest commit from the kernel tree.
"""
kernel_target_folder = target_folder
# checking if drm-tip is up-to-date
current_commit = bash.get_output(
'git -C {0} rev-parse HEAD'.format(kernel_target_folder))[0:7]
latest_commit = bash.get_output(
'git -C {0} ls-remote origin drm-tip'.format(
kernel_target_folder)).split()[0][0:7]
latest_commit_information = bash.get_output(
'git -C {path} show {commit} --format=fuller'.format(
path=kernel_target_folder, commit=latest_commit))
if current_commit != latest_commit:
self.log.info('{0} : is out-to-date, updating'.format(kernel_target_folder))
os.system('git -C {0} pull origin drm-tip'.format(kernel_target_folder))
os.system('git -C {0} reset --hard origin/drm-tip'.format(
kernel_target_folder))
else:
self.log.info('{0}: is up-to-date'.format(kernel_target_folder))
return latest_commit, latest_commit_information
def kill_wd(self, pid):
"""Kills an active watchdog
:param pid: the pid of the watchdog to be killed
"""
print('>>> killing pid: ' + str(pid))
try:
bash.get_output(
'cat /home/shared/suites/.miau | sudo -S kill -9 ' + str(pid))
print('>>> [' + str(pid) + '] killed successfully')
except ValueError:
print('>>> [' + str(pid) + '] failed to kill the pid')
def __init__(self):
self.config_file = '/home/custom/config.yml'
self.url = 'http://bifrost.intel.com:2020/watchdog'
self.distro = \
bash.get_output(
'lsb_release -d | awk \'{print $2 " " $3}\'').decode('utf-8')
self.codename = \
bash.get_output(
'lsb_release -c | awk \'{print $2}\'').decode('utf-8')
self.full_distro = self.distro + ' (' + self.codename + ')'
self.networking_service = \
bash.get_output(
'systemd-analyze blame | grep networking.service | '
'awk \'{print $1}\'').decode('utf-8')
def set_proxy(ip=None):
"""Automatically sets or unsets the proxy configuration.
The method will set or remove the environment variables related to
proxies depending on if the IP provided falls within the INTRANET range
(10.X.X.X) or SEDLINE range (192.168.X.X). This is useful because the hosts
that are connected to the sedline already have internet connection while those
in the intranet need some proxy configuration.
:param ip: the ip address of the host
:return: it configures the proxy settings but does not return anything.
"""
if not ip:
ip = get_ip()
# if there was a failure getting the ip just return False
if not ip:
return False
if ip.startswith('192.168.'):
# If the IP is from the public IP range unset proxies
proxies_list = [
'ALL_PROXY', 'all_proxy', 'http_proxy', 'https_proxy', 'ftp_proxy',
'socks_proxy', 'no_proxy'
]
for element in proxies_list:
if element in os.environ:
del os.environ[element]
else:
# intranet connection (setting proxies)
proxy_list = {
'ALL_PROXY':
'socks://proxy-socks.fm.intel.com:1080',
'all_proxy':
'socks://proxy-socks.fm.intel.com:1080',
'http_proxy':
'http://proxy.fm.intel.com:911',
'https_proxy':
'https://proxy.fm.intel.com:912',
'ftp_proxy':
'ftp://proxy.fm.intel.com:911',
'socks_proxy':
'socks://proxy-socks.fm.intel.com:1080',
'no_proxy':
'localhost,.intel.com,127.0.0.0/8,192.168.0.0/16,10.0.0.0/8'
}
for k, v in proxy_list.items():
os.environ[k] = v
if not bash.get_output('env | grep -i proxy'):
os.system('clear')
bash.center_message(
'\n{0}err:{1} (unable to configure proxies settings)\n'.format(
bash.RED, bash.END))
bash.exit_message(
bash.center_message(
'\n{0}please configure the proxies in your local bashrc{1}\n'
.format(bash.YELLOW, bash.END)))
return True
def upload_kernel_debian_packages(self, **kwargs):
"""Upload kernel debian packages.
:param kwargs:
- output_dir: which is the output kernel debian packages directory
where the packages will allocated in the server.
"""
output_dir = kwargs['output_dir']
# uploading the debian package
current_hostname = bash.get_output('echo ${HOSTNAME}')
if current_hostname != 'bifrost':
self.log.warning(
'the system is not allowed for upload the debian package')
self.log.info(
'please contact to the system-admin(s) if you need to upload it from here')
for system_admin in self.mailing_list:
self.log.info('- {0}'.format(system_admin))
sys.exit(1)
if self.ip.startswith('19'):
self.log.error(
'the package only will be uploaded on intranet connection')
sys.exit(1)
# creating the folder in the external server
path_to_upload_packages = os.path.join(
'/var', 'www', 'html', 'linuxkernel.com', 'kernels', self.kernel_id,
'WW{week_number}'.format(week_number=self.week_number))
self.log.info('creating the folder in : {cname}:{path}'.format(
cname=self.server_for_upload_package, path=path_to_upload_packages))
cmd = 'ssh {user}@{cname} mkdir -p {path} &> /dev/null'.format(
user=self.server_user, cname=self.server_for_upload_package,
path=path_to_upload_packages)
bash.check_output(
cmd,
'the folder was created successfully in: {0}'.format(
self.server_for_upload_package),
'could not created the folder in: {0}'.format(
self.server_for_upload_package),
print_messages=False
)
# uploading kernel debian packages
self.log.info('uploading debian package')
cmd = 'scp -o "StrictHostKeyChecking no" -r {input_path} ' \
'{user}@{cname}:{output_path} &> /dev/null'.format(
input_path=output_dir, user=self.server_user,
cname=self.server_for_upload_package,
output_path=path_to_upload_packages)
bash.check_output(
cmd,
'the debian package was uploaded successfully in : {0}'.format(
self.server_for_upload_package),
'could not uploaded the debian package in : {0}'.format(
self.server_for_upload_package),
print_messages=False
)
def create_qa_tag(self, commit):
"""Create a QA tag
The aim of this function is to check if a QA tag exists, if not this
this function create it.
:param commit, which is the commit for create the QA tag.
:return
does not return anything, only stops the execution in certain
point of the function.
"""
# qa tag name
date_today = bash.get_output('date +"%Y-%m-%d"')
qa_tag_name = 'drm-intel-qa-testing-' + date_today
# the qa folder is used for storage the qa tags
qa_tag_path = os.path.join('/home/shared/kernels_mx/qa-tag')
# checking if the current QA exits
kernel_qa_path = os.path.join(qa_tag_path, 'drm-tip')
if not os.path.exists(kernel_qa_path):
self.log.error('{0} : not exists'.format(kernel_qa_path))
return
check_qa_tag = os.system(
'git -C {path} tag -l | grep -w {tag}'.format(
path=kernel_qa_path, tag=qa_tag_name))
if check_qa_tag:
# updating drm-tip in order to create the tag
self.log.info(
'QA tag does not exits, creating QA tag ({tag}) in ({path})'.format(
tag=qa_tag_name, path=kernel_qa_path))
self.log.info('updating ({0})'.format(kernel_qa_path))
os.system(
'git -C {path} pull origin drm-tip'.format(
path=kernel_qa_path))
os.system(
'git -C {path} reset --hard origin/drm-tip'.format(
path=kernel_qa_path))
output = os.system(
'git -C {path} tag -a {tag} {commit} -m {tag}'.format(
path=kernel_qa_path, tag=qa_tag_name, commit=commit))
if output:
self.log.info(
'could not be create the tag: {0}'.format(
qa_tag_name))
return
self.log.info(
'the tag ({tag}) was created successfully in ({path})'.format(
tag=qa_tag_name, path=kernel_qa_path))
return
self.log.warning(
'QA tag ({tag}) already exits on ({path})'.format(
tag=qa_tag_name, path=kernel_qa_path))
def update_watchdog(self, ezbench_csv_file_path, elapsed_time):
"""Update watchdog web page
:param ezbench_csv_file_path, is the ezbench csv results path
:param elapsed_time, is the elapsed time.
:return
- trc_link: which is the trc link report.
- pass_test: which is passed tests.
- fail_test: which is the failed tests.
- total_test: which is the total test run.
- pass_rate_of_executed: which is the pass rate of the executed.
"""
trc_link = bash.get_output('cat {report}'.format(report=os.path.join(
'/home', self.dut_user, 'trc2.log'))).decode('utf-8')
pass_test = int(
bash.get_output('cat {0} | grep pass | wc -l'.format(
ezbench_csv_file_path)).decode('utf-8'))
fail_test = int(
bash.get_output('cat {0} | grep fail | wc -l'.format(
ezbench_csv_file_path)).decode('utf-8'))
total_test = int(
bash.get_output('cat {0} | wc -l'.format(
ezbench_csv_file_path)).decode('utf-8'))
# calculating the pass rate
try:
pass_rate_of_executed = round(pass_test / total_test * 100, 2)
except ZeroDivisionError:
pass_rate_of_executed = 'N/A'
apiutils.update_watchdog_db(
RaspberryNumber=self.raspberry_number,
PowerSwitchNumber=self.raspberry_power_switch,
Status='finished - {0} '.format(elapsed_time),
Pass=pass_test,
Fail=fail_test,
PassRate='{0}%'.format(pass_rate_of_executed),
OverallTime=elapsed_time,
TRCLink=trc_link)
return trc_link, pass_test, fail_test, total_test, pass_rate_of_executed
def dependencies(self):
dependencies_list = [
'sshpass', 'dos2unix', 'ssh', 'vim', 'git', 'w3m', 'vim-gnome',
'tmux', 'mate-terminal', 'synaptic', 'gparted'
]
for dependency in dependencies_list:
if not bash.get_output('dpkg -l | grep ' + dependency):
bash.message('info', 'Installing (' + dependency + ') ...')
os.system('echo ' + password + ' | sudo -S apt-get install ' +
dependency)
def __init__(self):
self.user = bash.get_output('whoami')
if self.user == 'root':
bash.message('info', 'please do not run this script as root')
sys.exit(1)
self.this_path = os.path.dirname(os.path.abspath(__file__))
self.main_path = os.path.join('/home/', self.user, 'intel-graphics',
'gfx_drivers_backup')
self.branch_to_check = 'master'
self.dut_user = '******'
self.package_name = 'xorgxserver'
def __init__(self):
self.suite = 'rendercheck'
self.easy_bugs = '/home/custom/graphic_stack/packages/easy-bugs'
if os.path.isfile(self.easy_bugs):
self.xserver_tag = bash.get_output(
'cat {0} | grep xserver -A 2 | grep tag | '
'sed \'s/ tag: //g\''.format(
self.easy_bugs)).decode('utf-8')
self.xserver_commit = bash.get_output(
'cat {0} | grep xserver -A 2 | grep commit | '
'sed \'s/ commit: //g\''.format(
self.easy_bugs)).decode('utf-8')
if not self.xserver_tag or not self.xserver_commit:
bash.message(
'err', 'xserver not found in this system on the following '
'file ({0})'.format(self.easy_bugs))
sys.exit(1)
else:
bash.message('err', '({0}) does not exist'.format(self.easy_bugs))
bash.message('info', 'is the DUT in the automated system ?')
sys.exit(1)
'''this dict indicates the relation between family and subtest
currently of rendercheck (from martin peres repository)
so far does not rendercheck does not shows the following families :
(shmblend/libreoffice_xrgb/gtk_argb_xbgr)'''
self.rendercheck_test_dict = {
'fill': 'fill',
'dcoords': 'dst coords',
'scoords': 'src coords',
'mcoords': 'mask coords',
'tscoords': 'transform src',
'tmcoords': 'transform mask',
'blend': 'blend',
'repeat': 'repeat',
'triangles': ['Triangles', 'TriStrip', 'TriFan'],
'bug7366': 'bug7366',
'cacomposite': 'composite CA',
'composite': 'composite',
'gradients': 'gradient'
}
def __init__(self):
# environment variables
os.environ['GIT_SSL_NO_VERIFY'] = '1'
self.thisPath = os.path.dirname(os.path.abspath(__file__))
self.data = yaml.load(open(os.path.join(self.thisPath, 'updater.yml')))
self.mailing_list = self.data['treeroot']['mailing_list']
self.repositories_path = self.data['treeroot']['repositories_path']
self.sender = self.data['treeroot']['sender']
# time variables
self.week_number = bash.get_output('date +"%-V"')
self.month = \
bash.get_output('month=`date +"%b"`; echo ${month^^}').lower()
self.week_day = bash.get_output('date +%A').lower()
self.year = bash.get_output('date +%G')
self.hour = bash.get_output('date +"%I-%M-%S %p"')
if not os.path.exists(self.repositories_path):
bash.center_message(bash.RED + '>>> (err)' + bash.END + ' (' +
self.repositories_path + ') does not exists')
sys.exit(1)
def get_software_configuration(self, goal):
if goal == 'graphic-stack':
drivers_configuration_file_path = \
'/home/custom/graphic_stack/packages/easy-bugs'
kernel_commit_file_path = \
'/home/custom/kernel/packages/commit_info'
if os.path.isfile(drivers_configuration_file_path) and \
os.path.isfile(kernel_commit_file_path):
if 'xserver' in bash.get_output(
'cat {drivers} | grep xserver'.format(
drivers=drivers_configuration_file_path)):
# in order to avoid that the length of the requested URL
# exceeds the capacity limit for TestReportCenter server
# only a few drivers will appears in easy-bugs
selected_drivers = [
'xserver', 'mesa', 'drm', 'macros', 'xf86-video-intel',
'libva', 'intel-vaapi-driver', 'cairo',
'intel-gpu-tools', 'piglit'
]
# creating a new easy-bugs
with open('/tmp/easy-bugs', 'w') as easy_bugs:
for driver in selected_drivers:
output = bash.get_output(
'cat {drivers} | grep {driver} -A2'.format(
drivers=drivers_configuration_file_path,
driver=driver))
easy_bugs.write('{0}\n\n'.format(output))
# moving easy-bugs as easy-bugs.old
os.rename(
drivers_configuration_file_path,
'/home/custom/graphic_stack/packages/easy-bugs.old')
# copying easy-bugs from /tmp
shutil.copy('/tmp/easy-bugs',
drivers_configuration_file_path)
drivers = bash.get_output('cat ' +
drivers_configuration_file_path)
kernel = bash.get_output('cat ' + kernel_commit_file_path)
drivers_header = '_' * 20 + 'Software Information' + '_' * 20
kernel_header = '_' * 20 + 'Kernel Information' + '_' * 20
string = \
drivers_header + '\n' + drivers + '\n\n' + \
kernel_header + '\n' + kernel
return string
else:
string = \
bash.get_output(
'python ../../tests/getdrivers.py -e graphic-stack')
return string
elif goal == 'update-tool':
string = \
bash.get_output(
'python ../../tests/getdrivers.py -e update-tool')
return string
def __init__(self):
self.this_path = os.path.dirname(os.path.abspath(__file__))
if os.path.exists(os.path.join(self.this_path, 'config.yml')):
self.system_user = bash.get_output('whoami')
self.data = \
yaml.load(open(os.path.join(self.this_path, 'config.yml')))
self.suite_list = self.data['testReportCenter']['suiteList']
self.now = datetime.datetime.now()
self.current_date = self.now.strftime('%Y-%m-%d')
r = str(
requests.get(
'http://10.64.95.35/api/BugManager/GetCurrentWW').json())\
.replace('"', '').split('-')
self.year = r[0]
self.work_week = 'W' + str(r[1])
else:
bash.message('err', '(config.yml) does not exists')
sys.exit(1)
def kill_api_process(self):
"""Kill the current api process id
:return
- True when the command return a exit status of 0.
- False when the command return a exit status different of 0.
"""
api_process = bash.get_output('pgrep -fan {0}'.format(self.api_script))
if not api_process:
self.log.warning('not process found for : {0}'.format(
self.api_script))
return True
api_pid = api_process.split()[0]
output = os.system('kill -9 {0}'.format(api_pid))
if output:
self.log.warning('could not kill : {0}'.format(self.api_script))
return False
return True
def get_wd_list(self):
"""Prints the list of active watchdogs"""
# Get the PIDs of active watchdogs
cmd = ("pgrep -af 'watchdog.py' | "
"awk '{for (I=1;I<=NF;I++) if ($I == \"watchdog.py\") "
"{print $(I+1) \"-\" $(I+2) \"-\" $1}; }'")
active_wd = bash.get_output(cmd).split()
watchdog_list = []
for wd in active_wd:
wd_info = wd.split('-')
raspberry = wd_info[0]
switch = wd_info[1]
pid = wd_info[2]
value = {
'RaspberryNumber': str(raspberry),
'switch': str(switch),
'pid': str(pid)
}
watchdog_list.append(value)
print(watchdog_list)
def send_email(remote_username, machine_logged, remote_user_ip):
"""Send a email notification
:param remote_username: the remote user name
:param machine_logged: the machine what is logged
:param remote_user_ip: the remote ip what is logged
"""
user_info = bash.get_output('id {0}'.format(remote_username)).replace(
' ', '\n')
sender = 'bifrost.intel.com@noreply'
mailing_list = '*****@*****.**'
subject = 'new ssh connection detected of: {0}'.format(remote_username)
message = '''
A new ssh connection was detected on bifrost, please see the details below:
{detail_info}
machine logged : {machine}
ip logged : {ip}
'''.format(detail_info=user_info, machine=machine_logged, ip=remote_user_ip)
utils.emailer(sender, mailing_list, subject, message, silent=True)
def check_for_manifest(self, target_folder):
"""Check for UTC integration manifest on kernel tree
:param target_folder, which is the kernel target folder to build.
"""
kernel_target_folder = target_folder
# checking where UTC manifest is
self.log.info('checking for manifest commit in : {0}'.format(
kernel_target_folder))
manifest = bash.get_output(
'git -C {0} log origin/drm-tip -n 1 --pretty=format:\'%s\' | '
'grep "UTC integration manifest"'.format(kernel_target_folder))
if manifest:
self.log.info('{0}: has UTC integration manifest'.format(
kernel_target_folder))
self.log.info(
'the kernel will be build with latest UTC integration manifest commit')
else:
self.log.warning('{0}: has not UTC integration manifest'.format(
kernel_target_folder))
self.log.info('the kernel will be build with latest commit')
def send_email(self, trc_link, pass_test, fail_test, total_test,
pass_rate_of_executed, elapsed_time):
"""Send a email to notify that the execution is finished
:param trc_link: which is the trc link report.
:param pass_test: which is passed tests.
:param fail_test: which is the failed tests.
:param total_test: which is the total test run.
:param pass_rate_of_executed: which is the pass rate of the executed.
:param elapsed_time: which is the elapsed time of the execution.
"""
self.log.info('sending the email')
# platform distribution
sys_info = platform.platform()
distro = sys_info.split(
'with-')[1] if 'with-' in sys_info else sys_info
# Getting the displays attached to the DUT
displays_attached = \
bash.get_output(
"sudo cat /sys/kernel/debug/dri/0/i915_display_info "
"| grep \"^connector\" | grep "
"-we \"connected\" | awk -F\"type \" '{print $2}' | "
"awk '{print $1}' | sed 's/,//g'").decode('utf-8').split()
displays_attached = ' & '.join(displays_attached)
body = '''
rendercheck execution has finished, please see the overall statistics
================================================
DUT configuration
================================================
User: {user}
Password: {pwd}
IP: {host_ip}
Hostname: {host}
Grub parameters: {grub}
guc: {guc}
huc: {huc}
dmc: {dmc}
Raspberry IP: {raspberry_ip}
Raspberry number: {raspberry_number}
Raspberry switch: {switch_number}
USB cutter serial: {usb_cutter}
Package: {package}
Gfx stack code: {stack_code}
Kernel branch: {kernel_branch}
Kernel commit: {kernel_commit}
Rounds: {iterations}
Default image: {image}
Distro: {distro}
================================================
Statistics
================================================
Total tests: {total_tests}
Passed: {tests_pass}
Failed: {tests_fail}
Pass rate: {pass_rate}%
{overall_time}
Attached displays: {displays}
TRC link: {trc_link}
'''.format(
user=self.dut_user,
pwd=self.data['dut_conf']['dut_password'],
host_ip=self.data['dut_conf']['dut_static_ip'],
host=self.dut_hostname,
grub=self.data['dut_conf']['grub_parameters'],
guc=self.data['firmwares']['guc'],
huc=self.data['firmwares']['huc'],
dmc=self.data['firmwares']['dmc'],
raspberry_ip=self.data['raspberry_conf']['raspberry_ip'],
raspberry_number=self.raspberry_number,
switch_number=self.raspberry_power_switch,
usb_cutter=self.data['raspberry_conf']['usb_cutter_serial'],
package=self.data['suite_conf']['default_package'],
stack_code=self.data['suite_conf']['gfx_stack_code'],
kernel_branch=self.data['suite_conf']['kernel_branch'],
kernel_commit=self.data['suite_conf']['kernel_commit'],
iterations=self.data['suite_conf']['igt_iterations'],
image=self.data['usb_conf']['default_image'],
distro=distro,
total_tests=total_test,
tests_pass=pass_test,
tests_fail=fail_test,
pass_rate=pass_rate_of_executed,
overall_time=elapsed_time,
displays=displays_attached,
trc_link=trc_link,
)
utils.emailer(
'*****@*****.**',
self.data['suite_conf']['default_mailing_list'],
'rendercheck execution finished on ({hostname})'.format(
hostname=self.dut_hostname), body)
def report_to_trc(self, ezbench_path, ezbench_folder_results):
"""Report to Test Report Center
This function generate a report from ezbench results and then report the
csv generated to TRC.
:param ezbench_path, is the ezbench path
:param ezbench_folder_results, is te ezbench folder results
:return
- ezbench_csv_file_path, which is the absolute path for the ezbench
csv results.
"""
# generating the report with the results from ezbench
this_path = os.path.dirname(os.path.abspath(__file__))
ezbench_input_folder = os.path.join(ezbench_path, 'logs',
ezbench_folder_results)
ezbench_output_folder = os.path.join(self.log_folder_path, 'results')
self.log.info('generating the report with the results from ezbench')
output = os.system('python {script} -s rendercheck -f {input_folder} '
'-o {output_folder} 2>> {log}'.format(
script=os.path.join(os.path.dirname(this_path),
'ezbench_reports.py'),
input_folder=ezbench_input_folder,
output_folder=ezbench_output_folder,
log=self.log_file))
if output:
self.log.error('an error occurred generating the report')
self.log.info('closing the log')
sys.exit(1)
# uploading the results to TestReportCenter
self.log.info('uploading the results to TestReportCenter')
autouploader_script = os.path.join(
os.path.dirname(os.path.dirname(this_path)), 'igt', 'autouploader',
'autoUploader.py')
ezbench_csv_file = bash.get_output('ls {0} | egrep "*.csv"'.format(
os.path.join(ezbench_output_folder, 'round_0'))).decode('utf-8')
ezbench_csv_file_path = os.path.join(ezbench_output_folder, 'round_0',
ezbench_csv_file)
environment = 'sandbox' if self.data['autouploader'][
'currentEnv'] == 'sand' else 'production'
is_sna_enabled = bash.get_output(
'cat /opt/X11R7/var/log/Xorg.0.log | grep -i sna').decode('utf-8')
current_driver = 'sna' if is_sna_enabled else 'modesetting'
self.log.info('current driver is: {0}'.format(current_driver))
autouploader_cmd = 'python2 {script} -p {platform} -s {suite} ' \
'-r {release} -t {title} -f {file} -e {environment} -o {objective} ' \
'-g {goal} -a {attachments} -c {comment}'.format(
script=autouploader_script,
platform=self.data['autouploader']['currentPlatform'],
suite=self.data['autouploader']['currentSuite'],
release=self.data['autouploader']['currentRelease'],
title=self.data['autouploader']['currentTittle'],
file=ezbench_csv_file_path,
environment=environment,
objective='installer',
goal='graphic-stack',
attachments=os.path.join(ezbench_output_folder, 'round_0', 'families'),
comment=current_driver)
self.log.info('autoUploader cmd')
self.log.info(autouploader_cmd)
output = os.system('{cmd} 2>> {log}'.format(cmd=autouploader_cmd,
log=self.log_file))
if output:
self.log.error('an error occurred uploading the report')
self.log.info('closing the log')
sys.exit(1)
return ezbench_csv_file_path
def check_for_updates(self, path_to_repo, branch):
"""Check for updates in each folder into updater.yml
:param path_to_repo: the selected path to the repository.
:param branch: the selected branch for the repository.
"""
local_commit = \
bash.get_output(
'cd ' + path_to_repo + ' && git rev-parse origin/' + branch)
remote_commit = \
bash.get_output(
'cd ' + path_to_repo + ' && timeout 5 git ls-remote origin ' +
branch + ' | awk \'{print $1}\'')
if local_commit != remote_commit:
bash.message(
'warn', '(' + path_to_repo + ') is (' + bash.YELLOW +
'out-to-date' + bash.END + ')')
bash.message('info', 'local commit (' + local_commit + ')')
bash.message('info', 'remote commit (' + remote_commit + ')')
bash.message('info', 'updating repository ...')
bash.message('info', 'counting the current commits ...')
old_commits_number = \
bash.get_output(
'cd ' + path_to_repo + ' && git rev-list origin/' +
branch + ' --count')
old_commit_info = \
bash.get_output(
'cd ' + path_to_repo +
' && git log -1 --format=fuller origin/' + branch)
bash.message('info', 'old commit information')
print(old_commit_info)
bash.message('cmd', 'git pull origin ' + branch)
os.system('cd ' + path_to_repo + ' && git pull origin ' + branch)
bash.message('cmd', 'git reset --hard origin/' + branch)
os.system('cd ' + path_to_repo + ' && git reset --hard origin/' +
branch)
new_commit_info = \
bash.get_output(
'cd ' + path_to_repo +
' && git log -1 --format=fuller origin/' + branch)
bash.message('info', 'new commit information')
print(old_commit_info)
bash.message('info', 'counting the new commits ...')
new_commits_number = bash.get_output('cd ' + path_to_repo +
' && git rev-list origin/' +
branch + ' --count')
new_commit = \
bash.get_output(
'cd ' + path_to_repo +
' && git rev-parse origin/' + branch)
if local_commit != new_commit:
bash.message(
'ok', '(' + os.path.basename(path_to_repo) + ') is (' +
bash.GREEN + 'up-to-date' + bash.END + ')')
commits_diff = int(new_commits_number) - int(
old_commits_number)
bash.message(
'info', 'commits downloaded : (' + str(commits_diff) + ')')
bash.message('info', 'sending a email notification')
utils.emailer(
self.sender, self.mailing_list,
'There is a new commit for (' +
os.path.basename(path_to_repo) + ') (' + self.month +
') (' + self.week_day + ') (' + self.hour + ')',
'There is a new commit for (' +
os.path.basename(path_to_repo) +
')\n\nCommits downloaded : (' + str(commits_diff) +
')\n\nLatest commit is:\n\n' + new_commit_info)
else:
bash.message('info', 'local commit (' + local_commit + ')')
bash.message('info', 'new commit (' + new_commit + ')')
bash.message(
'err', '(' + os.path.basename(path_to_repo) + ') is (' +
bash.YELLOW + 'out-to-date' + bash.END + ')')
bash.message(
'err', 'an error occurred trying to update (' +
os.path.basename(path_to_repo) + ')')
else:
bash.message(
'info', '(' + os.path.basename(path_to_repo) + ') is (' +
bash.GREEN + 'up-to-date' + bash.END + ')')
bash.message('info', 'local commit (' + local_commit + ')')
bash.message('info', 'remote commit (' + remote_commit + ')')
current_commit_info = \
bash.get_output(
'cd ' + path_to_repo +
' && git log -1 --format=fuller origin/' + branch)
bash.message('info', 'current commit information\n\n')
print(current_commit_info + '\n\n')
def reports(self, args):
"""Convert reports from ezbench to TRC format
:param args: this param contains the following variables:
- the folder which contains ezbench reports
- the current switch which is rendercheck
- the output folder where the results will be stored
"""
ezbench_reports = bash.get_output('ls {0} | grep stderr'.format(
args.folder)).decode('utf-8').split()
count = 0
for report in ezbench_reports:
round_folder = os.path.join(args.output, 'round_{0}'.format(count))
if not os.path.exists(round_folder):
os.makedirs(round_folder)
csv_output_a = '{0}_{1}_n_{2}.csv'.format(self.xserver_commit,
self.xserver_tag, count)
with open(os.path.join(args.folder, report), 'r') as item_a:
report_data = item_a.readlines()
for line in report_data:
test_result = line.split()[-1]
for key, value in self.rendercheck_test_dict.items():
csv_output_b = '{0}_{1}_{2}_n_{3}.csv'.format(
key, self.xserver_commit, self.xserver_tag, count)
# creating folder by family
round_folder_by_family = os.path.join(
round_folder, 'families')
if not os.path.exists(round_folder_by_family):
os.makedirs(round_folder_by_family)
if key == 'triangles':
for item in value:
if line.startswith('##') and item in line:
test_name = line \
.replace('## ', '') \
.replace(': {0}\n'.format(test_result), '')
# writing the main csv
write_csv(self.suite, key, test_name,
test_result, csv_output_a,
round_folder)
# writing the csv by family
write_csv(self.suite, key, test_name,
test_result, csv_output_b,
round_folder_by_family)
else:
if line.startswith('##') and value in line:
test_name = line \
.replace('## ', '') \
.replace('{0} '.format(value), '') \
.replace(': {0}\n'.format(test_result), '')
if key == 'composite' and 'CA' in line:
pass
else:
# writing the main csv
write_csv(self.suite, key, test_name,
test_result, csv_output_a,
round_folder)
# writing the csv by family
write_csv(self.suite, key, test_name,
test_result, csv_output_b,
round_folder_by_family)
count += 1
bash.message(
'info',
'the results are available in the following path ({0})'.format(
args.output))
def check_xserver_xorg():
"""Check the name of the configuration file xorg.conf in the system
After a debian package is setup through clonezilla environment
when Ubuntu starts for some reason the name of this file is changed to
xorg.conf<random_number> and if the file has not the correct name TTY7 (X)
will not be enabled.
"""
# validating if xserver is on graphic stack installed in the system
path_graphic_stack = '/home/custom/graphic_stack/packages'
xserver_in_easy_bugs = bash.get_output(
'ls {0} | grep xserver '.format(path_graphic_stack)).decode('utf-8')
if not xserver_in_easy_bugs:
graphic_stack_package = bash.get_output(
'ls {0} | grep .deb'.format(path_graphic_stack)).decode('utf-8')
logger.info('the current graphic stack does not contains xserver')
logger.info(graphic_stack_package)
utils.emailer(
sender, default_mailing_list,
'the current graphic stack ({stack}) does not contains xserver'.
format(stack=graphic_stack_package),
'The following system does not contains a graphic stack for xserver\n'
'host: {host}\nip: {ip}\n'.format(host=dut_hostname,
ip=dut_static_ip))
sys.exit(1)
if not os.path.exists(control_file):
xserver_original_name = 'xorg.conf'
xserver_system_name = bash.get_output(
'ls /etc/X11 | grep xorg.conf | grep -v failsafe').decode('utf-8')
logger.info(
'(xorg.conf) current name is: {0}'.format(xserver_system_name))
if xserver_original_name != xserver_system_name:
logger.info('changing to: {0}'.format(xserver_original_name))
output = \
os.system(
'sudo mv ' +
os.path.join('/etc', 'X11', xserver_system_name) +
' ' + os.path.join('/etc', 'X11', xserver_original_name))
if output != 0:
logger.error(
'an error has occurred trying to change the name of : {0}'.
format(xserver_system_name))
utils.emailer(
sender, default_mailing_list,
'an error has occurred trying to change the name of ({0}) on ({1}) ({2})'
.format(xserver_system_name, dut_hostname, dut_static_ip),
'The following system has poorly configured (xorg.conf), '
'please check it manually:\n - ({0})\n - ({1})'.format(
dut_hostname, dut_static_ip))
sys.exit(1)
else:
logger.info('({0}) was changed to ({1}) successfully'.format(
xserver_system_name, xserver_original_name))
logger.info(
'creating a control file into: {0}'.format(control_file))
os.system('touch ' + control_file)
logger.info('rebooting the system')
os.system('sudo reboot')
else:
logger.info(
'(xserver_system_name) and (xserver_original_name) are the same'
)
else:
bash.message('info', 'control file ({0}) exists'.format(control_file))
bash.message('info', 'nothing to do')
sys.exit(0)
def __init__(self, **kwargs):
"""Class constructor
:param kwargs:
- tag: build drm-tip kernel and create a QA-tag.
The accepted value is : True
- daily: build drm-tip kernel.
The accepted value is : True
- commit: build drm-tip kernel with a specific commit.
The commit must have at least 7 digits to be recognized by git.
"""
self.tag = kwargs.get('tag', None)
self.daily = kwargs.get('daily', None)
self.specific_commit = kwargs.get('commit', None)
list_to_validate = [self.tag, self.daily, self.specific_commit]
if list_to_validate.count(None) != 2:
raise RuntimeError('please set one value')
self.this_path = os.path.dirname(os.path.abspath(__file__))
self.data = yaml.load(open(os.path.join(self.this_path, 'kernel.yml')))
self.kernel_keys = self.data['drm-tip']['kernel_keys']
self.kernel_name = self.data['drm-tip']['kernel_name'].lower()
self.debian_packages_local_path = '/home/shared/kernels_mx/drm-tip'
self.kernel_folder_nickname = self.data['drm-tip']['kernel_folder_nickname']
self.kernel_id = 'drm-tip'
if self.tag:
self.log_path = '/home/shared/logs/kernel/drm-intel-qa'
self.kernel_keys = self.data['drm-intel-qa']['kernel_keys']
self.kernel_name = self.data['drm-intel-qa']['kernel_name'].lower()
self.debian_packages_local_path = '/home/shared/kernels_mx/drm-intel-qa'
self.kernel_folder_nickname = self.data['drm-intel-qa'][
'kernel_folder_nickname']
self.kernel_id = 'drm-intel-qa'
elif self.daily:
self.log_path = '/home/shared/logs/kernel/daily'
elif self.specific_commit:
self.log_path = '/home/shared/logs/kernel/specific_commit'
else:
self.log_path = '/home/shared/logs/kernel/drm-tip'
# initialize the logger
self.log_filename = 'kernel.log'
self.log = log.setup_logging(
name=self.log_filename, level='debug',
log_file='{path}/{filename}'.format(
path=self.log_path, filename=self.log_filename)
)
self.log.info('saving the log in: {log_file}'.format(
log_file=os.path.join(self.log_path, self.log_filename)))
# check if self.kernel_name does not exceeds 33 characters
if len(self.kernel_name) > 33:
self.log.error(
'{0} : exceeds 33 characters, please make it smaller'.format(
self.kernel_name))
sys.exit(1)
# check for characters not allowed in kernel name
# reference : https://www.debian.org/doc/debian-policy/#s-f-source
rule = 'package names must consist only of lower case letters (a-z), ' \
'digits (0-9), plus (+) and minus (-) signs, and periods (.)\n ' \
'They must be at least two characters long and must start with an ' \
'alphanumeric character'
characters_not_allowed = ['_', '~']
for character in characters_not_allowed:
if character in self.kernel_name:
self.log.error("character '{0}'not allowed in : {name}".format(
character, name=self.kernel_name))
self.log.info(rule)
sys.exit(1)
self.mailing_list = self.data['miscellaneous']['mailing_list']
# linuxgraphics.intel.com configuration
self.server_for_upload_package = self.data[
'miscellaneous']['server_for_upload_package']
self.server_user = self.data['miscellaneous']['server_user']
self.week_number = bash.get_output('date +"%-V"')
self.month = bash.get_output('month=`date +"%b"`; echo ${month^^}')
self.week_day = bash.get_output('date +%A').lower()
self.year = bash.get_output('date +%G')
self.hour = bash.get_output('date +"%I-%M-%S %p"')
self.enforce = self.data['miscellaneous']['enforce']
# getting the ip
self.ip = network.get_ip()
# environment variables
os.environ['GIT_SSL_NO_VERIFY'] = '1'
# this variable will change if there is a new commit for the kernel.
self.kernel_commit_built = None
def kernel_builder(self, **kwargs):
"""Dedicated function to build the kernel
The aim of this function is to build kernel from drm-tip and tagged as
drm-intel-qa in order to preserve the commit in our server.
:param kwargs: could has the following values
- kernel_target: which is the kernel source code to build
- latest_commit: which is the latest commit to build
- latest_commit_information: which is the full information from the
latest commit
- kernel_name: which is the kernel name
"""
kernel_target = kwargs['kernel_target']
latest_commit = kwargs['latest_commit']
latest_commit_information = kwargs['latest_commit_information']
kernel_name = kwargs['kernel_name']
utils.timer('start')
bash.check_output(
'git -C {0} checkout drm-tip'.format(kernel_target),
'git checkout branch drm-tip was successful',
'git checkout branch drm-tip was failed',
print_messages=False
)
bash.check_output(
'git -C {path} checkout {commit}'.format(
path=kernel_target, commit=latest_commit),
'git checkout commit drm-tip was successful',
'git checkout commit drm-tip was failed',
print_messages=False
)
# this is part of the new procedure in order to avoid issues with FWs
kernel_config = os.path.join(kernel_target, '.config')
self.log.info('copying config to: {0}'.format(kernel_config))
kernel_conf = os.path.join(self.this_path, 'conf.d', 'debug.conf')
copyfile(kernel_conf, kernel_config)
self.log.info('● setting kernel keys ●')
# naming the kernel
self.write_read_config_file(
config_file=kernel_config,
kernel_key='CONFIG_LOCALVERSION',
kernel_value='"-{name}-ww{week_number}-commit-{commit}"'.format(
name=kernel_name, week_number=self.week_number, commit=latest_commit)
)
# changing the kernel keys setup in kernel.yml
for key, value in self.kernel_keys.items():
self.write_read_config_file(
config_file=kernel_config,
kernel_key=key,
kernel_value=value
)
# creating config file to build the kernel
self.log.info('creating kernel config file')
os.system('cd {0} && make olddefconfig'.format(kernel_target))
self.log.info('creating a (tmp.d) folder')
os.makedirs(os.path.join(kernel_target, 'tmp.d'))
self.log.info('moving all kernel tree to (tmp.d)')
dest = os.path.join(kernel_target, 'tmp.d')
os.system('mv ' + kernel_target + '/{.,}* ' + dest + ' 2> /dev/null')
# build options
cores = int(bash.get_output('nproc')) + 2
self.log.info('compiling {kernel_name} with : {cores} cores'.format(
kernel_name=kernel_name, cores=cores))
# there is an internal Bash variable called "$PIPESTATUS" it’s an array
# that holds the exit status of each command in your last foreground
# pipeline of commands.
output = os.system(
'cd {path} && make -j{cores} deb-pkg | '
'tee {log_path}/kernel_compilation.log ; '
'bash -c "test ${{PIPESTATUS[0]}} -eq 0"'.format(
path=dest, cores=cores, log_path=self.log_path))
if output:
self.log.error('unable to compile drm-tip')
utils.emailer(
'*****@*****.**',
self.mailing_list,
'unable to compile {0} kernel'.format(kernel_name),
'bifrost.intel.com was unable to compile {name} kernel\n\n'
'The following commit was unable to compile: {commit}'.format(
name=kernel_name, commit=latest_commit))
utils.timer('stop', print_elapsed_time=False)
sys.exit(1)
self.log.info('{0} : was compiled successfully'.format(kernel_name))
# getting information from the kernel tree
custom_config = os.path.join(kernel_target, 'tmp.d', '.config')
dsc_file = bash.get_output('ls {0} | grep .dsc$'.format(kernel_target))
changes_file = bash.get_output('ls {0} | grep .changes$'.format(
kernel_target))
kernel_full_version = bash.get_output(
'cat {path} | grep -w ^Version'.format(
path=os.path.join(kernel_target, dsc_file))).split()[1]
kernel_version = kernel_full_version.split('-')[0]
my_list = kernel_full_version.split('-')
commit_index = my_list.index('commit') + 1
commit = my_list[commit_index].split('+')[0]
# this condition is when a new kernel does not contains the tag rc in
# their name
if re.search('rc', kernel_full_version):
release_candidate = kernel_full_version.split('-')[1]
kernel_version = '{version}-{rc}-{commit}'.format(
version=kernel_version, rc=release_candidate, commit=commit)
else:
kernel_version = '{version}-{commit}'.format(
version=kernel_version, commit=commit)
package_list = bash.get_output(
'cat {path} | grep -w ^Package-List -A 4 | '
'sed -e \'s/Package-List://g\' -e \'1d\''.format(
path=os.path.join(kernel_target, dsc_file)))
git_log = bash.get_output(
'git -C {path} show {commit} --format=fuller'.format(
path=os.path.join(kernel_target, 'tmp.d'), commit=commit))
# creating output directory for the kernel debian packages
pre_output_dir = os.path.join(
self.debian_packages_local_path, 'WW{0}'.format(self.week_number))
if self.enforce:
enforce_folders = int(bash.get_output(
'ls {path} | grep enforce | wc -l'.format(path=pre_output_dir))) + 1
if self.kernel_folder_nickname:
output_dir = os.path.join(
pre_output_dir,
'{kernel_version}-{week_day}-{nick_name}-enforce-{enforce}'.format(
kernel_version=kernel_version, week_day=self.week_day,
nick_name=self.kernel_folder_nickname, enforce=enforce_folders))
else:
output_dir = os.path.join(
pre_output_dir, '{kernel_version}-{week_day}-enforce-{enforce}'.format(
kernel_version=kernel_version, week_day=self.week_day,
enforce=enforce_folders))
else:
if self.kernel_folder_nickname:
output_dir = os.path.join(
pre_output_dir, '{kernel_version}-{week_day}-{nick_name}'.format(
kernel_version=kernel_version, week_day=self.week_day,
nick_name=self.kernel_folder_nickname))
else:
output_dir = os.path.join(
pre_output_dir, '{kernel_version}-{week_day}'.format(
kernel_version=kernel_version, week_day=self.week_day))
self.structure_debian_packages_folder(
output_dir=output_dir,
kernel_target=kernel_target,
git_log=git_log,
package_list=package_list,
dsc_file=dsc_file,
changes_file=changes_file,
custom_config=custom_config,
commit=commit
)
if self.tag:
# creating the QA tag
self.create_qa_tag(commit)
if self.data['miscellaneous']['upload_package']:
# uploading kernel debian packages
self.upload_kernel_debian_packages(output_dir=output_dir)
debian_packages_link = True
else:
self.log.info(
'the kernel debian packages will not be uploaded due to '
'(upload_package) key is set to False')
debian_packages_link = False
# sending a email notification
self.send_email(
output_dir=output_dir,
kernel_name=kernel_name,
debian_packages_link=debian_packages_link,
kernel_version=kernel_version,
git_log=git_log,
package_list=package_list,
latest_commit_information=latest_commit_information
)
utils.timer('stop', print_elapsed_time=False)
def test_get_output(self):
self.assertEqual('hello', bash.get_output('echo "hello"'))
def updates(self):
if not net_utils.get_ip():
bash.center_message('\n({0}Network is unreachable{1})\n'.format(
bash.RED, bash.END))
bash.exit_message(
bash.center_message(
'\n({0}To check updates from github.com you must be connected'
'to internet{1})\n'.format(bash.YELLOW, bash.END)))
else:
c_local_commit_sha = "git rev-parse origin/master"
c_remote_commit_sha = (
"timeout 2 git ls-remote origin master | awk '{print $1}'")
local_commit_sha = bash.get_output(c_local_commit_sha)
remote_commit_sha = bash.get_output(c_remote_commit_sha)
if local_commit_sha == remote_commit_sha:
os.system('clear')
bash.center_message('\nrepository [{0}up-to-date{1}]\n'.format(
bash.GREEN, bash.END))
else:
os.system('clear')
bash.center_message(
'\nrepository [{0}out-to-date{1}]\n'.format(
bash.RED, bash.END))
bash.center_message('({0}updating repository{1})'.format(
bash.YELLOW, bash.END))
count_current_commits = "git rev-list origin/master --count"
current_commit_info = "git log -1 origin/master"
c_updateBranch = "git pull origin master"
c_resetBranch = "git reset --hard origin/master"
c_new_commit_info = "git log -1 origin/master"
previous_commits = bash.get_output(count_current_commits)
print('({0}current sha{1})\n{2}\n'.format(
bash.CYAN, bash.END, bash.get_output(current_commit_info)))
bash.get_output(c_updateBranch, c_updateBranch)
bash.get_output(c_resetBranch, c_resetBranch)
print('\n({0}new sha{1})\n{2}\n'.format(
bash.CYAN, bash.END, bash.get_output(c_new_commit_info)))
c_current_commit_sha = "git rev-parse origin/master"
current_commit_sha = bash.get_output(c_current_commit_sha)
if local_commit_sha == current_commit_sha:
bash.center_message(
'({0}Something was wrong during updating master branch{1})\n'
.format(bash.RED, bash.END))
print('SHA before update : {0}'.format(local_commit_sha))
print('SHA after update : {0}'.format(
str(current_commit_sha)))
bash.center_message(
'({0}err{2} : {1}the SHAs are the same{2})\n'.format(
bash.RED, bash.YELLOW, bash.END))
else:
new_commits = bash.get_output(count_current_commits)
diff = int(new_commits) - int(previous_commits)
if diff > 0:
bash.center_message(
'({0}) commits downloaded during the update\n'.
format(str(diff)))
else:
bash.center_message(
'({0}no new commits were found{1})\n'.format(
bash.YELLOW, bash.END))
def get_data(self):
"""Grab data from the DUT
Creates a dictionary with the following information :
- with the info from config.yml located into /home/custom
- with the displays attached the the DUT reading i915_display_info
in order to send a post to watchdog database.
"""
if os.path.exists(self.config_file):
bash.message('info',
'(' + os.path.basename(self.config_file) + ') exists')
data = yaml.load(open(self.config_file))
default_package = data['suite_conf']['default_package']
raspberry_number = data['raspberry_conf']['raspberry_number']
raspberry_power_switch = \
data['raspberry_conf']['raspberry_power_switch']
kernel_branch = data['suite_conf']['kernel_branch']
kernel_commit = data['suite_conf']['kernel_commit']
gfx_stack_code = data['suite_conf']['gfx_stack_code']
dut_hostname = data['dut_conf']['dut_hostname']
dut_static_ip = data['dut_conf']['dut_static_ip']
grub_parameters = data['dut_conf']['grub_parameters']
guc = data['firmwares']['guc']
huc = data['firmwares']['huc']
dmc = data['firmwares']['dmc']
# Getting the displays attached to the DUT
displays_attached = \
bash.get_output(
"sudo cat /sys/kernel/debug/dri/0/i915_display_info "
"| grep \"^connector\" | grep "
"-we \"connected\" | awk -F\"type \" '{print $2}' | "
"awk '{print $1}' | sed 's/,//g'").decode('utf-8').split()
displays_dict = dict()
displays_dict['attachedDisplays'] = {}
for display in displays_attached:
displays_dict['attachedDisplays'][display] = 'active'
bash.message('info', 'default_package (' + default_package + ')')
data_out = {
'RaspberryNumber': raspberry_number,
'PowerSwitchNumber': raspberry_power_switch,
'Suite': default_package,
'Status': 'online',
'DutHostname': dut_hostname,
'Distro': self.full_distro,
'DutIP': dut_static_ip,
'KernelBranch': kernel_branch,
'KernelCommit': kernel_commit,
'GfxStackCode': gfx_stack_code,
'GrubParameters': grub_parameters,
'guc': guc,
'huc': huc,
'dmc': dmc,
'networking-service': self.networking_service
}
# Adding the current displays
data_out.update(displays_dict)
bash.message('info', 'printing dictionary ...')
print(data_out)
bash.message('info', 'sending post to database')
self.post_to_watchdog(data_out)
else:
bash.message(
'err',
'(' + os.path.basename(self.config_file) + ') does not exists')
sys.exit(1)
def uploader(self, url, token, platform, suite, release, title, file,
environment, objective, goal, attachments, comment):
# checking the architecture
arch = bash.get_output('arch')
if arch == 'x86_64':
architecture = '64 bits'
else:
architecture = '32 bits'
# fields for TestReportCenter
prefix = '\&'
report = 'report=@' + file
report_str = '--form ' + report + ' '
url_str = url + 'auth_token=' + token
release_version = 'release_version=' + release.replace(' ', '%20')
target = 'target=' + platform
testtype = 'testtype=' + urllib.quote_plus(suite)
test_environment = 'hwproduct=' + urllib.quote_plus(title)
image = 'image=' + self.year + '-' + self.work_week
build_id = 'build_id=' + architecture.replace(' ', '%20')
if comment:
title_string = \
urllib.quote_plus(
'[auto] ' + bash.get_output('uname -r').replace('+', '') +
' ' + bash.get_output('echo $HOSTNAME') + ' ' + suite +
' ' + architecture + ' ' + comment)
else:
title_string = \
urllib.quote_plus(
'[auto] ' + bash.get_output('uname -r').replace('+', '') +
' ' + bash.get_output('echo $HOSTNAME') + ' ' + suite +
' ' + architecture)
title = 'title="' + title_string + '"'
tested_at = 'tested_at=' + self.current_date
updated_at = 'updated_at=' + self.current_date
created_at = 'created_at=' + self.current_date
objective_txt = 'objective_txt=' + \
urllib.quote_plus(
bash.get_output(
'cat ' +
os.path.join(
self.this_path, 'objectives.d', objective + '.obj')))
environment_txt = 'environment_txt=' + \
urllib.quote_plus(self.get_software_configuration(goal))
qa_summary_txt = 'qa_summary_txt=' + \
urllib.quote_plus(bash.get_output(
'cat ' + os.path.join(
self.this_path, 'objectives.d', objective + '.obj')))
if attachments:
attachments_str = ''
file_list = \
[
os.path.join(attachments, name) for name in
os.listdir(attachments) if
os.path.isfile(os.path.join(attachments, name))
]
if len(file_list) == 1:
attachments_str += '--form attachment=@' + file_list[0] + ' '
else:
count = 1
for item in file_list:
attachments_str += '--form attachment.' + str(count) + \
'=@' + item + ' '
count += 1
cmd = report_str + attachments_str + url_str
else:
cmd = report_str + url_str
url_list = [
release_version, target, testtype, test_environment, image,
build_id, title, tested_at, updated_at, created_at, objective_txt,
environment_txt, qa_summary_txt
]
# creating the url
trc_url = ''
for item in url_list:
trc_url += prefix + item
bash.message('info', 'autoUploader.py v3.1')
bash.message(
'info', 'current environment for TRC is ({0})'.format(environment))
# ===============================================
# uncomment it to view the command line for curl
# ===============================================
# bash.message(
# 'cmd',
# 'curl -k --form ' + report + ' ' + url + 'auth_token=' + token +
# trc_url)
bash.message(
'info', 'uploading file (' + os.path.basename(file) +
') to TestReportCenter')
bash.message(
'info', '_________________' + bash.CYAN + 'Transfer rate' +
bash.END + '_________________')
# subprocess does not capture the STDERR of curl, for that reason
# is needed a file to capture it.
curl_stderr_file = '/tmp/autouploader.log'
output = bash.get_output('curl -k {0}{1} 2>&1 > {2}'.format(
cmd, trc_url, curl_stderr_file))
if os.path.exists(curl_stderr_file) and \
not os.stat(curl_stderr_file).st_size == 0:
# the following conditions indicates that the TRC is down
conditions_for_trc_down = [
'An error occurred'
in bash.get_output('cat {0}'.format(curl_stderr_file)),
'port 443: No route to host'
in bash.get_output('cat {0}'.format(curl_stderr_file)),
]
if True in conditions_for_trc_down:
bash.get_output('cat {0}'.format(curl_stderr_file), True)
bash.message('err', 'Oops! An unknown error occurred')
bash.message(
'info', 'looks like that TestReportCenter for (' +
environment + ') environment is down')
bash.message(
'info', 'please check if the site (' + bash.YELLOW +
url.replace('/api/import?', '') + bash.END + ') is online')
sys.exit(1)
elif 'Unknown CSV delimiter' in \
bash.get_output('cat {0}'.format(curl_stderr_file)):
bash.message('err', 'Oops! An error occurred')
bash.message(
'warn',
'the file ({0}) has unknown CSV format'.format(file))
sys.exit(1)
elif 'Cannot find project_to_product_id' in \
bash.get_output('cat {0}'.format(curl_stderr_file)):
bash.message('err', 'Oops! An error occurred')
bash.message(
'warn', 'the platform ({0}) is not in ({1}) '.format(
platform, release))
bash.message(
'info',
'if this is not a human error, please verify it with the '
'TRC admins\n'
'- [email protected]\n'
'- [email protected]')
sys.exit(1)
elif 'The length of the requested URL exceeds the capacity limit' in \
bash.get_output('cat {0}'.format(curl_stderr_file)):
bash.message('err', 'Oops! An error occurred')
bash.message(
'warn',
'The length of the requested URL exceeds the capacity limit this server '
'If you think this is a server error, please contact to: '
'*****@*****.**')
sys.exit(1)
# this print the stdout from curl command
print(output)
dictionary_from_trc = bash.get_output(
'cat {0}'.format(curl_stderr_file))
response = json.loads(dictionary_from_trc)
if int(response['ok']) == 1:
bash.message('info',
'_______________________________________________\n')
bash.message(
'info', 'the report was uploaded' + bash.GREEN +
' successfully' + bash.END)
bash.message(
'info', 'You can view your report here : (' + bash.YELLOW +
response['url'] + bash.END + ')')
bash.message('info', 'project : https://01.org/linuxgraphics')
bash.message('info',
'maintainer : [email protected]')
os.system('echo ' + response['url'] + '> /tmp/trc2.log')
os.system('echo ' + response['url'] + '> ' +
os.path.join('/home', self.system_user, 'trc2.log'))
sys.exit(0)
elif int(response['ok']) == 0:
bash.message('info',
'_______________________________________________\n')
bash.message('err', 'an error was occurred')
bash.message('info', output)
bash.message('info', 'please report this issue to the maintainer')
bash.message('info',
'maintainer : [email protected]')
sys.exit(1)
else:
bash.message('info',
'_______________________________________________\n')
bash.message('err', 'unhandled error')
bash.message('info', output)
sys.exit(1)
level='debug',
log_file='{path}/{filename}'.format(
path=log_path, filename=log_filename))
path_to_files = '/var/log'
# the files to check can have rotatory files.
files_to_check = ['syslog', 'kern.log']
# size in bytes
maximum_size_allowed = 15360
while True:
for archive in files_to_check:
# getting a list for the rotatory files (if any)
rotatory_files = bash.get_output('ls {0} | grep {1}'.format(
path_to_files, archive)).split()
for rotatory_file in rotatory_files:
# getting file size in bytes
file_size_bytes = os.path.getsize('{0}/{1}'.format(
path_to_files, rotatory_file)) / 1024
if file_size_bytes > maximum_size_allowed:
logger.warning(
'{0} : exceeds of the maximum size allowed'.format(
rotatory_file))
logger.info('current size : {0} MB'.format(file_size_bytes /
1024))
logger.info('maximum size allowed : {0} MB'.format(
maximum_size_allowed / 1024))
logger.info('removing : {0}'.format(rotatory_file))
output = os.system('sudo rm -rf {0}/{1}'.format(
path_to_files, rotatory_file))
