def performFakeUpgrade(scs, rootpath):
if not scs:
return
con = uSysDB.connect()
cur = con.cursor()
for sc, sc_rootpath in scs.values():
if not sc_rootpath:
sc_rootpath = rootpath
new_location = '%s:%s' % (sc.package.name,
os.path.join(sc_rootpath, sc.content.bin))
cur.execute(
"UPDATE sc_instances SET location_id = %s WHERE sc_id IN (SELECT sc_id FROM service_classes WHERE name= %s)",
(new_location, sc.package.name))
con.commit()
uAction.progress.do("fake upgrading service controllers")
cur.execute(
"SELECT component_id, name FROM service_classes sc JOIN sc_instances si ON (si.sc_id = sc.sc_id) ORDER BY component_id"
)
changed_scs_component_ids = [(x[0], x[1]) for x in cur.fetchall()
if x[1] in scs]
cur.close()
con.commit()
for sc in changed_scs_component_ids:
uLogging.info('%s', sc)
cid, name = sc
cid = int(cid)
uLogging.info("Upgrading %s(%s)", name, cid)
uAction.retriable(uFakePackaging.upgradeSC)(cid)
uAction.progress.done()
def removeComponent(host_id,
component_id,
hostname="host",
pkg_name="package",
removeDepends=None):
uAction.progress.do("removing %s(id = %s) from %s(id = %s)", pkg_name,
component_id, hostname, host_id)
deps = getDependencies(component_id)
while deps:
uLogging.info("%s-%s-%s (%s) installed on %s(%d) depends on %s (%s)",
deps[0].ctype, deps[0].name, deps[0].version,
deps[0].component_id, deps[0].hostname, deps[0].host_id,
pkg_name, component_id)
if ((removeDepends is None) and uDialog.askYesNo(
"Shall I remove it also?", False)) or removeDepends:
removeComponent(deps[0].host_id, deps[0].component_id,
deps[0].hostname, deps[0].name, removeDepends)
else:
raise Exception("Cannot deinstall package, there are dependencies")
deps = getDependencies(component_id)
con = uSysDB.connect()
cur = con.cursor()
cur.execute('SELECT pkg_id FROM components WHERE component_id = %s',
(component_id))
pkg_id = cur.fetchone()[0]
api = openapi.OpenAPI()
api.pem.packaging.uninstallPackageSync(host_id=host_id, package_id=pkg_id)
uAction.progress.done()
def handle_async_exec_error(exit_code, task_item, check_cmd_args, check_request_result):
cluster_upgrade_err_pattern = re.compile("ERROR CODE: (.+)")
cluster_upgrade_error_parsed = cluster_upgrade_err_pattern.search(check_request_result["out"])
if cluster_upgrade_error_parsed is not None:
app_code = int(cluster_upgrade_error_parsed.group(1))
if app_code == 1604:
uLogging.info("Agent already upgraded on host {host}".format(host=task_item.host))
elif app_code == 3010: # node needs to restart
uLogging.info("Some files were locked during upgrade and a restart of node {host} is required "
"to complete the installation. \nWARNING: You can do it later "
"but before the next upgrade.".format(host=task_item.host))
else:
# try to interpret MSI code
uLogging.err("Agent update on host {host} failed. MSI error: {error_description} ({error_id})."
"\n{manual_upgrade_instruction}".format(
host=task_item.host, error_description=msi_errors.get(app_code, '<unknown error>'),
error_id=app_code,
manual_upgrade_instruction=get_manual_upgrade_inst(exit_code, task_item.host)))
raise uUtil.ExecFailed(command=" ".join(check_cmd_args), status=check_request_result['exit_code'],
out=check_request_result['out'], err=check_request_result['err'])
else:
uLogging.err("Agent update on host {host} failed:\nUnknown Error\n{manual_upgrade_instruction}".format(
host=task_item.host, manual_upgrade_instruction=get_manual_upgrade_inst(exit_code, task_item.host)))
raise uUtil.ExecFailed(command=" ".join(check_cmd_args), status=check_request_result['exit_code'],
out=check_request_result['out'], err=check_request_result['err'])
def configureSystem(config, progress):
initSysLog(config)
def setupService(name, binaryPath, displayName, description):
nameQuoted = "\"%s\"" % name
uUtil.execCommand("net stop %s" % nameQuoted, [0, 2]) # service name invalid
uLogging.info("service %s stopped" % name)
uUtil.execCommand("sc delete %s" % nameQuoted, [0, 1060]) # service not exist
uLogging.info("service %s deleted" % name)
uUtil.execCommand("sc create %s binPath= \"%s\" DisplayName= \"%s\" start= auto" %
(nameQuoted, binaryPath, displayName))
uUtil.execCommand("sc description %s %s" % (nameQuoted, description))
uLogging.info("service %s created" % name)
setupService("pleskd", os.path.join(config.rootpath, "bin", "watchdog.exe"),
"PEM", "\"Operations Automation service\"")
syslogdName = "PEM syslogd"
setupService(syslogdName, os.path.join(config.rootpath, "syslog", "syslogd.exe") + " --service",
syslogdName, "\"Operations Automation Syslog Service\"")
uUtil.execCommand("net start \"%s\"" % syslogdName)
uLogging.info("service %s started" % syslogdName)
return
def importAPSTypePackages(aps_core_types_pkg, poa_core_services_pkg):
assert os.path.exists(aps_core_types_pkg)
assert os.path.exists(poa_core_services_pkg)
caFilename = uPEM.getSCProperty(con, 'SaaS',
'saas.aps.restful.server.CA_filename')
serverPort = uPEM.getSCProperty(
con, 'SaaS', 'saas.aps.resources.end.point.server.internalPort')
serverPath = uPEM.getSCProperty(
con, 'SaaS', 'saas.aps.resources.end.point.server.path')
certFilename = uPEM.getSCProperty(
con, 'SaaS',
'saas.aps.resources.end.point.server.certificate_filename')
for x in (False, True):
uLogging.info("importing POA-Core-Services package.")
try:
uPEM.execCtl('saas_ctl',
('install', serverPort, serverPath,
poa_core_services_pkg, caFilename, certFilename))
except Exception, e:
if x:
raise e
if not x:
uLogging.info("importing APSv2-Core-Types package.")
uPEM.execCtl('saas_ctl', ('import', aps_core_types_pkg))
def importPackages(newestPackages, plesk_root, pkg_counter=None):
# import pleskds - it registers platforms in system.
uLogging.info("Importing new packages")
new_pleskds = []
total_packages = 0
for pform_ctypes in newestPackages.values():
for ctype_packages in pform_ctypes.values():
total_packages += len(ctype_packages)
if not pkg_counter:
pkg_counter = uUtil.CounterCallback()
pkg_counter.set_total(total_packages)
for pform in newestPackages:
if newestPackages[pform].has_key(
'other') and newestPackages[pform]['other'].has_key('pleskd'):
new_pleskds.append(newestPackages[pform]['other']['pleskd'])
if new_pleskds:
uLogging.debug("importing new agent packages")
uAction.retriable(doMassImport)(plesk_root, new_pleskds, pkg_counter)
to_update = [x for x in new_pleskds if x.old]
if to_update:
updateManifestSources(plesk_root, to_update, pkg_counter)
for pform in newestPackages:
uLogging.debug("importing new packages for %s", pform)
for ctype in newestPackages[pform]:
if newestPackages[pform][ctype]:
uAction.retriable(doMassImport)(
plesk_root, newestPackages[pform][ctype].values(),
pkg_counter)
def execute(self, readonly, precheck=False):
if readonly:
raise Exception("SQLScript does not support readonly operations")
if not uPEM.is_sc_installed(self.owner):
uLogging.info("%s is not installed, skipping", self.owner)
return None
con = uSysDB.connect()
cur = con.cursor()
rv = None
for stmt in self.get_code():
uLogging.debug('executing %s', stmt)
kind = stmtKind(stmt)
if kind in ('BEGIN', 'COMMIT', 'ROLLBACK'):
uLogging.warn(
'%s statements are ignored, <SQL> action always implicitly begins and commits transaction',
kind)
else:
cur.execute(stmt)
con.commit()
uSysDB.close(con)
return rv
def checkUncompatibleJDK():
uLogging.info("Checking for incompatible JDK rpms (OpenJDK)...")
s = uUtil.readCmd(['rpm', '-qa', '*openjdk*', 'jdk'])
rpms = s.strip().split('\n')
bad_rpms = filter(lambda p: 'openjdk' in p, rpms)
if bad_rpms:
raise uPrecheck.PrecheckFailed('incompatible java rpms found: %s' %
bad_rpms, 'replace with JDK rpm shipped with PA')
def _getIds(self, index, package, platformId):
legacy = []
for version in index[package][platformId]:
if version[2]:
continue
legacy.append(version[0])
uLogging.info('Legacy arch-changed package: %s %s %s' %
(package, platformId, version))
return legacy
def checkNumberOfActiveTasks():
import uTasks
task_number = uTasks.getNumberOfActiveTasks()
if task_number > 2000:
uLogging.warn("Number of active tasks in Task Manager: %s. Too many active tasks.", task_number)
raise uPrecheck.PrecheckFailed("There are %s unprocessed, scheduled, running or failed tasks in Task Manager (including periodic).\nUpdate cannot be performed if this number is more than 2000" %
task_number, "Cancel failed tasks or wait for them to complete.")
else:
uLogging.info("Number of active tasks in Task Manager: %s. OK.", task_number)
def update_extensions_from_xml(doc, con):
api_exts = doc.getElementsByTagName('API:EXTENSION')
if not api_exts:
uLogging.info(
"Nothing to do: document does not describe any API Extensions.")
return
for ext in api_exts:
process_api_extension(con, ext)
def tunePostgresParams(pgsql, config):
if pgsql is None:
pgsql = uPgSQL.PostgreSQLConfig(version = uPgSQL.TargetPostgreSQLVersion, commander = config.__dict__.get("commander"))
tunePostgresLogs(pgsql.get_commander())
uLogging.info("Stopping PostgreSQL service before its tuning is initiated...")
pgsql.stop()
tunePostgresParamsImpl(pgsql)
pgsql.start()
def _cleanup_verify_db(config_v):
uLogging.info('Clear reference DB')
uSysDB.init(config_v)
assert config_v.database_name.endswith(
'_verify'
), "Can't clear database. DB name is %s, which is not verify DB!" % config_v.database_name
con_v = uSysDB.connect()
uDBSchema.dropOwnedBy(con_v, config_v.dsn_login)
con_v.commit()
uSysDB.close(con_v)
def syncStores(cls):
uLogging.info("Synchronization stores")
request = uHCL.Request(cls.get_host_id(), user='******', group='root')
request.command("python /usr/local/bm/tools_py/syncstores.py",
stdout='stdout',
stderr='stderr',
valid_exit_codes=[0])
output = request.perform()
uLogging.debug('done, output \n%s' % output['stdout'])
return output
def waitTasksComplete(where):
con = uSysDB.connect()
cur = con.cursor()
prev_num = 0
task_status_dict = {
# According to modules/platform/u/EAR/poakernel-ejb/src/main/clientidl/TaskManagement.idl
"n": "not queued",
"u": "unprocessed",
"b": "being scheduled",
"f": "failed",
"s": "rescheduled",
"e": "running",
"r": "restarted",
"c": "canceled"
}
while True:
cur.execute(
"SELECT t.name, t.task_id, t.status FROM tm_tasks t LEFT JOIN tm_task_references ref "
"ON (t.task_id = ref.task_id) WHERE " + where +
" and t.status != 'c' ORDER BY task_id")
tasks = [(row[0], row[1], row[2]) for row in cur.fetchall()]
if not tasks:
return
tasks_printable = "\n".join([
"\tName: {0},\tID: {1},\tStatus: {2} ({3})".format(
row[0], row[1], task_status_dict[row[2]], row[2])
for row in tasks
]).strip("[]")
uLogging.debug("Current unfinished tasks: \n" + tasks_printable)
failed = [t for t in tasks if t[2] not in ('s', 'u', 'e', 'b')]
if failed:
for t in failed:
name, tid, status = t
cur2 = con.cursor()
cur2.execute(
"SELECT action_output FROM tm_logs WHERE task_id = %s ORDER BY finished DESC",
tid)
row = cur2.fetchone()
if row:
output = str(row[0])
else:
output = ' no output'
uLogging.err("%s(id=%s) failed with %s", name, tid, output)
raise Exception("There are failed update tasks")
if prev_num != len(tasks):
# nm, tid, status = tasks[0]
running = ', '.join([t[0] for t in tasks if t[2] == 'e'])
uLogging.info("%s (%s more to go)", running, len(tasks))
else:
sys.stdout.write('.')
sys.stdout.flush()
prev_num = len(tasks)
time.sleep(1)
def checkOneHostAvailability(host, report_only=False):
try:
uLogging.info("Checking if host %s is available" % host)
ping(host.host_id)
except Exception, e:
msg = "Failed to ping host %s: %s" % (host, e)
if report_only:
return msg
else:
raise Exception(
msg + "\nYou can skip this host and upgrade it manually later with 'ignore', or abort the update. ")
def create_ssl_proxy(config, port=9443):
uLogging.info('Creating SSL proxy')
uUtil.execCommand(["yum", "-y", "install", "mod_ssl"])
key, cert = create_ssl_cert(config)
vh = ssl_proxy_vhost_template % {
'host': socket.gethostname(), 'port': port, 'ip': config.external_ip, 'key': key, 'cert': cert}
conf_file = 'ppa_ssl_proxy.conf'
f = open(os.path.join(httpd_conf_dir, conf_file), "w")
f.write(vh)
f.close()
service_control('restart', 'httpd')
def setSCProperty(con, scname, propname, propvalue, description=''):
cur = con.cursor()
cur.execute(
"SELECT c.pkg_id, si.component_id FROM sc_instances si JOIN service_classes sc ON (sc.sc_id = si.sc_id) JOIN components c ON (c.component_id = si.component_id) WHERE sc.name=%s", scname)
rows = cur.fetchall()
if rows:
for row in rows:
pkg_id = row[0]
component_id = row[1]
_setComponentProperty(component_id, pkg_id, propname, propvalue, description, scname, con, None)
else:
uLogging.info('sc %s is not installed, cannot set property for it', scname)
def set_progress(self, pval, what=""):
if what:
uLogging.info("*** %s ***", what)
self.percents = pval
self.scale.set(self.percents)
if what:
self.status.setText(what)
self.form.draw()
self.form.setTimer(1)
self.form.run()
_screen.refresh()
def updatePackagesRecords(con, host_id, old_platform_id, new_platform_id):
old_platform = uPEM.getPlatform(con, old_platform_id)
platforms = uPEM.getPlatformLine(con, old_platform)
cur = con.cursor()
cur.execute((
"SELECT p.pkg_id, c.component_id, p.name, p.ctype FROM packages p JOIN components c ON c.pkg_id = p.pkg_id WHERE c.host_id = %%s AND p.platform_id IN (%s)"
% ','.join(['%s'] * len(platforms))),
[host_id] + [p.platform_id for p in platforms])
for row in cur.fetchall():
uLogging.info("Update DB records for package %s" % row[2])
updateComponentPackage(con, row[1], row[2], row[3], new_platform_id,
row[0])
def call(self, verb, path, headers=None, data=None, cert=None, rheaders=None):
if isinstance(data, JsonNode):
data = repr(data)
elif (isinstance(data, (dict, list))):
data = json.dumps(data)
url = self.url + path
uLogging.info('%s %s ...' % (verb, url))
if headers and self.verbose:
uLogging.debug('Request headers:\n\t%s' % "\t".join(["%s: %s" % (h, headers[h]) for h in headers.keys()]))
if data and self.verbose:
uLogging.debug("Request:\n%r" % data)
if not(headers):
headers = dict()
req = urllib2.Request(url, headers=headers, data=data)
req.get_method = lambda: verb
resp = None
try:
if cert:
opener = urllib2.build_opener(HTTPSClientAuthHandler(cert, cert))
resp = opener.open(req)
else:
if _PYTHON_2_7_9_COMPAT:
context = ssl._create_unverified_context()
resp = urllib2.urlopen(req, context=context)
else:
resp = urllib2.urlopen(req)
except urllib2.HTTPError, error:
contents = error.read()
uLogging.err('API Error:')
uLogging.err(contents)
# APS Exceptions have the following structure (example):
# {
# "code": 500,
# "type": "APS::Hosting::Exception",
# "message": "Limit for resource ..."
# }
# In order to allow simple processing of this exception like
# ('something' in error.aps.message) we convert this exception
# to JSON directly here.
error.aps = API.APSExcStruct()
if len(contents):
error.aps = json.loads(contents, object_hook=lambda x: JsonNode(**x))
raise error
def perform_aps2_sys_pkgs_upgrade(binfo):
from poaupdater import uPEM, uAction
if not uPEM.is_sc_installed("SaaS"):
uLogging.info(
"No 'SaaS' SC installed found. Skip bundled APS Type's packages importing."
)
return
con = uSysDB.connect()
uAction.progress.do("importing APS Type's packages")
performAPSTypesUpgrade(con, binfo)
uAction.progress.done()
uSysDB.close(con)
def install_aps_db(scriptsDir, con=None):
if con is None:
con = uSysDB.connect()
initial = os.path.join(scriptsDir, "dbschema-%s.sql" % get_db_type())
uLogging.info("Installing APS database from %s" % initial)
execute_db_script(cursor=con.cursor(), path=initial)
con.commit()
db_ver = get_db_version(con)
uSysDB.close(con)
uLogging.info("New APS DB v%d.%d installed." % (db_ver[0], db_ver[1]))
def upgrade_aps_db(scriptsDir, con=None):
if con is None:
con = uSysDB.connect()
# To be sure
if not is_aps_db_installed(con):
install_aps_db(scriptsDir, con)
db_ver = get_db_version(con)
uLogging.debug("APS DB v%d.%d found." % (db_ver[0], db_ver[1]))
uLogging.info("Looking for application upgrades in '%s'" % scriptsDir)
cursor = con.cursor()
upgrade_found = None
sc_pattern = get_update_scripts_pattern()
sc_list = list_db_scripts(scriptsDir, sc_pattern)
for path in sc_list:
sc_matcher = sc_pattern.match(path)
sc_ver = (int(sc_matcher.group(1)), int(sc_matcher.group(2)))
if sc_ver[0] > db_ver[0] or (sc_ver[0] == db_ver[0]
and sc_ver[1] > db_ver[1]):
path = os.path.join(scriptsDir, path)
execute_db_script(path, cursor)
set_db_version(con, sc_ver[0], sc_ver[1])
con.commit()
uLogging.info("'%s' applied." % path)
upgrade_found = True
db_ver = get_db_version(con)
uSysDB.close(con)
if upgrade_found:
uLogging.info("APS DB upgraded to v%d.%d." % (db_ver[0], db_ver[1]))
else:
uLogging.info("No new upgrades for APS DB found.")
def activate(self, result):
global _screen
if result != self.browse_button:
return
uLogging.info("%s", result)
height = _screen.height * 2 / 3
width = 20
box = Listbox(height, scroll=True, returnExit=1, width=width)
chosen = None
form = Form()
grid = Grid(1, 1)
grid.setField(box, 0, 0)
form.add(grid)
form.addHotKey(ESC)
_screen.gridWrappedWindow(grid, self.dialog_title)
i = 0
selected = '..'
while not chosen:
files = os.listdir(self.cwd)
for i, f in enumerate(files):
if os.path.isdir(os.path.join(self.cwd, f)):
files[i] += '/'
files.sort(cmp_fn)
if self.cwd != '/':
files = ['..'] + files
for f in files:
box.append(f[0:width], f)
try:
box.setCurrent(selected + '/')
except KeyError:
pass
k = form.run()
if k == ESC:
break
curr = box.current()
selected = '..'
if curr == '..':
selected = os.path.basename(self.cwd)
self.cwd = os.path.dirname(self.cwd)
elif curr.endswith('/'):
self.cwd = os.path.realpath(os.path.join(self.cwd, curr))
else:
chosen = os.path.join(self.cwd, curr)
box.clear()
_screen.popWindow()
if chosen:
self.filename.set(chosen)
def verifyPostgresCertificate(pgsqlOnMaster):
uLogging.info("Make sure PostgreSQL server.crt satisfies the installation requirements according to the KB '%s'" % PG_CERT_VERIFY_KB_URL)
serverCrtPath = pgsqlOnMaster.get_data_dir() + "/server.crt"
runOnMaster = pgsqlOnMaster.get_commander()
# make sure certificate exists
try:
checkExists = runOnMaster("[ -e '%s' ] && echo 'OK'" % serverCrtPath)
if checkExists.strip() != 'OK':
raise Exception("Certificate '%s' not found" % serverCrtPath)
except Exception, ex:
uLogging.err(ex.message)
exceptionMsg = "Failed to validate existence of '%s' with error '%s'. Please refer to the KB %s" % (serverCrtPath, ex.message, PG_CERT_VERIFY_KB_URL)
uLogging.warn("\n%s\n%s\n%s" %( "*" * 150, exceptionMsg, "*" * 150))
raise Exception(exceptionMsg)
def _oneUniquePhase(table, con, columns, other_columns):
uniq_columns_str = ', '.join(columns)
other_column_names = [c.name for c in other_columns]
other_columns_str = ', '.join(other_column_names)
cur = con.cursor()
cur.execute("SELECT %(columns)s FROM %(tab)s GROUP BY %(columns)s HAVING COUNT(*) > 1" %
{'columns': uniq_columns_str, 'tab': table.name})
l_cur = con.cursor()
rows = [row for row in cur.fetchall()]
for row in rows:
comm_cond, comm_params = generateCondition(columns, [x for x in row])
l_cur.execute(("SELECT %(other)s FROM %(tab)s WHERE %(cond)s " %
{"other": other_columns_str, 'tab': table.name, 'cond': comm_cond}), comm_params)
not_uniq_rows = l_cur.fetchall()
uLogging.info('There are %s records in table %s with same %s (%s), but with different %s\n', len(
not_uniq_rows), table.name, uniq_columns_str, ', '.join([str(i) for i in row]), other_columns_str)
uLogging.info(
"(%s) should be unique, please select one row that should be kept intact (others will be deleted), or press 'a' to abort. Also, you can fix the issue manually and press 'r'\n", uniq_columns_str)
choice_map = {}
rows_map = {}
for i in xrange(len(not_uniq_rows)):
choice_map[str(i)] = 'Keep row %s\n' % ', '.join(['%s=%s' % (x[0].name, str(x[1]))
for x in zip(other_columns, not_uniq_rows[i])])
rows_map[str(i)] = not_uniq_rows[i]
choice_map['a'] = 'Abort'
choice_map['r'] = 'Retry'
answer = uDialog.select([' ', 'What should I do?'], choice_map)
if answer == 'a':
raise Exception("Duplicate records in table %s" % table.name)
elif rows_map.has_key(answer):
left_cond, left_params = generateNegativeCondition(other_column_names, [x for x in rows_map[answer]])
statement = "DELETE FROM %(tab)s WHERE %(cond)s AND (%(left_cond)s)" % {
'tab': table.name, 'cond': comm_cond, 'left_cond': left_cond}
uLogging.debug("%s (%s, %s)", statement, comm_params, left_params)
l_cur.execute(statement, comm_params + left_params)
con.commit()
else:
return False
return True
def depends(what, on, depend_map):
counter = 0
max_counter = len(depend_map) + 1
cur = what
while counter < max_counter:
counter += 1
if not depend_map.has_key(cur):
uLogging.info("%s does not depend on %s", what, on)
return False
elif depend_map[cur] == on:
uLogging.info("%s depends on %s", what, on)
return True
else:
cur = depend_map[cur]
raise Exception, "Cyclic dependency, update '%s' depends on itself" % cur
def removePgSlave(slaveHostID, masterRootPwd):
if not uLogging.logfile:
uLogging.init2("/var/log/pa/deregister_slave.log", True, False)
slave = uPEM.getHost(slaveHostID)
slaveCommunicationIP = uPEM.getHostCommunicationIP(slaveHostID)
runOnSlave = lambda cmd: uHCL.runHCLCmd(slaveHostID, cmd)
uLogging.info("Slave database server at %s (%s) is going to be removed.", slaveCommunicationIP, slave.name)
pghaSettings = getPghaSettings()
pgsqlOnSlave = None
try:
pgsqlOnSlave = uPgSQL.PostgreSQLConfig(commander = runOnSlave)
except Exception, e:
uLogging.info("Could not find slave database server at %s (%s): %s", slaveCommunicationIP, slave.name, str(e))
return
def check_unfinished_installation_tasks():
unfinished_tasks_num = uTasks.get_num_of_unfinished_installation_tasks()
if not unfinished_tasks_num:
uLogging.info("No unfinished installation tasks in Task Manager: OK.")
else:
msg = "A number of unfinished installation tasks have been fetched out during pre-check, " \
"total: %s. " \
"These issues prevent the update of being started." % unfinished_tasks_num
uLogging.warn(msg)
raise uPrecheck.PrecheckFailed(msg, "To resolve these issues, log in to Odin Automation, "
"go to Operations > Tasks, filter the tasks by the 'Install' "
"sample in their name, eliminate blocker factors "
"or fix the cause of the tasks failure"
"then wait for these tasks until they will finish "
"with the Successful status."
"Also you can cancel these tasks, but only if you know what you doing.")