def _check_ccache_install():
"""Check ccache is installed or not. """
CC = os.getenv('CC')
CXX = os.getenv('CXX')
# clang scan-build always fail with ccache.
if CC and os.path.basename(
CC) == 'ccc-analyzer' and CXX and os.path.basename(
CXX) == 'c++-analyzer':
console.debug('ccache is disabled for scan-build')
return False
try:
p = subprocess.Popen(['ccache', '-V'],
env=os.environ,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode == 0:
version_line = stdout.splitlines(True)[0]
if version_line and version_line.find('ccache version') != -1:
console.debug('ccache found')
return True
except OSError:
pass
return False
def execute(envcopy, command, path = "", preexec = "", silent = False):
if path:
envcopy.append('cd "%s"' % path)
if preexec:
envcopy.append(preexec)
# envcopy.append('entryenv=`env`')
# envcopy.append('if %s; then exitenv=`env`; if [[ "$entryenv" == "$exitenv" ]]; then exit 0; fi; echo "Env change!" >&2; echo $entryenv >&2; echo $exitenv >&2; fi; exit 1' % command)
envcopy.append('if %s; then exit 0; fi; exit 1' % command)
std = puke.Std()
console.info('Running command:')
for i in envcopy:
console.info(i)
puke.sh(envcopy, std=std, output = False)
if std.code == 1:
if silent:
raise()
console.debug("Monade shell stdout:", std.out)
console.fail("Monade shell stderr: %s" % std.err)
console.debug("Monade shell stdout:", std.out)
if std.err:
console.warn("Monade shell stderr:", std.err)
return std
def run(self, db, request):
if request.method == HTTPMethods.GET: # GET
if request.params is not None and 'email' in request.params.keys(
) and 'session' in request.params.keys():
console.debug('Stream Refresh from {} : {}'.format(
request.params['email'], request.params['session']))
if db.check_session(request.params['email'],
request.params['session']):
fn, ft = self.get_fn()
data = open(fn, 'rb').read()
console.debug('Sending {}'.format(fn))
return Response(code=HTTPResponseCodes.OK,
content_type=ft,
data=data)
else:
return Response(code=HTTPResponseCodes.UNAUTHORIZED,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
return Response(code=HTTPResponseCodes.BAD_REQUEST,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
return Response(code=HTTPResponseCodes.METHOD_NOT_ALLOWED,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
def change_ip():
'''
更换IP后，检查是否已经使用过。
如果以前用过，需要另换一个，直到获取未使用过的IP为止。
'''
IP_TEST_URL = 'http://api.ipify.org?format=json'
while True:
try:
send_signal_to_change_ip()
response = requests.get(IP_TEST_URL, proxies=const.PROXIES)
except Exception as ee:
console.error('error during change ip')
console.error('REASON: %s' % (str(ee)))
time.sleep(0.5)
else:
if response.status_code == 200:
data = json.loads(response.text)
new_ip_string = data['ip'] + '\n'
with open(const.USED_IP_FILE, 'r') as file:
used_ips = file.readlines()
file.close()
if new_ip_string not in used_ips:
with open(const.USED_IP_FILE, 'a') as file:
file.write(new_ip_string)
file.close()
print('CURRENT IP: %s' % (data['ip']))
break
else:
console.debug('retry...')
def _check_dccc_install():
"""Check dccc is installed or not. """
home_dir = os.environ.get('HOME', '')
if home_dir and os.path.exists(os.path.join(home_dir, 'bin', 'dccc')):
console.debug('dccc found')
return True
return False
def ssh_execute_cli(ip, user, passwd, port, cli_list, prompt, timeout, retry_times, logdir, logfile, is_debug, is_shell, shellpasswd, wait_time, bootpasswd=[]):
###define format
timeout = int(timeout)
wait_time = float(wait_time)
absolute_logfile = logdir + '/' + logfile
debug('logfile path is %s' % absolute_logfile, is_debug)
port = int(port)
###set private var
login_timeout = 2
login_retry_times = retry_times
cli_timeout = timeout
cli_retry_times = 2
logout_timeout = 2
logout_retry_times = 10
###start ssh process
ssh = ssh_host(ip, user, passwd, port, absolute_logfile, prompt, wait_time, is_debug)
ssh_host_login = ssh.login(login_timeout, login_retry_times)
if ssh_host_login:
cli_sendmode_expect_timeout_wait_list = generate_cli_sendmode_expect_timeout_wait_list(cli_list, prompt, cli_timeout, wait_time, passwd, shellpasswd, bootpasswd)
debug('Cli_sendmode_expect_timeout_wait_list is as below: %s' % str(cli_sendmode_expect_timeout_wait_list), is_debug)
ssh_host_execute = ssh.execute_command_via_cli_sendmode_expect_timeout_wait_list(ssh_host_login, cli_sendmode_expect_timeout_wait_list, cli_retry_times)
else:
print 'ssh login failed'
return None
###logout
if ssh_host_execute:
ssh_host_logout = ssh.logout(ssh_host_execute, logout_timeout, logout_retry_times)
else:
print 'ssh execute cli failed'
return None
return ssh_host_logout
def open(self):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
console.debug("Socket successfully created")
except socket.error as err:
console.error("socket creation failed with error {}".format(err))
s.connect((settings.GARAGE_DOOR_IP_ADDRESS, settings.GARAGE_DOOR_PORT))
console.debug("the socket has successfully connected to {}".format(
settings.GARAGE_DOOR_IP_ADDRESS))
data = self.build(cmd=0)
s.send(data)
console.debug(data.decode('utf-8'))
console.debug("Successfully sent message")
resp = Response(raw=s.recv(4096))
console.debug("RESPONSE: {}".format(resp.status))
if resp.status == HTTPResponseCodes.OK:
s.close()
return True
else:
s.close()
return False
def _check_ccache_install():
"""Check ccache is installed or not. """
CC = os.getenv('CC')
CXX = os.getenv('CXX')
# clang scan-build always fail with ccache.
if CC and os.path.basename(CC) == 'ccc-analyzer' and CXX and os.path.basename(CXX) == 'c++-analyzer':
console.debug('ccache is disabled for scan-build')
return False
try:
p = subprocess.Popen(
['ccache', '-V'],
env=os.environ,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode == 0:
version_line = stdout.splitlines(True)[0]
if version_line and version_line.find('ccache version') != -1:
console.debug('ccache found')
return True
except OSError:
pass
return False
def generate(self):
if self.status is None or self.content_type is None or self.data is None:
console.error('Unable to generate response.')
return -1
else:
# http version
response = self.version + ' '
# response status
response += '{} {}\r\n'.format(self.status.value,
self.status.name.replace('_', ' '))
# response date
response += 'Date: {}\r\n'.format(time.ctime())
# response content type
response += 'Content-Type: {}\r\n'.format(self.content_type.value)
console.debug('Content type: {}'.format(self.content_type.value))
# response content length
# response += 'Content-Length: {}\r\n'.format(sys.getsizeof(self.data))
response += 'Content-Length: {}\r\n'.format(len(self.data))
# MIME Header
# response += 'X-Content-Type-Options: nosniff\r\n'
console.debug('RAW RESPONSE HEADER: ' + response.strip())
response += '\r\n'
response = response.encode()
response += self.data
return response
def parse(self, raw):
utf_raw = raw.decode('utf-8')
console.debug('RAW RESPONSE: ' + utf_raw)
self.version = utf_raw[0:8]
console.debug("VERSION: " + self.version)
s = int(utf_raw[9:12])
self.status = HTTPResponseCodes(s)
def get_local_ip() -> str:
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
try:
s.connect(("10.255.255.255", 1))
ip = s.getsockname()[0]
except Exception as e:
debug(e)
ip = "127.0.0.1"
return ip
def check_sock(ip: str, port: int) -> bool:
with socket.socket() as sock:
try:
sock.bind((ip, port))
except Exception as e:
debug(e)
return False
else:
return True
def generate_fat_jar(target, jars):
"""Generate a fat jar containing the contents of all the jar dependencies. """
target_dir = os.path.dirname(target)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
target_fat_jar = zipfile.ZipFile(target, 'w', zipfile.ZIP_DEFLATED)
# Record paths written in the fat jar to avoid duplicate writing
path_jar_dict = {}
conflict_logs = []
for dep_jar in jars:
jar = zipfile.ZipFile(dep_jar, 'r')
name_list = jar.namelist()
for name in name_list:
if name.endswith('/') or not _is_fat_jar_excluded(name):
if name not in path_jar_dict:
target_fat_jar.writestr(name, jar.read(name))
path_jar_dict[name] = os.path.basename(dep_jar)
else:
if name.endswith('/'):
continue
message = ('%s: duplicated path %s found in {%s, %s}' %
(target, name, path_jar_dict[name],
os.path.basename(dep_jar)))
# Always log all conflicts for diagnosis
if console_logging:
console.debug(message)
if '/.m2/repository/' not in dep_jar:
# There are too many conflicts between maven jars,
# so we have to ignore them, only count source code conflicts
conflict_logs.append(message)
jar.close()
if conflict_logs:
log = '%s: Found %d conflicts when packaging.' % (target,
len(conflict_logs))
if console_logging:
console.warning(log)
else:
print >> sys.stdout, log
print >> sys.stderr, '\n'.join(conflict_logs)
# TODO(wentingli): Create manifest from dependency jars later if needed
contents = [
'Manifest-Version: 1.0',
'Created-By: Python.Zipfile (Blade)',
'Built-By: %s' % os.getenv('USER'),
'Build-Time: %s' % time.asctime(),
]
contents += _manifest_scm(target.split(os.sep)[0])
contents.append('\n')
target_fat_jar.writestr(_JAR_MANIFEST, '\n'.join(contents))
target_fat_jar.close()
def generate_fat_jar(target, jars):
"""Generate a fat jar containing the contents of all the jar dependencies. """
target_dir = os.path.dirname(target)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
target_fat_jar = zipfile.ZipFile(target, 'w', zipfile.ZIP_DEFLATED)
# Record paths written in the fat jar to avoid duplicate writing
path_jar_dict = {}
conflicts = []
for dep_jar in jars:
jar = zipfile.ZipFile(dep_jar, 'r')
name_list = jar.namelist()
for name in name_list:
if name.endswith('/') or not _is_fat_jar_excluded(name):
if name not in path_jar_dict:
target_fat_jar.writestr(name, jar.read(name))
path_jar_dict[name] = dep_jar
else:
if name.endswith('/'):
continue
message = ('%s: duplicate path %s found in {%s, %s}' % (
target, name,
os.path.basename(path_jar_dict[name]),
os.path.basename(dep_jar)))
# Always log all conflicts for diagnosis
console.debug(message)
if '/.m2/repository/' not in dep_jar:
# There are too many conflicts between maven jars,
# so we have to ignore them, only count source code conflicts
conflicts.append('\n'.join([
'Path: %s' % name,
'From: %s' % path_jar_dict[name],
'Ignored: %s' % dep_jar,
]))
jar.close()
if conflicts:
console.warning('%s: Found %d conflicts when packaging.' % (target, len(conflicts)))
generate_fat_jar_metadata(target_fat_jar, jars, conflicts)
contents = [
'Manifest-Version: 1.0',
'Created-By: Python.Zipfile (Blade)',
'Built-By: %s' % os.getenv('USER'),
'Build-Time: %s' % time.asctime(),
]
contents += _manifest_scm(target.split(os.sep)[0])
contents.append('\n')
target_fat_jar.writestr(_JAR_MANIFEST, '\n'.join(contents))
target_fat_jar.close()
def telnet_execute_cli(ip, user, passwd, serial, cli_list, prompt, timeout, retry_times, logdir, logfile, is_debug, is_shell, shellpasswd, wait_time, bootpasswd=[]):
timeout = int(timeout)
wait_time = float(wait_time)
absolute_logfile = logdir + '/' + logfile
debug('logfile path is %s' % absolute_logfile, is_debug)
telnet = telnet_host(ip, user, passwd, serial, absolute_logfile, prompt, wait_time, is_debug)
serial = int(serial)
###set private var
login_timeout = 2
login_retry_times = retry_times
cli_timeout = timeout
cli_retry_times = 2
logout_timeout = 2
logout_retry_times = 10
if serial != 23 and serial:
debug('serial mode login', is_debug)
telnet_host_login = telnet.login_via_serial(login_timeout, login_retry_times)
else:
debug('ip mode login', is_debug)
telnet_host_login = telnet.login_via_ip(login_timeout, login_retry_times)
if telnet_host_login:
cli_sendmode_expect_timeout_wait_list = generate_cli_sendmode_expect_timeout_wait_list(cli_list, prompt, cli_timeout, wait_time, passwd, shellpasswd, bootpasswd)
debug('Cli_sendmode_expect_timeout_wait_list is as below: %s' % str(cli_sendmode_expect_timeout_wait_list), is_debug)
telnet_host_execute = telnet.execute_command_via_cli_sendmode_expect_timeout_wait_list(telnet_host_login, cli_sendmode_expect_timeout_wait_list, cli_retry_times)
else:
print 'Telnet login failed'
return None
###logout
if telnet_host_execute:
telnet_host_logout = telnet.logout(telnet_host_execute, logout_timeout, logout_retry_times)
else:
print 'Telnet execute cli failed'
return None
return telnet_host_logout
def run(self, db, request):
if request.method == HTTPMethods.POST: # POST
if request.data is None:
return Response(code=HTTPResponseCodes.BAD_REQUEST,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
new, _ = CameraStreamRefresh.get_fn()
fn = os.path.basename(new)
console.debug('Newest filename = {}'.format(
os.path.basename(new)))
console.debug('Newest number = {}'.format(fn[:-4]))
num = int(fn[:-4])
n_fn = os.path.join(settings.STREAM_DIR, str(num + 1) + '.jpg')
console.debug('Saving to {}...'.format(n_fn))
if num % 10 == 0:
os.remove(settings.STREAM_DIR + str(num - 10) + '.jpg')
with open(n_fn, 'wb+') as f:
f.write(request.data)
console.debug('Saved file to {}'.format(n_fn))
return Response(code=HTTPResponseCodes.OK,
content_type=HTTPContentTypes.PLAIN,
data='Processed'.encode())
else:
return Response(code=HTTPResponseCodes.METHOD_NOT_ALLOWED,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
def _detect_maven_conflicted_deps(self, scope, dep_jars):
"""
Maven dependencies might have conflict: same group and artifact
but different version. Select higher version by default unless
a specific version of maven dependency is specified as a direct
dependency of the target
"""
# pylint: disable=too-many-locals
dep_jars, conflicted_jars = set(dep_jars), set()
maven_dep_ids = self._get_maven_dep_ids()
maven_jar_dict = {} # (group, artifact) -> (version, set(jar))
maven_repo = '.m2/repository/'
for dep_jar in dep_jars:
if maven_repo not in dep_jar or not os.path.exists(dep_jar):
console.debug('%s: %s not found in local maven repository' %
(self.fullname, dep_jar))
continue
parts = dep_jar[dep_jar.find(maven_repo) +
len(maven_repo):].split('/')
if len(parts) < 4:
continue
name, version, artifact, group = (parts[-1], parts[-2], parts[-3],
'.'.join(parts[:-3]))
key = (group, artifact)
id = ':'.join((group, artifact, version))
if key in maven_jar_dict:
old_version, old_jars = maven_jar_dict[key]
if version == old_version:
# jar name must be different because dep_jars is a set
old_jars.add(dep_jar)
continue
old_id = ':'.join((group, artifact, old_version))
if old_id in maven_dep_ids:
conflicted_jars.add(dep_jar)
elif id in maven_dep_ids or LooseVersion(
version) > LooseVersion(old_version):
conflicted_jars |= old_jars
maven_jar_dict[key] = (version, set([dep_jar]))
else:
conflicted_jars.add(dep_jar)
value = maven_jar_dict[key]
console.debug('%s: Maven dependency version conflict '
'%s:%s:{%s, %s} during %s. Use %s' %
(self.fullname, key[0], key[1], version,
old_version, scope, value[0]))
else:
maven_jar_dict[key] = (version, set([dep_jar]))
dep_jars -= conflicted_jars
return sorted(dep_jars)
def ssh_logout(spawn_child, logout_timeout=2, logout_retry_times=20, is_debug=True):
ssh_logout_result = spawn_child
debug('....................Quit login status....................', is_debug)
ssh_logout_result.sendcontrol('d')
index = ssh_logout_result.expect([pexpect.TIMEOUT, 'Connection to .* closed'], timeout=logout_timeout)
if index == 0:
logout_retry_index = 0
###v13 modify logout retry times from 2 to 5
logout_retry_times = 5
logout_retry_num = 0
while logout_retry_index == 0:
logout_retry_num += 1
debug('%s time retry begin' % logout_retry_num, is_debug)
ssh_logout_result.sendcontrol('d')
logout_retry_index = ssh_logout_result.expect([pexpect.TIMEOUT, 'Connection to .* closed'], timeout=logout_timeout)
debug('%s time retry over' % logout_retry_num, is_debug)
###add retry_num check here, when retry_num = retry_times, return none
if logout_retry_num == logout_retry_times:
print 'Retry %s times and logout still failed, return none' % logout_retry_times
print 'before is %s' % ssh_logout_result.before
print 'after is %s' % ssh_logout_result.after
ssh_logout_result.close(force=True)
return None
elif index == 1:
pass
debug('Free ssh successfully', is_debug)
return ssh_logout_result
def _check_distcc_install():
"""Check distcc is installed or not. """
p = subprocess.Popen('distcc --version',
env={},
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True,
universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode == 0:
version_line = stdout.splitlines(True)[0]
if version_line and version_line.find('distcc') != -1:
console.debug('distcc found')
return True
def generate_fat_jar(target, jars):
"""Generate a fat jar containing the contents of all the jar dependencies. """
target_dir = os.path.dirname(target)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
target_fat_jar = zipfile.ZipFile(target, 'w', zipfile.ZIP_DEFLATED)
# Record paths written in the fat jar to avoid duplicate writing
zip_path_dict = {}
zip_path_conflicts, zip_path_logs = 0, []
for dep_jar in jars:
jar = zipfile.ZipFile(dep_jar, 'r')
name_list = jar.namelist()
for name in name_list:
if name.endswith('/') or not _is_fat_jar_excluded(name):
if name not in zip_path_dict:
target_fat_jar.writestr(name, jar.read(name))
zip_path_dict[name] = os.path.basename(dep_jar)
else:
if not name.endswith('/'): # Not a directory
zip_path_conflicts += 1
zip_path_logs.append(
'%s: duplicate path %s found in {%s, %s}' %
(target, name, zip_path_dict[name],
os.path.basename(dep_jar)))
jar.close()
if zip_path_conflicts:
log = '%s: Found %d conflicts when packaging.' % (target,
zip_path_conflicts)
if console_logging:
console.warning(log)
console.debug('\n'.join(zip_path_logs))
else:
print >> sys.stdout, log
print >> sys.stderr, '\n'.join(zip_path_logs)
# TODO(wentingli): Create manifest from dependency jars later if needed
contents = [
'Manifest-Version: 1.0',
'Created-By: Python.Zipfile (Blade)',
'Built-By: %s' % os.getenv('USER'),
'Build-Time: %s' % time.asctime(),
]
contents += _manifest_scm(target.split(os.sep)[0])
contents.append('\n')
target_fat_jar.writestr(_JAR_MANIFEST, '\n'.join(contents))
target_fat_jar.close()
def telnet_login_via_ip(ip, user, passwd, prompt, login_timeout=2, login_retry_times=20, log_file_path=[], log_file_open=[], is_debug=True):
login_retry_times=int(login_retry_times)
login_timeout=int(login_timeout)
is_user=False
is_passwd=False
is_no=False
is_prompt=False
telnet_login_command='telnet %s' % ip
debug('''Telnet host via IP''', is_debug)
debug('''Telnet login command is "%s"''' % telnet_login_command, is_debug)
debug('............Step1 send command to login target via telnet(IP)............',is_debug)
cli_mode_tuple_list=[(telnet_login_command,'sendline')]
expect_list=[pexpect.TIMEOUT, 'Connection timed out', 'No route to host.*', 'Connected.*Escape character.*User Name:.*', 'Connected.*[Pp]assword:.*', 'Welcome to Aerohive Product.*login:.*']
timeout=login_timeout
###The last one send enter to confirm
retry_cli_mode_tuple_list=[[('','sendnone')]]*(login_retry_times-1)+[['','sendline']]
telnet_login_info=spawn_timeout_retry(cli_mode_tuple_list, expect_list, timeout, retry_cli_mode_tuple_list , login_retry_times, '', is_debug)
telnet_login_result=telnet_login_info[0]
telnet_login_index=telnet_login_info[1]
if log_file_path == './stdout':
telnet_login_result.logfile_read = sys.stdout
else:
telnet_login_result.logfile_read = log_file_open
if telnet_login_index == 0:
print 'Telnet host timeout, please confirm you can reach the host'
return None
elif telnet_login_index == 1:
print 'The mpc connect to the remote target timeout, please confirm'
print 'before is %s, after is %s' % (telnet_login_result.before, telnet_login_result.after)
telnet_login_result.close(force=True)
return None
elif telnet_login_index == 2:
print 'The mpc has no route to the target, please confirm'
print 'before is %s, after is %s' % (telnet_login_result.before, telnet_login_result.after)
telnet_login_result.close(force=True)
return None
elif telnet_login_index == 3 or telnet_login_index == 5:
is_user=True
debug('''From 'telnet command' mode jump to is_user process ''',is_debug)
elif telnet_login_index == 4:
is_passwd=True
debug('''From 'telnet command' mode jump to is_passwd process ''',is_debug)
else:
print 'Not match any expect in step1 expect_list, please check'
print 'before is %s, after is %s' % (telnet_login_result.before, telnet_login_result.after)
telnet_login_result.close(force=True)
return None
telnet_login_result=general_login(telnet_login_result,user,passwd,prompt,login_timeout,login_retry_times,is_debug,is_user,is_passwd,is_no,is_prompt)
return telnet_login_result
def _check_distcc_install():
"""Check distcc is installed or not. """
p = subprocess.Popen(
'distcc --version',
env={},
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True,
universal_newlines=True)
(stdout, stderr) = p.communicate()
if p.returncode == 0:
version_line = stdout.splitlines(True)[0]
if version_line and version_line.find('distcc') != -1:
console.debug('distcc found')
return True
def login(self, email, password):
self.read()
if email in self.users.keys(
) and password == self.users[email]['password']:
# session = os.urandom(32)
session = str(uuid.uuid4())
self.users[email]['session'] = {
'key': session,
'time': time.time()
}
console.debug('User Database (w/sessions): {}'.format(
self.users[email]))
self.write()
return session
else:
return None
def run(self, db, request):
if request.method == HTTPMethods.POST: # POST
if request.data is not None and 'status' in request.data.keys(
) and 'time' in request.data.keys():
console.debug("Garage Status - {}: {}".format(
request.data['time'], request.data['status']))
return Response(code=HTTPResponseCodes.OK,
content_type=HTTPContentTypes.PLAIN,
data='Processed'.encode())
else:
return Response(code=HTTPResponseCodes.BAD_REQUEST,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
return Response(code=HTTPResponseCodes.METHOD_NOT_ALLOWED,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
def __init__(self, blade_root_dir, distcc_host_list=None):
# ccache
self.blade_root_dir = blade_root_dir
self.ccache_installed = self._check_ccache_install()
# distcc
self.distcc_env_prepared = False
self.distcc_installed = self._check_distcc_install()
self.distcc_host_list = distcc_host_list or os.environ.get('DISTCC_HOSTS', '')
if self.distcc_installed and self.distcc_host_list:
self.distcc_env_prepared = True
console.info('distcc is enabled automatically due DISTCC_HOSTS set')
distcc_log_file = os.environ.get('DISTCC_LOG', '')
if distcc_log_file:
console.debug('distcc log: %s' % distcc_log_file)
self.rules_buf = []
def _detect_maven_conflicted_deps(self, scope, dep_jars):
"""
Maven dependencies might have conflict: same group and artifact
but different version. Select higher version by default unless
a specific version of maven dependency is specified as a direct
dependency of the target
"""
dep_jars, conflicted_jars = set(dep_jars), set()
maven_dep_ids = self._get_maven_dep_ids()
maven_jar_dict = {} # (group, artifact) -> (version, set(jar))
maven_repo = '.m2/repository/'
for dep_jar in dep_jars:
if maven_repo not in dep_jar:
continue
parts = dep_jar[dep_jar.find(maven_repo) + len(maven_repo):].split('/')
if len(parts) < 4:
continue
name, version, artifact, group = (parts[-1], parts[-2],
parts[-3], '.'.join(parts[:-3]))
key = (group, artifact)
id = ':'.join((group, artifact, version))
if key in maven_jar_dict:
old_value = maven_jar_dict[key]
if version == old_value[0]:
# jar must be different because dep_jars is a set
old_value[1].add(dep_jar)
continue
old_id = ':'.join((group, artifact, old_value[0]))
if old_id in maven_dep_ids:
conflicted_jars.add(dep_jar)
elif id in maven_dep_ids or LooseVersion(version) > LooseVersion(old_value[0]):
conflicted_jars |= old_value[1]
maven_jar_dict[key] = (version, set([dep_jar]))
else:
conflicted_jars.add(dep_jar)
value = maven_jar_dict[key]
console.debug('%s: Maven dependency version conflict '
'%s:%s:{%s, %s} during %s. Use %s' % (
self.fullname, key[0], key[1],
version, old_value[0], scope, value[0]))
else:
maven_jar_dict[key] = (version, set([dep_jar]))
dep_jars -= conflicted_jars
return sorted(list(dep_jars))
def run(self, db, request):
if request.method == HTTPMethods.POST: # POST
if request.data is None or 'epoch' not in request.data.keys() or 'time' not in request.data.keys() \
or 'message' not in request.data.keys():
return Response(code=HTTPResponseCodes.BAD_REQUEST,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
console.debug("Alert - {}: {}".format(request.data['time'],
request.data['message']))
db.alert(request.data['epoch'], request.data['time'],
request.data['message'])
return Response(code=HTTPResponseCodes.OK,
content_type=HTTPContentTypes.PLAIN,
data='Processed'.encode())
else:
return Response(code=HTTPResponseCodes.METHOD_NOT_ALLOWED,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
def __init__(self, blade_root_dir, distcc_host_list=None):
# ccache
self.blade_root_dir = blade_root_dir
self.ccache_installed = self._check_ccache_install()
# distcc
self.distcc_env_prepared = False
self.distcc_installed = self._check_distcc_install()
self.distcc_host_list = distcc_host_list or os.environ.get(
'DISTCC_HOSTS', '')
if self.distcc_installed and self.distcc_host_list:
self.distcc_env_prepared = True
console.info(
'distcc is enabled automatically due DISTCC_HOSTS set')
distcc_log_file = os.environ.get('DISTCC_LOG', '')
if distcc_log_file:
console.debug('distcc log: %s' % distcc_log_file)
self.rules_buf = []
def communicate(self, client, address, message):
# PARSE REQUEST
req = Request(msg=message)
resp = None
console.info(
'Received {} request from {} at {} with {} parameters'.format(
req.method.name, address[0], req.path,
0 if req.params is None else len(req.params.keys())))
# CHECK TO MAKE SURE POST REQUEST WAS RECEIVED
if req.method == HTTPMethods.POST:
# CHECK TO MAKE SURE DATA WAS RECEIVED
if req.data is not None:
# CHECK TO MAKE SURE JSON HAS CORRECT FIELDS
if 'command' in req.data.keys() and 'key' in req.data.keys():
console.debug('Received command: {}'.format(
req.data['command']))
# AUTHENTICATE KEY
if req.data['key'] == settings.MASTER_KEY:
# CALL APPROPRIATE FUNCTION (OPEN/CLOSE)
if req.data['command'] == 0:
garage.open_g()
resp = self.SUCCESS
elif req.data['command'] == 1:
garage.close_g()
resp = self.SUCCESS
else:
resp = self.BAD_REQUEST
else:
resp = self.UNAUTHORIZED
else:
resp = self.BAD_REQUEST
else:
resp = self.BAD_REQUEST
else:
resp = self.METHOD_NOT_ALLOWED
# SEND RESPONSE
client.send(resp.generate())
# CLOSE CLIENT CONNECTION
client.close()
def generate_scm(build_dir):
# TODO(wentingli): Add git scm
p = subprocess.Popen('svn info', shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if p.returncode != 0:
console.debug('Failed to generate scm: %s' % stderr)
return
revision = url = 'unknown'
for line in stdout.splitlines():
if line.startswith('URL: '):
url = line.strip().split()[-1]
if line.startswith('Revision: '):
revision = line.strip().split()[-1]
break
path = os.path.join(build_dir, 'scm.json')
with open(path, 'w') as f:
json.dump({
'revision' : revision,
'url' : url,
}, f)
def run(self, db, request):
if request.method == HTTPMethods.GET: # GET
if request.params is not None and 'email' in request.params.keys() and 'session' in request.params.keys() and 'epoch' in request.params.keys():
console.debug('Refresh from {} : {}, {}'.format(request.params['email'], request.params['session'], request.params['epoch']))
alerts = db.get_alerts(int(request.params['epoch']))
data = dict()
if len(alerts.keys()) == 0:
data['status'] = 1
else:
data['alerts'] = alerts
data['status'] = 0
data = json.dumps(data).encode()
# TODO add another field that has the epoch time to be used for comparisons in the JS code - whats the
# latest notification received?
return Response(code=HTTPResponseCodes.OK, content_type=HTTPContentTypes.JSON, data=data)
else:
return Response(code=HTTPResponseCodes.BAD_REQUEST, content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
return Response(code=HTTPResponseCodes.METHOD_NOT_ALLOWED, content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
def run(self, db, request):
if request.method == HTTPMethods.POST: # POST
if request.data is None or 'email' not in request.data.keys(
) or 'password' not in request.data.keys():
return Response(code=HTTPResponseCodes.BAD_REQUEST,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
console.debug("Login Attempt - {}: {}".format(
request.data['email'], request.data['password']))
session = db.login(request.data['email'],
request.data['password'])
if session is not None:
console.debug("Login Successful - {}: {}".format(
request.data['email'], session))
return Response(code=HTTPResponseCodes.OK,
content_type=HTTPContentTypes.JSON,
data=json.dumps({
'status': 0,
'session': session
}).encode())
else:
console.debug("Login Unsuccessful - {}".format(
request.data['email']))
return Response(code=HTTPResponseCodes.OK,
content_type=HTTPContentTypes.JSON,
data=json.dumps({
'status': 1,
'session': 0
}).encode())
else:
return Response(code=HTTPResponseCodes.METHOD_NOT_ALLOWED,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
def main():
args = parse.parse_args()
ip = args.desip
user = args.user
passwd = args.passwd
###v4 transfer \$ to \\$
prompt = args.prompt
prompt = re.sub('\$', '\\$', prompt)
timeout = args.timeout
logdir = args.logdir
###v17
logfile = args.logfile.strip()
cli_list = args.cli_list
is_debug = args.is_debug
shellpasswd = args.shellpasswd
is_shell = args.is_shell
serial = args.serial
config_file_path = args.configfilepath
###v10
wait_time = args.wait_time
retry_times = args.retry_times
bootpasswd = args.bootpasswd
input_para_list = sys.argv
debug('''Type command is "python %s"''' % (' '.join(input_para_list)), is_debug)
execute_cli_list = generate_cli_list(cli_list, config_file_path, input_para_list, is_debug)
debug('execute_cli_list is as below', is_debug)
debug(execute_cli_list, is_debug)
try:
telnet_result = telnet_execute_cli(ip, user, passwd, serial, execute_cli_list, prompt, timeout, retry_times, logdir, logfile, is_debug, is_shell, shellpasswd, wait_time, bootpasswd)
except Exception, e:
print str(e)
def main():
args = parse.parse_args()
ip = args.desip
user = args.user
passwd = args.passwd
###v4 transfer \$ to \\$
prompt = args.prompt
prompt = re.sub('\$', '\\$', prompt)
###v13 add support win client
prompt = '%s|(root|logger)@.*~.*\$' % prompt
timeout = args.timeout
logdir = args.logdir
logfile = args.logfile.strip()
cli_list = args.cli_list
is_debug = args.is_debug
shellpasswd = args.shellpasswd
is_shell = args.is_shell
port = args.port
retry_times = args.retry_times
config_file_path = args.configfilepath
wait_time = args.wait_time
bootpasswd = args.bootpasswd
input_para_list = sys.argv
debug('''Type command is "python %s"''' % (' '.join(input_para_list)), is_debug)
execute_cli_list = generate_cli_list(cli_list, config_file_path, input_para_list, is_debug)
debug('execute_cli_list is as below', is_debug)
debug(execute_cli_list, is_debug)
try:
ssh_result = ssh_execute_cli(ip, user, passwd, port, execute_cli_list, prompt, timeout, retry_times, logdir, logfile, is_debug, is_shell, shellpasswd, wait_time, bootpasswd)
except Exception, e:
print str(e)
def _wait_worker_threads(self, threads):
"""Wait for worker threads to complete. """
test_timeout = config.get_item('global_config', 'test_timeout')
try:
while threads:
time.sleep(1) # Check every second
now = time.time()
dead_threads = []
for t in threads:
if t.isAlive():
if test_timeout is not None:
t.check_job_timeout(now)
else:
dead_threads.append(t)
for dt in dead_threads:
threads.remove(dt)
except KeyboardInterrupt:
console.debug('KeyboardInterrupt: Terminate workers...')
for t in threads:
t.terminate()
raise
def _wait_worker_threads(self, threads):
"""Wait for worker threads to complete. """
test_timeout = config.get_item('global_config', 'test_timeout')
try:
while threads:
time.sleep(1) # Check every second
now = time.time()
dead_threads = []
for t in threads:
if t.isAlive():
if test_timeout is not None:
t.check_job_timeout(now)
else:
dead_threads.append(t)
for dt in dead_threads:
threads.remove(dt)
except KeyboardInterrupt:
console.debug('KeyboardInterrupt: Terminate workers...')
for t in threads:
t.terminate()
raise
def __init__(self, blade_root_dir, distcc_hosts_list=None):
# ccache
self.blade_root_dir = blade_root_dir
self.ccache_installed = self._check_ccache_install()
# distcc
self.distcc_env_prepared = False
self.distcc_installed = self._check_distcc_install()
if distcc_hosts_list:
self.distcc_host_list = distcc_hosts_list
else:
self.distcc_host_list = os.environ.get('DISTCC_HOSTS', '')
if self.distcc_installed and self.distcc_host_list:
self.distcc_env_prepared = True
if self.distcc_installed and not self.distcc_host_list:
console.warning('DISTCC_HOSTS not set but you have '
'distcc installed, will just build locally')
self.distcc_log_file = os.environ.get('DISTCC_LOG', '')
if self.distcc_log_file:
console.debug('distcc log: %s' % self.distcc_log_file)
self.rules_buf = []
def run(self, db, request):
if request.method == HTTPMethods.POST: # POST
if 'email' in request.params.keys(
) and 'session' in request.params.keys():
console.debug('Database: {}'.format(db.users))
if db.check_session(request.params['email'],
request.params['session']):
return Response(code=HTTPResponseCodes.OK,
content_type=HTTPContentTypes.PLAIN,
data='Logged out'.encode())
else:
return Response(code=HTTPResponseCodes.UNAUTHORIZED,
content_type=HTTPContentTypes.PLAIN,
data='Authorization Failure'.encode())
else:
return Response(code=HTTPResponseCodes.BAD_REQUEST,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
return Response(code=HTTPResponseCodes.METHOD_NOT_ALLOWED,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
def setup_scons_cache(self, options):
"""Setup scons cache"""
cache_dir = getattr(options, 'cache_dir', os.environ.get('BLADE_CACHE_DIR', '~/.bladescache'))
if not cache_dir:
# '' to disable cache
return
cache_size = getattr(options, 'cache_size', os.environ.get('BLADE_CACHE_SIZE', '2'))
if cache_size == 'unlimited':
cache_size = -1
else:
cache_size = int(cache_size) * 1024 * 1024 * 1024
cache_dir = os.path.expanduser(cache_dir)
self._add_rule('CacheDir("%s")' % cache_dir)
self._add_rule('scache_manager = build_environment.ScacheManager("%s", cache_limit=%d)' % (
cache_dir, cache_size))
self._add_rule('Progress(scache_manager, interval=100)')
console.debug('using cache directory %s' % cache_dir)
console.debug('scache size %d' % cache_size)
def run(self, db, request):
if request.method == HTTPMethods.POST: # POST
if request.data is not None and 'status' in request.data.keys() and 'email' in request.data.keys() \
and 'session' in request.data.keys():
if db.check_session(request.data['email'],
request.data['session']):
console.debug("Garage Status Updating to {}".format(
request.data['status']))
if request.data['status'] == 0:
s = self.open()
elif request.data['status'] == 1:
s = self.close()
else:
return Response(code=HTTPResponseCodes.BAD_REQUEST,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
if s:
return Response(code=HTTPResponseCodes.OK,
content_type=HTTPContentTypes.PLAIN,
data='Success'.encode())
else:
return Response(
code=HTTPResponseCodes.INTERNAL_SERVER_ERROR,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
return Response(code=HTTPResponseCodes.UNAUTHORIZED,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
return Response(code=HTTPResponseCodes.BAD_REQUEST,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
return Response(code=HTTPResponseCodes.METHOD_NOT_ALLOWED,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
def run(self):
console.debug('{0} getting page ...'.format(self.page_num))
flag = self.get_page(self.page_num)
if flag is False:
console.warning('FAILED TO GET PAGE: %04d' % (self.page_num))
else:
console.debug('{0} extracting info ...'.format(self.page_num))
flag = self.extract_information()
if flag is False:
console.warning('FAILED TO EXTRACT INFO: %04d' %
(self.page_num))
else:
console.debug('{0} saving info ...'.format(self.page_num))
# 用锁控制写入同步
const.lock.acquire()
self.save_shop_url_list()
const.lock.release()
console.debug('save done')
def main():
args = parse.parse_args()
ip = args.desip
user = args.user
passwd = args.passwd
# ##v4 transfer \$ to \\$
prompt = args.prompt
prompt = re.sub('\$', '\\$', prompt)
# ##v13 add support win client
prompt = '%s|(root|logger)@.*~.*\$' % prompt
timeout = args.timeout
logdir = args.logdir
logfile = args.logfile.strip()
cli_list = args.cli_list
is_debug = args.is_debug
shellpasswd = args.shellpasswd
is_shell = args.is_shell
port = args.port
retry_times = args.retry_times
config_file_path = args.configfilepath
#v16
wait_time_list = args.wait_time_list
#v16 select the last one as current wait time, set to float mode
wait_time = float(wait_time_list[-1])
bootpasswd = args.bootpasswd
#v17 start
loop = str2list(args.loop)
#v17 end
input_para_list = sys.argv
debug('''Type command is "python %s"''' % (' '.join(input_para_list)), is_debug)
#v17 start
execute_cli_list = []
reg = re.compile('%')
for i in loop:
for cli in cli_list:
#Check how many % in the cli
reg_result = reg.findall(cli)
if len(reg_result) == 0:
execute_cli_list.append(cli)
else:
i_num = len(reg_result)
i_tuple = tuple([i] * i_num)
execute_cli_list.append(cli % i_tuple)
#v17 end
execute_cli_list = generate_cli_list(execute_cli_list, config_file_path, input_para_list, is_debug)
debug('execute_cli_list is as below', is_debug)
debug(execute_cli_list, is_debug)
try:
ssh_result = ssh_execute_cli(ip, user, passwd, port, execute_cli_list, prompt, timeout, retry_times, logdir, logfile, is_debug, is_shell, shellpasswd, wait_time, bootpasswd)
except Exception, e:
print str(e)
def main():
args = parse.parse_args()
ip = args.desip
user = args.user
passwd = args.passwd
###v4 transfer \$ to \\$
prompt = re.sub('\$', '\\$', args.prompt)
timeout = args.timeout
logdir = args.logdir
###v17
logfile = args.logfile.strip()
cli_list = args.cli_list
is_debug = args.is_debug
shellpasswd = args.shellpasswd
is_shell = args.is_shell
#v19 start
port_list = str2list(args.port_list)
#v19 end
config_file_path = args.configfilepath
###v10
wait_time = args.wait_time
retry_times = args.retry_times
bootpasswd = args.bootpasswd
#v19 start
loop = str2list(args.loop)
#v19 end
input_para_list = sys.argv
debug('''Type command is "python %s"''' % (' '.join(input_para_list)), is_debug)
#v19 start
execute_cli_list = []
reg = re.compile('%')
for i in loop:
for cli in cli_list:
#Check how many % in the cli
reg_result = reg.findall(cli)
if len(reg_result) == 0:
execute_cli_list.append(cli)
else:
i_num = len(reg_result)
i_tuple = tuple([i] * i_num)
execute_cli_list.append(cli % i_tuple)
#v19 end
execute_cli_list = generate_cli_list(execute_cli_list, config_file_path, input_para_list, is_debug)
debug('execute_cli_list is as below', is_debug)
debug(execute_cli_list, is_debug)
try:
for port in port_list:
telnet_result = telnet_execute_cli(ip, user, passwd, port, execute_cli_list, prompt, timeout, retry_times, logdir, logfile, is_debug, is_shell, shellpasswd, wait_time, bootpasswd)
except Exception, e:
print str(e)
def telnet_logout(spawn_child, logout_timeout=2, logout_retry_times=20, is_debug=True):
telnet_logout_result = spawn_child
debug('....................Quit login status....................', is_debug)
cli_mode_tuple_list = [(']', 'sendcontrol')]
expect_list = [pexpect.TIMEOUT, 'telnet>.*']
retry_cli_mode_tuple_list = [[('', 'sendnone')]] + [[('', 'sendline')]] + [[(']', 'sendcontrol')]] + [[('', 'sendnone')]] * (logout_retry_times - 3)
telnet_logout_info = spawn_timeout_retry(cli_mode_tuple_list, expect_list, logout_timeout, retry_cli_mode_tuple_list , logout_retry_times, telnet_logout_result, is_debug)
telnet_logout_result = telnet_logout_info[0]
telnet_logout_index = telnet_logout_info[1]
if telnet_logout_index == 0:
print '''TimeOut when send ctrl+] to to logout telnet prompt status'''
print 'before is %s, after is %s' % (telnet_logout_result.before, telnet_logout_result.after)
return None
elif telnet_logout_index == 1:
debug('''Meet telnet> status, should send q now''', is_debug)
cli_mode_tuple_list = [('q', 'sendline')]
expect_list = [pexpect.TIMEOUT, 'Connection closed.*']
retry_cli_mode_tuple_list = [[('', 'sendnone')]] * logout_retry_times
telnet_logout_info = spawn_timeout_retry(cli_mode_tuple_list, expect_list, logout_timeout, retry_cli_mode_tuple_list , logout_retry_times, telnet_logout_result, is_debug)
telnet_logout_result = telnet_logout_info[0]
telnet_logout_index = telnet_logout_info[1]
if telnet_logout_index == 0:
print '''TimeOut when send ctrl+] to to logout telnet prompt status'''
print 'before is %s, after is %s' % (telnet_logout_result.before, telnet_logout_result.after)
return None
elif telnet_logout_index == 1:
debug('Quit telnet successfully', is_debug)
else:
print '''Not match any expect in 'logout' expect_list, please check'''
print 'before is %s, after is %s' % (telnet_logout_result.before, telnet_logout_result.after)
telnet_logout_result.close(force=True)
return None
else:
print '''Not match any expect in 'logout' expect_list, please check'''
print 'before is %s, after is %s' % (telnet_logout_result.before, telnet_logout_result.after)
telnet_logout_result.close(force=True)
return None
return telnet_logout_result
def run(self, db, request):
if request.method == HTTPMethods.GET: # GET
if 'email' in request.params.keys(
) and 'session' in request.params.keys():
console.debug('Session Exists: {}'.format(
db.users[request.params['email']]))
if db.check_session(request.params['email'],
request.params['session']):
console.debug('Session is valid.')
return load_static(settings.PROJECT_DIR + HOME_P)
else:
console.debug('Session is invalid.')
return Response(code=HTTPResponseCodes.UNAUTHORIZED,
content_type=HTTPContentTypes.PLAIN,
data='Authorization Failure'.encode())
else:
return Response(code=HTTPResponseCodes.BAD_REQUEST,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
else:
return Response(code=HTTPResponseCodes.METHOD_NOT_ALLOWED,
content_type=HTTPContentTypes.PLAIN,
data='Failure'.encode())
def telnet_login_via_ip(ip, user, passwd, prompt, login_timeout=2, login_retry_times=20, log_file_path=[], log_file_open=[], is_debug=True):
###transfer all para to correct format
login_retry_times = int(login_retry_times)
login_timeout = int(login_timeout)
###define private para
is_user = False
is_passwd = False
is_no = False
is_prompt = False
is_error = False
###use timeout retry func to login
######define timeoute retry func's para
telnet_login_command = 'telnet %s' % ip
debug('''Telnet IP prcoess start, command is "%s"''' % telnet_login_command, is_debug)
cli_mode_tuple_list = [(telnet_login_command, 'sendline')]
expect_list = [pexpect.TIMEOUT, 'Connection timed out', 'No route to host.*', 'Connected.*Escape character.*User Name:.*', 'Connected.*[Pp]assword:.*', 'Welcome to Aerohive Product.*login:.*']
timeout = login_timeout
###The last one send enter to confirm
retry_cli_mode_tuple_list = [[('', 'sendnone')]] * (login_retry_times - 1) + [['', 'sendline']]
telnet_login_info = spawn_timeout_retry(cli_mode_tuple_list, expect_list, timeout, retry_cli_mode_tuple_list , login_retry_times, '', is_debug)
telnet_login_result = telnet_login_info[0]
telnet_login_index = telnet_login_info[1]
if log_file_path == './stdout':
telnet_login_result.logfile_read = sys.stdout
else:
telnet_login_result.logfile_read = log_file_open
if telnet_login_index == 0:
print 'Telnet host timeout, please confirm you can reach the host'
is_error = True
debug('''From 'telnet command' mode jump to is_error''', is_debug)
elif telnet_login_index == 1:
print 'The mpc connect to the remote target timeout, please confirm'
is_error = True
debug('''From 'telnet command' mode jump to is_error''', is_debug)
elif telnet_login_index == 2:
print 'The mpc has no route to the target, please confirm'
is_error = True
debug('''From 'telnet command' mode jump to is_error''', is_debug)
elif telnet_login_index == 3 or telnet_login_index == 5:
is_user = True
debug('''From 'telnet command' mode jump to is_user''', is_debug)
elif telnet_login_index == 4:
is_passwd = True
debug('''From 'telnet command' mode jump to is_passwd''', is_debug)
else:
print 'Not match any expect in step1 expect_list, please check'
is_error = True
debug('''From 'telnet command' mode jump to is_error''', is_debug)
telnet_login_result = general_login(telnet_login_result, user, passwd, prompt, login_timeout, login_retry_times, is_user, is_passwd, is_no, is_prompt, is_error, is_debug)
return telnet_login_result
def generate_compliation_flags(self):
"""Generates compliation flags. """
# pylint: disable=too-many-locals
toolchain_dir = os.environ.get('TOOLCHAIN_DIR', '')
if toolchain_dir and not toolchain_dir.endswith('/'):
toolchain_dir += '/'
cpp = toolchain_dir + os.environ.get('CPP', 'cpp')
cc = toolchain_dir + os.environ.get('CC', 'gcc')
cxx = toolchain_dir + os.environ.get('CXX', 'g++')
ld = toolchain_dir + os.environ.get('LD', 'g++')
console.debug('CPP=%s' % cpp)
console.debug('CC=%s' % cc)
console.debug('CXX=%s' % cxx)
console.debug('LD=%s' % ld)
self.ccflags_manager.set_cc(cc)
# To modify CC, CXX, LD according to the building environment and
# project configuration
build_with_distcc = (self.distcc_enabled
and self.build_environment.distcc_env_prepared)
cc_str = self._append_prefix_to_building_var(
prefix='distcc', building_var=cc, condition=build_with_distcc)
cxx_str = self._append_prefix_to_building_var(
prefix='distcc', building_var=cxx, condition=build_with_distcc)
build_with_ccache = self.build_environment.ccache_installed
cc_str = self._append_prefix_to_building_var(
prefix='ccache', building_var=cc_str, condition=build_with_ccache)
cxx_str = self._append_prefix_to_building_var(
prefix='ccache', building_var=cxx_str, condition=build_with_ccache)
cc_config = config.get_section('cc_config')
cc_env_str = ('CC="%s", CXX="%s", SECURECXX="%s %s"' %
(cc_str, cxx_str, cc_config['securecc'], cxx))
ld_env_str = 'LINK="%s"' % ld
extra_incs = cc_config['extra_incs']
extra_incs_str = ', '.join(['"%s"' % inc for inc in extra_incs])
if not extra_incs_str:
extra_incs_str = '""'
(cppflags_except_warning,
linkflags) = self.ccflags_manager.get_flags_except_warning()
linkflags += cc_config['linkflags']
self._add_rule('top_env.Replace(%s, '
'CPPPATH=[%s, "%s", "%s"], '
'CPPFLAGS=%s, CFLAGS=%s, CXXFLAGS=%s, '
'%s, LINKFLAGS=%s)' %
(cc_env_str, extra_incs_str, self.build_dir,
self.python_inc, cc_config['cppflags'] +
cppflags_except_warning, cc_config['cflags'],
cc_config['cxxflags'], ld_env_str, linkflags))
cc_library_config = config.get_section('cc_library_config')
# By default blade use 'ar rcs' and skip ranlib
# to generate index for static library
arflags = ''.join(cc_library_config['arflags'])
self._add_rule('top_env.Replace(ARFLAGS="%s")' % arflags)
ranlibflags = cc_library_config['ranlibflags']
if ranlibflags:
self._add_rule('top_env.Replace(RANLIBFLAGS="%s")' %
''.join(ranlibflags))
else:
self._add_rule('top_env.Replace(RANLIBCOM="", RANLIBCOMSTR="")')
# The default ASPPFLAGS of scons is same as ASFLAGS,
# this is incorrect for gcc/gas
options = self.options
if options.m:
self._add_rule('top_env.Replace(ASFLAGS=["-g", "--%s"])' %
options.m)
self._add_rule('top_env.Replace(ASPPFLAGS="-Wa,--%s")' % options.m)
self._setup_cache()
if build_with_distcc:
self.build_environment.setup_distcc_env()
for rule in self.build_environment.get_rules():
self._add_rule(rule)
self._setup_envs()
def ssh_login(ip, user, passwd, prompt, port=22, login_timeout=2, login_retry_times=20, log_file_path=[], log_file_open=[], is_debug=True):
###transfer all para to correct format
port = int(port)
login_timeout = int(login_timeout)
login_retry_times = int(login_retry_times)
###define private para
is_user = False
is_passwd = False
is_no = False
is_prompt = False
is_error = False
###start login process
ssh_login_command = 'ssh %s@%s -p %s' % (user, ip, port)
debug('''SSH login process start, the command is "%s"''' % ssh_login_command, is_debug)
###use timeout retry func to login
######define timeoute retry func's para
cli_mode_tuple_list = [(ssh_login_command, 'sendline')]
expect_list = [pexpect.TIMEOUT, 'Connection timed out', 'No route to host.*', 'Are you sure you want to continue connecting .*\?', '[Pp]assword:']
timeout = login_timeout
###v13
retry_cli_mode_tuple_list = [[('', 'sendnone')]] * 5 + [[('', 'sendline')]] * (login_retry_times - 5)
ssh_login_info = spawn_timeout_retry(cli_mode_tuple_list, expect_list, timeout, retry_cli_mode_tuple_list , login_retry_times, '', is_debug)
ssh_login_result = ssh_login_info[0]
ssh_login_index = ssh_login_info[1]
if log_file_path == './stdout':
ssh_login_result.logfile_read = sys.stdout
else:
ssh_login_result.logfile_read = log_file_open
if ssh_login_index == 0:
print 'SSH host timeout, please confirm you can reach the host'
is_error = True
debug('''From 'SSH command' mode jump to is_error''', is_debug)
elif ssh_login_index == 1:
print 'The mpc connect to the remote target timeout, please confirm the target is reachable.'
is_error = True
debug('''From 'SSH command' mode jump to is_error''', is_debug)
elif ssh_login_index == 2:
print 'The mpc has no route to the target, please confirm the route table and interface status'
is_error = True
debug('''From 'SSH command' mode jump to is_error''', is_debug)
elif ssh_login_index == 3:
debug('The target is not in known host list, need send yes to confirm login', is_debug)
cli_mode_tuple_list = [('yes', 'sendline')]
expect_list = [pexpect.TIMEOUT, '[Pp]assword:']
timeout = login_timeout
spwan_child = ssh_login_result
###v13
retry_cli_mode_tuple_list = [[('', 'sendnone')]] * 5 + [[('', 'sendline')]] * (login_retry_times - 5)
ssh_auth_info = spawn_timeout_retry(cli_mode_tuple_list, expect_list, timeout, retry_cli_mode_tuple_list , login_retry_times, spwan_child, is_debug)
ssh_auth_result = ssh_auth_info[0]
ssh_auth_index = ssh_auth_info[1]
if ssh_auth_index == 0:
print 'SSH host send yes to confirm login timeout, please confirm the host is reachable'
is_error = True
debug('''From 'send yes to pass auth' mode jump to is_error''', is_debug)
elif ssh_auth_index == 1:
debug('Add host to known host list successfully, and meet password part', is_debug)
is_passwd = True
debug('''From 'send yes to pass auth' mode jump to is_passwd''', is_debug)
elif ssh_login_index == 4:
is_passwd = True
debug('''From 'SSH command' mode jump to is_passwd''', is_debug)
else:
print 'Not match any expect in step1 expect_list, please check'
is_error = True
debug('''From 'SSH command' mode jump to is_error''', is_debug)
ssh_login_result = general_login(ssh_login_result, user, passwd, prompt, login_timeout, login_retry_times, is_user, is_passwd, is_no, is_prompt, is_error, is_debug)
return ssh_login_result
def telnet_login_via_serial(ip, serial, user, passwd, prompt, login_timeout=2, login_retry_times=20, log_file_path=[], log_file_open=[], is_debug=True):
###transfer all para to correct format
login_retry_times = int(login_retry_times)
login_timeout = int(login_timeout)
###define private para
is_user = False
is_passwd = False
is_no = False
is_prompt = False
is_error = False
###use timeout retry func to login
######define timeoute retry func's para
telnet_login_command = 'telnet %s %s' % (ip, serial)
debug('''Telnet serial process start, command is "%s"''' % telnet_login_command, is_debug)
cli_mode_tuple_list = [(telnet_login_command, 'sendline')]
expect_list = [pexpect.TIMEOUT, 'No route to host.*', 'Unable .* Connection refused.*', 'Escape character is.*']
timeout = login_timeout
###The last one send enter to confirm
retry_cli_mode_tuple_list = [[('', 'sendnone')]] * (login_retry_times - 1) + [['', 'sendline']]
telnet_login_info = spawn_timeout_retry(cli_mode_tuple_list, expect_list, timeout, retry_cli_mode_tuple_list , login_retry_times, '', is_debug)
telnet_login_result = telnet_login_info[0]
telnet_login_index = telnet_login_info[1]
if log_file_path == './stdout':
telnet_login_result.logfile_read = sys.stdout
else:
telnet_login_result.logfile_read = log_file_open
if telnet_login_index == 0:
print 'Telnet host timeout, please confirm you can reach the host'
is_error = True
debug('''From 'telnet command' mode jump to is_error''', is_debug)
elif telnet_login_index == 1:
print 'The mpc has no route to the target, please confirm'
is_error = True
debug('''From 'telnet command' mode jump to is_error''', is_debug)
return None
elif telnet_login_index == 2:
debug('''The target serial is in using or not alive, please confirm''', is_debug)
is_error = True
debug('''From 'telnet command' mode jump to is_error''', is_debug)
return None
elif telnet_login_index == 3:
debug('''Meet 'Escape' status, send enter to confirm login''', is_debug)
cli_mode_tuple_list = [('', 'sendline')]
###0 If the cisco serial server's port connect nothing, would stay'Escape character is '^]'.' when you send 'enter', cannot diff it from the normal way ,use timeout to mark it
###1 May meet aerohive pruduct powerdown on vmwarw ---EOF, telnet command is EOF already
###2 Aerohive product already login---#
###3 Aerohive product already login, but is the first time to login after reset---'Use the Aerohive.*<yes|no>:'
###4 Aerohive product login normally
###5 login switch via serial(meet password)
expect_list = [pexpect.TIMEOUT, pexpect.EOF, 'login.*', '[Pp]assword.*', 'yes\|no>:.*', prompt]
timeout = login_timeout
###The last one send enter to confirm
retry_cli_mode_tuple_list = [[('', 'sendnone')]] * (login_retry_times - 1) + [['', 'sendline']]
telnet_login_info = spawn_timeout_retry(cli_mode_tuple_list, expect_list, timeout, retry_cli_mode_tuple_list , login_retry_times, telnet_login_result, is_debug)
telnet_login_result = telnet_login_info[0]
telnet_login_index = telnet_login_info[1]
if telnet_login_index == 0:
print '''Send enter to confirm telnet serial timeout, please confirm the serial is alive'''
is_error = True
debug('''From 'send enter' mode jump to is_error''', is_debug)
elif telnet_login_index == 1:
print '''Telnet vmware simulate OS timeout, please confirm the simulate OS is alive'''
is_error = True
debug('''From 'send enter' mode jump to is_error''', is_debug)
elif telnet_login_index == 2:
is_user = True
debug('''From 'send enter confirm login' jump to is_user''', is_debug)
elif telnet_login_index == 3:
is_passwd = True
debug('''From 'send enter confirm login' jump to is_passwd''', is_debug)
elif telnet_login_index == 4:
is_no = True
debug('''From 'send enter confirm login' jump to is_no''', is_debug)
elif telnet_login_index == 5:
is_prompt = True
debug('''From 'send enter confirm login' jump to is_prompt''', is_debug)
else:
print '''Not match any expect values, please check'''
is_error = True
debug('''From 'send enter' mode jump to is_error''', is_debug)
else:
print '''Not match any expect in expect_list, please check'''
is_error = True
debug('''From 'telnet command' mode jump to is_error''', is_debug)
telnet_login_result = general_login(telnet_login_result, user, passwd, prompt, login_timeout, login_retry_times, is_user, is_passwd, is_no, is_prompt, is_error, is_debug)
return telnet_login_result
def main():
args = parse.parse_args()
ip = args.desip
user = args.user
passwd = args.passwd
# ##v4 transfer \$ to \\$
prompt = args.prompt
prompt = re.sub('\$', '\\$', prompt)
# ##v13 add support win client
prompt = '%s|(root|logger)@.*~.*\$' % prompt
timeout = args.timeout
logdir = args.logdir
logfile = args.logfile.strip()
cli_list = args.cli_list
is_debug = args.is_debug
shellpasswd = args.shellpasswd
is_shell = args.is_shell
port = args.port
retry_times = args.retry_times
config_file_path = args.configfilepath
# v16
wait_time_list = args.wait_time_list
# v16 select the last one as current wait time, set to float mode
wait_time = float(wait_time_list[-1])
bootpasswd = args.bootpasswd
# v18 start
loop_list = args.loop_list
if len(cli_list) > len(loop_list):
debug('''CLI num %s is more than LOOP num %s, add default loop value to complete LoopList''' % (len(cli_list), len(loop_list)), is_debug)
delta_len = len(cli_list) - len(loop_list)
for i in range(delta_len):
# if not the same , add invalid sign to complete the loop list
loop_list.append(':')
elif len(cli_list) < len(loop_list):
print '''CLI num %s is more than LOOP num %s, add default loop value to complete LoopList''' % (len(cli_list), len(loop_list))
return None
# process cli and loop
debug('Execute CLI list is %s' % str(cli_list), is_debug)
debug('Execute LOOP list is %s' % str(loop_list), is_debug)
execute_cli_list = []
sub_re = re.compile(r'%')
for i in range(len(cli_list)):
cli = cli_list[i]
if loop_list[i] == ':':
continue
else:
loop_info = loop_list[i].split(':')
if loop_info[0] == 'f':
loop_num = int(loop_info[1])
debug('The %s CLI enter fixed mode, loop num is %s' % ((i + 1), loop_num))
for loop_i in range(loop_num):
#support abc;123
cli_split_list = cli.split(';')
for cli_split in cli_split_list:
execute_cli_list.append(cli_split)
elif loop_info[0] == 'i':
loop_i_list = str2list(loop_info[1])
debug('The %s CLI enter Increment mode, loop list is %s' % ((i + 1), str(loop_i_list)))
for loop_i in loop_i_list:
cli_split_list = cli.split(';')
for cli_split in cli_split_list:
sub_num = len(sub_re.findall(cli_split))
loop_i_tuple = tuple([loop_i] * sub_num)
execute_cli_list.append(cli_split % loop_i_tuple)
else:
print '''The %d -c parameters '%s' error, not match 'f:...' or 'i:...' ''' % (i + 1, loop_list[i])
return None
# v18 end
input_para_list = sys.argv
debug('''Type command is "python %s"''' % (' '.join(input_para_list)), is_debug)
execute_cli_list = generate_cli_list(execute_cli_list, config_file_path, input_para_list, is_debug)
debug('execute_cli_list is as below', is_debug)
debug(execute_cli_list, is_debug)
try:
ssh_result = ssh_execute_cli(ip, user, passwd, port, execute_cli_list, prompt, timeout, retry_times, logdir, logfile, is_debug, is_shell, shellpasswd, wait_time, bootpasswd)
except Exception, e:
print str(e)
