def runCommand(command):
import bashlex
cmd = list(bashlex.split(command))
p = subprocess.run(cmd, capture_output=True) # capture_output=True -> capture stdout&stderr
stdout = p.stdout.decode('utf-8')
stderr = p.stderr.decode('utf-8')
return stdout, stderr, p.returncode
def process_command_output(queue, session):
while True:
#print("Inside Process Command")
if queue.empty():
break
command = queue.get()
print(command)
command = command.replace('\n', '')
execute = list(bashlex.split(command))
if ';' in command:
p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
test = datetime.now()
try:
start_time = datetime.now()
outs = p.communicate(timeout=60)
diff = (datetime.now() - start_time).total_seconds()
if isinstance(outs[0], bytes):
outs = outs[0].decode()
#print(outs)
value = Command(command.replace("'", "''"), len(command),
round(diff, 3),
bytes(outs.replace("'", "''"), 'utf8'))
session.add(value)
except subprocess.TimeoutExpired:
p.kill()
outs = p.communicate()
diff = (datetime.now() - start_time).total_seconds()
if isinstance(outs[0], bytes):
outs = outs[0].decode()
#print(outs)
value = Command(command.replace("'", "''"), len(command), 0,
bytes(outs.replace("'", "''"), 'utf8'))
session.add(value)
except subprocess.CalledProcessError:
pass
else:
output = ""
try:
start_time = datetime.now()
output = subprocess.check_output(
execute, timeout=60) #check for timeout
diff = (datetime.now() - start_time).total_seconds()
if isinstance(output, bytes):
output = output.decode()
#print(output)
value = Command(command.replace("'", "''"), len(command),
round(diff, 3),
bytes(output.replace("'", "''"), 'utf8'))
session.add(value)
except subprocess.TimeoutExpired:
#print(output)
if isinstance(output, bytes):
output = output.decode()
value = Command(command.replace("'", "''"), len(command), 0,
bytes(output.replace("'", "''"), 'utf8'))
session.add(value)
except subprocess.CalledProcessError:
pass
session.commit()
def split_command(self, command):
generic = Generic()
# If bashlex fails for some reason, fallback to shlex
# See https://github.com/idank/bashlex#limitations
try:
return generic.decode_utf8(list(bashlex.split(generic.encode_utf8(command))))
except:
return generic.split_command(command)
def sh2option(filepath, parser, quiet=False):
with open(filepath, 'r') as file:
data = file.read().replace('\n', ' ')
args = [string.lstrip().rstrip() for string in bashlex.split(data)][1:]
args = [string for string in args if string != '']
previous_argv = sys.argv
sys.argv = args
opt = parser.parse(quiet=quiet)
sys.argv = previous_argv
return opt
def split_command(self, command):
generic = Generic()
# If bashlex fails for some reason, fallback to shlex
# See https://github.com/idank/bashlex#limitations
try:
return generic.decode_utf8(
list(bashlex.split(generic.encode_utf8(command))))
except:
return generic.split_command(command)
def split_command(self, command):
"""Split the command using shell-like syntax.
If bashlex fails for some reason, fallback to shlex
See https://github.com/idank/bashlex#limitations
"""
encoded = self.encode_utf8(command)
try:
splitted = list(bashlex.split(encoded))
except Exception:
splitted = shlex.split(encoded)
return self.decode_utf8(splitted)
def parse_shell(data, shell_str):
global gShellResultList
pattern = "https?://[\w/:%#\$&\?\(\)~\.=\+\-]+"
result = {}
if data != None:
result["uuid"] = data["uuid"]
try:
result["src_ip"] = data["source_ip"]
except:
result["source_ip"] = data["src_ip"]
try:
result["destination_port"] = data["destination_port"]
except:
result["destination_port"] = data["dest_port"]
result["timestamp"] = data["timestamp"]
result["command_list"] = []
result["url_list"] = []
result["contents"] = []
result["download_shell"] = []
tmp_cmd_str = ""
cmd_list = []
try:
shell_str = shell_str.replace("`", " ")
shell_str = shell_str.replace("'$", " ")
cmd_list = list(bashlex.split(shell_str))
except:
print("#### list(bashlex.split( shell_str )) #### ")
print(shell_str)
pass
# 侵入で実行されたコマンド一覧取得
for item in cmd_list:
if item == ";":
tmp_cmd_str = tmp_cmd_str.strip()
result["command_list"].append(tmp_cmd_str)
url_list = re.findall(pattern, tmp_cmd_str)
result["url_list"].extend(url_list)
tmp_cmd_str = ""
else:
tmp_cmd_str += item + " "
if tmp_cmd_str != "":
result["command_list"].append(tmp_cmd_str)
url_list = re.findall(pattern, tmp_cmd_str)
result["url_list"].extend(url_list)
# 検体のダウンロード処理
downloader(result)
return result
async def process_command(self, session, process, cmdline, only=True):
ts = time.time()
username = process.get_extra_info('username')
try:
await self._db.log_command(username, session, ts, cmdline, only)
except Exception as e:
self._logger.exception("Command log failed: %s", e)
self._logger.info("ts=%.3f. session=%s, user=%s, command=%s, only=%s",
ts, session.hex, repr(username), repr(cmdline), only)
try:
firstcmd = next(bashlex.split(cmdline))
except StopIteration:
pass
except:
process.stderr.write("bash: syntax error near unexpected token "
"`%s'" % cmdline)
else:
if firstcmd == 'exit':
process.stderr.write("logout\n\n")
raise ExitCommand()
elif firstcmd == 'uname':
process.stdout.write('Linux localhost 4.19.0-0.bpo.2-amd64 #1 '
'SMP Debian 4.19.16-1~bpo9+1 (2019-02-07)'
' x86_64 GNU/Linux\n')
elif firstcmd == 'uptime':
date = datetime.datetime.utcnow().strftime("%H:%M:%S")
process.stdout.write(" %s up 5 days, 2:48, 1 user, "
"load average: 0.00, 0.00, 0.00\n" % date)
elif firstcmd == 'date':
date = datetime.datetime.utcnow().strftime("%a %b %d %H:%M:%S UTC %Y")
process.stdout.write(date + '\n')
elif firstcmd == 'whoami':
process.stdout.write(username + '\n')
else:
message = self._banner.safe_substitute(username=username, cmdline=cmdline)
process.stderr.write(message)
def get_rpm_install_list(self, valid_pkg_list=None):
"""Parse dockerfile and find any RPMs that are being installed
It will automatically do any bash variable replacement during this parse
"""
if self._distgit_repo:
# Already cloned, load from there
with Dir(self._distgit_repo.distgit_dir):
dfp = DockerfileParser('Dockerfile')
else:
# not yet cloned, just download it
dfp = DockerfileParser()
dfp.content = self.fetch_cgit_file("Dockerfile")
def env_replace(envs, val):
# find $VAR and ${VAR} style replacements
for k, v in envs.iteritems():
opts = ['${}'.format(k), '${{{}}}'.format(k)]
for opt in opts:
if opt in val:
val = val.replace(opt, v)
return val
def is_assignment(line):
# if this is an assignment node we need to add
# to env dict for later replacement in commands
try:
parts = bashlex.parse(line)
except:
# bashlex does not get along well with some inline
# conditionals and may emit ParsingError
# if that's the case, it's not an assigment, so move along
return None
for ast in parts:
if ast.kind != 'compound': # ignore multi part commands
for part in ast.parts:
if part.kind == 'assignment':
return part.word.split('=')
return None
envs = dict(dfp.envs)
run_lines = []
for entry in json.loads(dfp.json):
if isinstance(entry, dict) and 'RUN' in entry:
line = entry['RUN']
for line in line.split("&"):
line = line.strip()
if line:
line = env_replace(envs, line)
assign = is_assignment(line)
if assign:
envs[assign[0]] = assign[1]
run_lines.append(line)
rpms = []
for line in run_lines:
split = list(bashlex.split(line))
if 'yum' in split and 'install' in split:
# remove as to not mess with checking below
split.remove('yum')
split.remove('install')
i = 0
rpm_start = 0
while i < len(split):
sub = split[i]
if sub.startswith('-'):
if sub in YUM_NON_FLAGS:
i += 1
continue
else:
rpm_start = i
break # found start of rpm names, exit
i += 1
rpms.extend(split[rpm_start:])
return [str(r) for r in rpms] # strip unicode
def search_for_smells(doc, project_name, node_only, smells_to_check):
r = defaultdict(int)
results = set()
temp_res = []
if Smell.irrelevant_properties.value in smells_to_check:
for k, v in doc.items():
traverse_node(k, v, temp_res, k, r, project_name)
if temp_res:
results.add(Smell.irrelevant_properties.value)
if not node_only:
for phase in PHASES:
if phase in doc and doc[phase]:
commands_in_phase = set()
phase_arr = doc[phase]
if isinstance(phase_arr, str):
phase_arr = [phase_arr]
if not isinstance(phase_arr, Iterable):
continue
for line in phase_arr:
if not isinstance(line, str):
continue
try:
commands = list(bashlex.split(line))
except AttributeError:
print("Bash Parsing Error...", project_name)
continue
# analysis in the script phase
if phase == 'after_script' and Smell.analysis_after_script.value in smells_to_check:
analysis_commands = ['coveralls','codeclimate', 'audit-package','istanbul']
for c in analysis_commands:
if c in line:
print(project_name, line, "analysis in the script phase")
# curling in to interpreter smell
if Smell.curling_to_interpreter.value in smells_to_check:
downloaders = ['wget', 'curl']
redirection = ['|','>']
has_r = False
has_d = False
has_c = False
for r in redirection:
if r in commands:
has_r = True
for d in downloaders:
if d in commands:
has_d = True
for c in commands:
if 'sh' in c:
has_c = True
if has_r and has_d and has_c:
print('\n[DANGER] Smell1: curling to interpreter detected in', project_name,'!')
results.add(Smell.curling_to_interpreter.value)
# bypassing security checks
if Smell.bypassing_security_checks.value in smells_to_check:
smelly_commands = ['ssh_known_hosts', 'StrictHostKeyChecking', 'UserKnownHostsFile=/dev/null']
for sc in smelly_commands:
for c in commands:
if sc in c:
print('\n[DANGER] Smell2: Bypassing security check detected in',
project_name, '!')
print('Command:', c)
results.add(Smell.bypassing_security_checks.value)
# Digging deeper
if Smell.other.value in smells_to_check:
parse_line(line, {}, {}, commands_in_phase)
# not making use of defaults smell
if Smell.not_using_defaults.value in smells_to_check:
if phase == 'script' and len(phase_arr) == 1 and phase_arr[0] == 'npm test':
print(project_name, 'not making use of defaults')
if phase == 'install' and len(phase_arr) == 1 and phase_arr[0] == 'npm install':
print(project_name, 'not making use of defaults')
# commands unrelated to phase smell
if Smell.unrelated_commands.value in smells_to_check:
for line in phase_arr:
line = str(line)
if 'install' in phase:
if 'deploy' in line or ('test' in line and 'ghost_testing' not in line):
print('[WARNING] Unrelated commands in install phase!', project_name)
print(line)
if 'deploy' in phase:
if 'install' in line or 'test' in line:
print('[WARNING] Unrelated commands in deploy phase!', project_name)
print(line)
if 'script' in phase:
if 'deploy' in line or 'install' in line:
print('[WARNING] Unrelated commands in script phase!', project_name)
print(line)
install_dependencies = ['npm install', 'apt-get install', 'jspm install','tsd']
testing = ['npm test', 'mocha', 'jasmine-node', 'karma', 'java -jar selenium']
interpreters = ['node', 'meteor', 'jekyll', 'cordova', 'ionic']
static_analysis = ['codeclimate', 'istanbul', 'codecov', 'coveralls', 'jscover']
if 'install' not in phase:
for s in install_dependencies:
if line.startswith(s):
print('[DANGER] install command out of phase!', project_name)
results.add(Smell.unrelated_commands.value)
if 'deploy' not in phase:
if 'deploy' in line:
print('[DANGER] deploy command out of phase!', project_name)
results.add(Smell.unrelated_commands.value)
if 'script' not in phase:
for t in testing:
if line.startswith(t):
print('[DANGER] script command out of phase!', project_name)
results.add(Smell.unrelated_commands.value)
for i in interpreters:
if line.startswith(i):
print('[DANGER] script command out of phase!', project_name)
results.add(Smell.unrelated_commands.value)
# # count by first word
# line = line.split()[0]
# addone(line, results[phase])
# # count by line
# addone(line, results[phase])
# # count by each word
# line = line.split()
# for word in line:
# addone(word, results[phase])
else:
pass
return results
print("Beggining")
while True:
try:
#Checks if any new sms came through
getDataRequest = requests.get("""http://localhost:5000/get-data""")
newMessages = getDataRequest.json()["messages"]
if (len(newMessages) > 0) or True:
print(newMessages)
pass
for messageId, message in newMessages.items(
): #Adds the new commands and begins running them
#print(message)
commandParts = list(bashlex.split(message['Body']))
process = subprocess.Popen(commandParts,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
processID = secrets.token_urlsafe(16)
processes[processID] = {
"processObject": process,
"userID": message["AccountSid"],
"outputLines": [],
"from": message["From"],
"to": message["To"]
}
#Updates the output lines
keysToDie = []
for processID in processes.keys():