def __repr__(self):
attrs = self._res.attrs
attrs.update({"name": self._res.name})
for attr in attrs:
L.i(attrs[attr], tag=attr)
print(" ------------------------------------ ")
return ''
def list_cmd():
"""
list all load in redis's cmd
"""
cmd_groups = redis.keys()
for group in cmd_groups:
LogControl.info(group, '\n\t', redis.redis.hgetall(group))
def create_single_template(cmd_str, debug=False, **args_map):
"""
create single-cmd templates
example:
@cmd_str: 'ls some | grep some'
@args_map: { 'dir': 1}
will make a : 'ls {dir} | grep some' str to save to db.
then will save data:
{
cmd : 'ls {dir} | grep some' ,
keys: 'dir'
}
"""
cmd_str = cmd_str.decode('utf-8') if isinstance(cmd_str, bytes) else cmd_str
cmd_args = cmd_str.split()
for arg in args_map:
# if cmd_str.find(arg) == -1:
# LogControl.err("No such args can be replace")
# raise CmdMakeException("No arg %s can be replace in %s" %(arg, cmd_str))
try:
cmd_args[args_map[arg]] = '{%s}' % arg
except IndexError as e:
LogControl.err("No such args can be replace")
raise CmdMakeException("No arg %s can be replace in %s" %(arg, cmd_str))
return False
replaced_cmd = ' '.join(cmd_args)
keys = ' '.join(list(args_map.keys()))
LogControl.info(replaced_cmd, keys, txt_color='white') if debug else ''
return replaced_cmd, keys, '%s.log' % cmd_args[0]
def shell(self, cmd_str, out=False, console=True):
"""
@cmd_str
@out=False
@console=True
@self.options['asyn'] = False
if set True will use popen no log but can see it in "/tmp/xxx.log"
if console set False:
no print
if out set True:
return res
"""
if self.options['asyn']:
rrun(cmd_str, "/tmp", cmd_str.split()[0])
return
sta, res = getstatusoutput(cmd_str)
if sta == 0:
for line in res.split("\n"):
LogControl.ok(line)
else:
LogControl.err(res, cmd_str)
if out:
return res
def form_check(self, id, data):
form = self.forms[id]
i_keys = set(data.keys())
t_keys = set(form.names())
inputs = form.input
t_table = {
i.name: i.value if hasattr(i, 'value') else None
for i in inputs if i.type.lower() != 'submit'
}
if i_keys <= t_keys:
pass
else:
L.err("only supported\n", t_table)
return None
r_ks = []
for k in data:
if data[k] == "*None*":
r_ks.append(k)
t_table.update(data)
need_to_fill = {}
for k in t_table:
if not t_table[k]:
need_to_fill[k] = None
t_table.update(dict_cmd(need_to_fill))
for k in r_ks:
t_table.pop(k)
return t_table
def __init__(self, tag):
super().__init__(tag)
if tag.name == 'div' and 'g' in tag['class']:
self._res = tag
else:
self._res = None
L.err("not google text item")
def next(self):
"""
see next page 's result simplely.
"""
if not self.next_url:
LogControl.fail("no next url found !")
return ''
return self.proxy_to(self.host + self.next_url).json()
def __repr__(self):
if isinstance(self._res, list):
for i, v in enumerate(self._res):
L.i(v, tag=i)
elif isinstance(self._res, dict):
for k in self._res:
v = self._res[k]
L.i(v, tag=k)
return ''
def do_list(self, name):
data_tmp = SOCIAL[name]
cols = list(data_tmp.keys())
res = self.db.search(name, *cols)
for i in res:
v = dict(zip(cols, i))
for k in v:
print("\t", end='')
v[k] = v[k][:90] + '...' if len(v[k]) > 90 else v[k]
LogControl.i(v[k], tag=k, txt_color='yellow')
print( "---------------------")
def do_setting(self, args):
for k in SOCIAL:
LogControl.i(k, txt_color='red')
m = dict_cmd({"which table":None})
for k in SOCIAL[m["which table"]]:
LogControl.i(k, txt_color='red', tag=m["which table"])
res = dict_cmd({
'Title': None,
'Value': None,
})
new_cols = {res['Title']: res['Value']}
self.db.alter(m["which table"], **new_cols)
def display(code, content):
'''
display content
'''
if int(code / 200) == 1:
res = '\n'.join([
i.text for i in self.xpath(
content, "div[%s]" %
Baidu.config['translate_tag'], 'td', '*')
])
LogControl.ok(res)
else:
LogControl.fail("not found this")
def _decode_raw(self):
ok = False
for e in BASE_ENCODING:
try:
self.content = self.raw_response.content.decode(e)
ok = True
break
except UnicodeDecodeError:
L.title(e, " err")
continue
if not ok:
self.content = self.raw_response.content.decode("iso-8859-1")
def upload_history(sh=SHELL, debug=False):
"""
find cmd in sh's history
@sh='zsh' / can choose bash
"""
history_file = J(ENV("HOME"), '.%s_history' % sh)
keys_dict = dict()
with open(history_file, 'rb') as fp:
for line in fp:
# filter some unused cmd
try:
# remove prompt and decode as utf-8
# zsh
if sh == 'zsh':
cmd_str = line.decode('utf8').split(";")[1]
# bash
else:
cmd_str = line.decode('utf8')
args = cmd_str.split()
# count cmd
cmd_title = args[0]
if cmd_title not in keys_dict:
keys_dict.setdefault(cmd_title, 1)
else:
keys_dict[cmd_title] += 1
ID = keys_dict[cmd_title]
if len(cmd_title) > 20:
continue
if cmd_title.find("/") != -1:
continue
except IndexError as e:
LogControl.err(e, '\n', '\t', line)
continue
except UnicodeDecodeError as e:
continue
redis(cmd_title, ID, cmd_str)
LogControl.i(cmd_title, end='\r')
sys.stdout.flush()
redis.redis.save()
def GeneratorApi(module_instance):
# sys.argv.pop[0]
kargs = module_instance.init_args()
doc = kargs.pop('doc')
upcase = set([ i for i in 'QWERTYUIOPASDFGHJKLZXCVBNM'])
# print(doc)
parser = argparse.ArgumentParser(usage=module_instance.module_name, description=doc)
parser.add_argument("-T", "--thread", default=1, type=int, help="add some new data to module's DB")
parser.add_argument("--Editor", default=False, action='store_true', help="edit this module")
parser.add_argument("-ad", "--add-data", default=None, help="add some new data to module's DB")
parser.add_argument("-dd", "--delete-data", default=False, action='store_true', help="delte some data to module's DB")
for key in kargs:
key_set = set([i for i in key])
val = kargs[key]
pk = []
pko = {}
if key.endswith('*'):
pko['nargs'] = '*'
key = key[:-1]
# key area
if (key_set & upcase) == key_set:
pk.append(key.lower())
elif key[0] in key_set:
pk += ['-%s' % key[0].lower() , '--%s' % key]
else:
pk.append(key)
# val area
if isinstance(val, str):
pko['default'] = None
pko['help'] = val
elif isinstance(val, tuple):
pko['default'] = val[0] if isinstance(val[0], bool) else v[1]
pko['action'] = ('store_%s' % True if not pko['default'] else 'store_%s' % False).lower()
pko['help'] = val[1] if isinstance(val[1], str) else v[0]
else:
LogControl.err(key, val)
continue
parser.add_argument(*pk, **pko)
return parser.parse_args()
def main():
cmd_str = ' '.join(sys.argv[1:])
if '--help' in sys.argv:
L.i(DOC, tag="help")
sys.exit(0)
if ',,' in cmd_str:
fuzz, cmd = cmd_str.split(",,")
else:
cmd = cmd_str
fuzz = input_default(colored('Host run:', "blue"), "")
choose_hosts(fuzz)
try:
for h in env.hosts:
env.host_string = h
shell(cmd, h)
except (KeyboardInterrupt, InterruptedError) as e:
L.ok("Bye~")
def tree_dict(d, func, key=None, lfunc=None):
if isinstance(d, dict):
for item in d:
tree_dict(d[item], func, key=item)
elif isinstance(d, list):
for item in d:
if lfunc:
tree_dict(item, lfunc)
else:
tree_dict(item, func)
else:
if key:
if key == 'dst':
LogControl.ok(d,
txt_color='green',
txt_attr=['underline', 'bold'])
else:
func(d, key)
else:
func(d)
def create_multi_templates(debug=False):
"""
create a multi-cmd template.
"""
cmd_group_name = dinput("cmd group's name?\n[q: exit]>", default='q')
if cmd_group_name == 'q':
return False
while 1:
cmd_t = dinput('which cmd?\n[q :exit]>', default='q').split()
if cmd_t[0] == 'q':
break
# print(cmd_t)
vals = list(search_cmd(*cmd_t).values())
if len(vals) == 0:
continue
if isinstance(vals[0], list) :
vals = [i.strip() for i in vals[0]]
if not vals:
continue
for i, v in enumerate(vals):
LogControl.ok(i, v, txt_color='cyan')
cmd_d = vals[int(dinput('choose a ID , (deafult 0)', default=0))]
k_in = 2
while 1:
k_in = dinput('\n%s\nchoose parts to replace ,separate by ' ' [default 2] set :' % colored(cmd_d, attrs=['bold']), default='1')
k = [int(i) for i in k_in.split()]
cmd_args = cmd_d.decode('utf8').split() if isinstance(cmd_d, bytes) else cmd_d.split()
mvals = {}
for kv in k:
cmd_args[kv] = colored(cmd_args[kv], attrs=['underline'])
v = dinput('%s |[-r: reset parts, default: -r] %d=' % (' '.join(cmd_args), kv), default='-r')
if v == '-r':
LogControl.fail("reset")
continue
mvals[v] = kv
st, key, out = create_single_template(cmd_d, debug=debug, **mvals)
save_template_to_sqlite_db(cmd_group_name, st, key, out)
break
def update(self, ys):
for i, y in enumerate(ys):
try:
line = self.ax.get_lines()[i]
tdata = line.get_xdata()
ydata = line.get_ydata()
t = tdata[-1] + self.dt # new x data
tdata.append(t)
ydata.append(y)
self.resize_panel(t, y) # resize the fig
line.set_data(tdata, ydata)
except IndexError:
line = Line2D([0], [0])
self.ax.add_line(line)
except Exception as e:
LogControl.err(e)
return self.ax.get_lines()
def do_search(self, name):
data_tmp = SOCIAL[name]
tmp = {
"set search key": str,
}
tmp = dict_cmd(tmp)
keys = dict.fromkeys(tmp['set search key'].split(',') if tmp['set search key'].find(",") != -1 else tmp['set search key'].split() )
keys = dict_cmd(keys)
cols = input_default("which interesting?\n %s\n>" % ' '.join([colored(i, attrs=['underline']) for i in data_tmp.keys() ]) )
if 'all' in cols:
cols = ' '.join(list(data_tmp.keys()))
cols = cols.split(",") if cols.find(",") != -1 else cols.split()
res = self.db.search(name, *cols, **keys)
for i in res:
v = dict(zip(cols, i))
for k in v:
print("\t", end='')
v[k] = v[k][:90] + '...' if len(v[k]) > 90 else v[k]
LogControl.i(v[k], tag=k, txt_color='yellow')
def post(self, key, **data):
link = None
form = None
method = 'post'
id = None
for i, f in enumerate(self.forms):
if f.action.find(key) != -1 and f.method.lower() == 'post':
link = f.action
form = f
id = i
break
if not link:
L.err("no search form action be found !", txt_color='red')
L.i('type search("xxx", key="[ here]")',
tag="only found 'key' :\n")
for i, v in enumerate(self.forms):
L.i(v.action, tag=i)
return None
data = self.form_check(id, data)
with show_pro('loging', link):
res = to(urljoin(self.url, link),
session=self.session,
data=data,
agent=True,
method=method)
return Analyze(res, session=self.session)
def show_pro(sta, text):
try:
L.save()
L.loc(0, 0, True)
L.i(text,
tag=sta,
end='\r',
tag_color='green',
txt_color='yellow',
txt_attr=['underline'])
yield
finally:
print('\r')
L.load()
def ex(self):
# print(self.options)
if self.options['Editor']:
os.system("vi %s" % J(MODULE_PATH, self.module_name + '.py'))
raise SystemExit("0")
if self.options['thread'] > 1:
self.exe = Exe(self.options['thread'])
LogControl.i('thread: %d' % self.options['thread'], txt_color='blue')
if self.options['add_data']:
self.add_res(self.options['add_data'])
raise SystemExit("0")
if self.options['delete_data']:
self.del_res()
raise SystemExit("0")
self.parser()
try:
if not isinstance(self.payloads, (list, dict, tuple,)):
# LogControl.wrn("check here")
self.run(self.options)
# LogControl.wrn("check here1")
raise SystemExit(0)
for payload in self.payloads:
self.options['payload'] = payload
LogControl.title("load payload: ", colored(payload, attrs=['underline', 'bold']) , txt_color="blue",end="\r")
self.run(self.options)
except (KeyboardInterrupt, InterruptedError):
LogControl.i("~~bye")
raise SystemExit(0)
def _request():
if grep_str.lower().find(url.split(".").pop().lower()) != -1:
return num, []
res = self.proxy_to(url)
if res.status_code / 100 == 2:
encoding = 'utf8' #res.encoding if res.encoding else 'utf8'
LogControl.err(url, res.encoding)
if res.content:
try:
h_content = res.content.decode(encoding)
return num, [
i.attrib.get('href')
for i in HTML(h_content).xpath('//a')
if i.attrib.get('href', '').find(grep_str) != -1
]
except UnicodeError as e:
LogControl.err(e, '\n', res.encoding, encoding, url)
return num, []
except KeyError as e:
LogControl.err(e, url)
return num, []
else:
return num, []
def choose_hosts(fuzz=None):
# print(fuzz)
res = list(get_records(fuzz.strip()))
# print(res)
roles = []
passwd = {}
if res:
for I, i in enumerate(res):
L.i(i[0], tag=I)
for i in par(input_default(colored("exm: 1,2-4 >>", "cyan"), '0')):
if isinstance(i, tuple):
for m in i:
roles.append(res[m][0] + ":" + res[m][1])
passwd[res[m][0] + ":" + res[m][1]] = res[m][2]
else:
roles.append(res[i][0] + ":" + res[i][1])
passwd[res[i][0] + ":" + res[i][1]] = res[i][2]
else:
r = dict_cmd({
"user": None,
"host": None,
"port": None,
"passwd": None,
})
roles = [r['user'] + "@" + r['host'] + ":" + r['port']]
if not roles:
L.i("nil.")
return None
sql.insert("host", ['login', 'port', 'passwd'],
r['user'] + "@" + r['host'], r['port'], r['passwd'])
passwd[roles[0]] = r['passwd']
print(roles)
res = {"t": roles}
env.roledefs.update(res)
env.hosts = res['t']
env.passwords = passwd
def __init__(self,
ssl=True,
asyn=True,
debug=False,
db=False,
database=None,
proxy=False):
self.url_pre = 'https://www.' if ssl else 'https//www.'
self.search_name = self.__class__.__name__.lower()
self.host = self.url_pre + self.search_name + '.com'
self.agent = random_choice(AGS)
self.asyn = None
self.db = None
self.debug = debug
self.proxy = None
self.grep_html_content = dict()
if asyn:
self.asyn = Exe(20)
if proxy:
LogControl.info("loading proxy..", end='')
self.proxy = proxy #Mongo('local').find("setting")[0].get("proxy")
if self.proxy:
LogControl.ok("")
if db:
self.use_db = db
db_path = os.path.join(
os.getenv("HOME"),
'.Search-engine-sqlite3-db.db') if not database else database
self.db_path = db_path
self.db = SqlEngine(database=db_path)
if not self.db.table_list():
self.db.create(self.search_name,
query=str,
content=str,
type='web')
def _link(self, url, encoding='utf8'):
try:
agent = random_choice(AGS)
res = to(url, headers={'user-agent': agent})
if res.status_code != 200:
raise Error404("not reachable", res.status_code)
if self.charset:
return url, self.parser(
res.content.decode(self.charset, 'ignore'))
return url, self.parser(res.content.decode(encoding, 'ignore'))
except HTTPError as e:
LogControl.fail(url, e.code)
return url, None
except Exception as e:
LogControl.wrn(url)
LogControl.err(url, e, txt_color='yellow')
return url, None
else:
LogControl.ok(url)
def show(self, t='c'):
"""
t == 'c':
show in console
t == 'b':
show in browser
"""
if t == 'c':
L.i(self.link,
tag=self.title,
txt_color='yellow',
txt_attr=['underline'])
if self.time:
L.i(self.time, tag='T', txt_attr=['bold'])
if self.img:
L.i(self.img, tag='img')
L.i(self.doc, tag='doc', end='\n' + '-' * (L.SIZE[1] - 10) + '\n')
else:
pass
def search(self, query, type='web', url=None):
"""
supported web , news, video, images
"""
def get_vqd(query):
vqd_url = 'https://duckduckgo.com/?q={query}&t=h_&ia={type}'.format(
query=query, type=type)
LogControl.info(vqd_url) if self.debug else ''
sss = self.proxy_to(vqd_url).content.decode('utf8')
return sss[sss.rfind("vqd"):].split("&").pop(0).split("=").pop()
if url is None:
url_query = '+'.join(query.split())
args = self.search_args
vqd = get_vqd(url_query)
url = self.search_url.format(use_js=self.search_pools[type],
query=quote(query),
options=args,
vqd=vqd) + '&sp=1&yhs=1'
LogControl.info(url) if self.debug else ''
response = self.proxy_to(url, headers={'cookie': 'ak=-1'})
if response.status_code / 100 == 2:
self.last_search_type = type # record successful request's type
json_content = ''
try:
json_content = response.json()
except Exception:
LogControl.err(response.content)
sys.exit(0)
self.result = json_content.get('results')
self.deepanswer = json_content.get('deep_answers')
self.next_url = self.result[-1].get('n')
if self.use_db:
self.db.insert(self.search_name, ['query', 'content', 'type'],
query, json.dumps(json_content), type)
return json_content.get('results')
else:
LogControl.err(response.status_code, 'err')
return ''
def display(code, content):
'''
display content
'''
if int(code / 200) == 1:
try:
tree_dict(content['trans_result'], single_display, 'base')
if detail:
tree_dict(content['dict_result'], single_display,
'dict')
if example:
tree_dict(content['liju_result'], single_display,
'sample')
except KeyError:
LogControl.fail("Only: ", *content.keys())
for k in content:
LogControl.err(k, content[k])
else:
LogControl.fail("not found this")
def shell(cmd, h):
output = run(cmd, quiet=True)
L.i('\n' + output, tag=h)
