def form_check(self, id, data):
form = self.forms[id]
i_keys = set(data.keys())
t_keys = set(form.names())
inputs = form.input
t_table = {
i.name: i.value if hasattr(i, 'value') else None
for i in inputs if i.type.lower() != 'submit'
}
if i_keys <= t_keys:
pass
else:
L.err("only supported\n", t_table)
return None
r_ks = []
for k in data:
if data[k] == "*None*":
r_ks.append(k)
t_table.update(data)
need_to_fill = {}
for k in t_table:
if not t_table[k]:
need_to_fill[k] = None
t_table.update(dict_cmd(need_to_fill))
for k in r_ks:
t_table.pop(k)
return t_table
def execute(cmd, help=False, console=False, **args):
LogControl.ok('\n')
t_dir = dir_gen()
try:
os.makedirs(t_dir)
except Exception as e:
pass
DB_Handler = SqlEngine(database=DB_PATH)
cmds = []
options = set()
for i in DB_Handler.select("templates", 'cmd', 'keys', 'output', group_key=cmd):
cmds.append([i[0], i[2]])
[ options.add(m) for m in i[1].split()]
if help:
LogControl.i("need to set options: ", options)
for cmd, out in cmds:
LogControl.i(cmd, ' -> ', out, txt_color='yellow')
return True
else:
for cmd, out in cmds:
try:
rrun(cmd.format(**args), t_dir, out)
except Exception as e:
LogControl.err(e,cmd)
LogControl.i("need to set options: ", options)
continue
if console:
try:
os.system('sleep 2 && tail -f %s/*' % t_dir)
except KeyboardInterrupt as e:
LogControl.info("~bye")
def post(self, key, **data):
link = None
form = None
method = 'post'
id = None
for i, f in enumerate(self.forms):
if f.action.find(key) != -1 and f.method.lower() == 'post':
link = f.action
form = f
id = i
break
if not link:
L.err("no search form action be found !", txt_color='red')
L.i('type search("xxx", key="[ here]")',
tag="only found 'key' :\n")
for i, v in enumerate(self.forms):
L.i(v.action, tag=i)
return None
data = self.form_check(id, data)
with show_pro('loging', link):
res = to(urljoin(self.url, link),
session=self.session,
data=data,
agent=True,
method=method)
return Analyze(res, session=self.session)
def shell(self, cmd_str, out=False, console=True):
"""
@cmd_str
@out=False
@console=True
@self.options['asyn'] = False
if set True will use popen no log but can see it in "/tmp/xxx.log"
if console set False:
no print
if out set True:
return res
"""
if self.options['asyn']:
rrun(cmd_str, "/tmp", cmd_str.split()[0])
return
sta, res = getstatusoutput(cmd_str)
if sta == 0:
for line in res.split("\n"):
LogControl.ok(line)
else:
LogControl.err(res, cmd_str)
if out:
return res
def delete_mode():
"""
delete some cmds.
"""
while 1:
c = dinput("[l:list all comba-cmds, q:exit, s:search cmd, d xxx :delete xxx comba cmd] \n>", default='q')
if c == 'q':
LogControl.i('~ by')
break
elif c == 'l':
for m,res in search_comba_cmds():
LogControl.i(m, res)
else:
if c[0] == 's':
cc = ''
try:
cc = c.split()[1]
except Exception as e:
LogControl.err(e)
continue
for m,res in search_comba_cmds(cc):
LogControl.i(m, res)
elif c[0] == 'd':
cc = ''
try:
cc = c.split()[1]
except Exception as e:
LogControl.err(e)
continue
delete_template_in_sqlite_db(cc)
LogControl.ok("delete it.")
else:
continue
def create_single_template(cmd_str, debug=False, **args_map):
"""
create single-cmd templates
example:
@cmd_str: 'ls some | grep some'
@args_map: { 'dir': 1}
will make a : 'ls {dir} | grep some' str to save to db.
then will save data:
{
cmd : 'ls {dir} | grep some' ,
keys: 'dir'
}
"""
cmd_str = cmd_str.decode('utf-8') if isinstance(cmd_str, bytes) else cmd_str
cmd_args = cmd_str.split()
for arg in args_map:
# if cmd_str.find(arg) == -1:
# LogControl.err("No such args can be replace")
# raise CmdMakeException("No arg %s can be replace in %s" %(arg, cmd_str))
try:
cmd_args[args_map[arg]] = '{%s}' % arg
except IndexError as e:
LogControl.err("No such args can be replace")
raise CmdMakeException("No arg %s can be replace in %s" %(arg, cmd_str))
return False
replaced_cmd = ' '.join(cmd_args)
keys = ' '.join(list(args_map.keys()))
LogControl.info(replaced_cmd, keys, txt_color='white') if debug else ''
return replaced_cmd, keys, '%s.log' % cmd_args[0]
def __init__(self, tag):
super().__init__(tag)
if tag.name == 'div' and 'g' in tag['class']:
self._res = tag
else:
self._res = None
L.err("not google text item")
def upload_history(sh=SHELL, debug=False):
"""
find cmd in sh's history
@sh='zsh' / can choose bash
"""
history_file = J(ENV("HOME"), '.%s_history' % sh)
keys_dict = dict()
with open(history_file, 'rb') as fp:
for line in fp:
# filter some unused cmd
try:
# remove prompt and decode as utf-8
# zsh
if sh == 'zsh':
cmd_str = line.decode('utf8').split(";")[1]
# bash
else:
cmd_str = line.decode('utf8')
args = cmd_str.split()
# count cmd
cmd_title = args[0]
if cmd_title not in keys_dict:
keys_dict.setdefault(cmd_title, 1)
else:
keys_dict[cmd_title] += 1
ID = keys_dict[cmd_title]
if len(cmd_title) > 20:
continue
if cmd_title.find("/") != -1:
continue
except IndexError as e:
LogControl.err(e, '\n', '\t', line)
continue
except UnicodeDecodeError as e:
continue
redis(cmd_title, ID, cmd_str)
LogControl.i(cmd_title, end='\r')
sys.stdout.flush()
redis.redis.save()
def GeneratorApi(module_instance):
# sys.argv.pop[0]
kargs = module_instance.init_args()
doc = kargs.pop('doc')
upcase = set([ i for i in 'QWERTYUIOPASDFGHJKLZXCVBNM'])
# print(doc)
parser = argparse.ArgumentParser(usage=module_instance.module_name, description=doc)
parser.add_argument("-T", "--thread", default=1, type=int, help="add some new data to module's DB")
parser.add_argument("--Editor", default=False, action='store_true', help="edit this module")
parser.add_argument("-ad", "--add-data", default=None, help="add some new data to module's DB")
parser.add_argument("-dd", "--delete-data", default=False, action='store_true', help="delte some data to module's DB")
for key in kargs:
key_set = set([i for i in key])
val = kargs[key]
pk = []
pko = {}
if key.endswith('*'):
pko['nargs'] = '*'
key = key[:-1]
# key area
if (key_set & upcase) == key_set:
pk.append(key.lower())
elif key[0] in key_set:
pk += ['-%s' % key[0].lower() , '--%s' % key]
else:
pk.append(key)
# val area
if isinstance(val, str):
pko['default'] = None
pko['help'] = val
elif isinstance(val, tuple):
pko['default'] = val[0] if isinstance(val[0], bool) else v[1]
pko['action'] = ('store_%s' % True if not pko['default'] else 'store_%s' % False).lower()
pko['help'] = val[1] if isinstance(val[1], str) else v[0]
else:
LogControl.err(key, val)
continue
parser.add_argument(*pk, **pko)
return parser.parse_args()
def search(self, query, type='web', url=None):
"""
supported web , news, video, images
"""
def get_vqd(query):
vqd_url = 'https://duckduckgo.com/?q={query}&t=h_&ia={type}'.format(
query=query, type=type)
LogControl.info(vqd_url) if self.debug else ''
sss = self.proxy_to(vqd_url).content.decode('utf8')
return sss[sss.rfind("vqd"):].split("&").pop(0).split("=").pop()
if url is None:
url_query = '+'.join(query.split())
args = self.search_args
vqd = get_vqd(url_query)
url = self.search_url.format(use_js=self.search_pools[type],
query=quote(query),
options=args,
vqd=vqd) + '&sp=1&yhs=1'
LogControl.info(url) if self.debug else ''
response = self.proxy_to(url, headers={'cookie': 'ak=-1'})
if response.status_code / 100 == 2:
self.last_search_type = type # record successful request's type
json_content = ''
try:
json_content = response.json()
except Exception:
LogControl.err(response.content)
sys.exit(0)
self.result = json_content.get('results')
self.deepanswer = json_content.get('deep_answers')
self.next_url = self.result[-1].get('n')
if self.use_db:
self.db.insert(self.search_name, ['query', 'content', 'type'],
query, json.dumps(json_content), type)
return json_content.get('results')
else:
LogControl.err(response.status_code, 'err')
return ''
def _link(self, url, encoding='utf8'):
try:
agent = random_choice(AGS)
res = to(url, headers={'user-agent': agent})
if res.status_code != 200:
raise Error404("not reachable", res.status_code)
if self.charset:
return url, self.parser(
res.content.decode(self.charset, 'ignore'))
return url, self.parser(res.content.decode(encoding, 'ignore'))
except HTTPError as e:
LogControl.fail(url, e.code)
return url, None
except Exception as e:
LogControl.wrn(url)
LogControl.err(url, e, txt_color='yellow')
return url, None
else:
LogControl.ok(url)
def display(code, content):
'''
display content
'''
if int(code / 200) == 1:
try:
tree_dict(content['trans_result'], single_display, 'base')
if detail:
tree_dict(content['dict_result'], single_display,
'dict')
if example:
tree_dict(content['liju_result'], single_display,
'sample')
except KeyError:
LogControl.fail("Only: ", *content.keys())
for k in content:
LogControl.err(k, content[k])
else:
LogControl.fail("not found this")
def update(self, ys):
for i, y in enumerate(ys):
try:
line = self.ax.get_lines()[i]
tdata = line.get_xdata()
ydata = line.get_ydata()
t = tdata[-1] + self.dt # new x data
tdata.append(t)
ydata.append(y)
self.resize_panel(t, y) # resize the fig
line.set_data(tdata, ydata)
except IndexError:
line = Line2D([0], [0])
self.ax.add_line(line)
except Exception as e:
LogControl.err(e)
return self.ax.get_lines()
def _request():
if grep_str.lower().find(url.split(".").pop().lower()) != -1:
return num, []
res = self.proxy_to(url)
if res.status_code / 100 == 2:
encoding = 'utf8' #res.encoding if res.encoding else 'utf8'
LogControl.err(url, res.encoding)
if res.content:
try:
h_content = res.content.decode(encoding)
return num, [
i.attrib.get('href')
for i in HTML(h_content).xpath('//a')
if i.attrib.get('href', '').find(grep_str) != -1
]
except UnicodeError as e:
LogControl.err(e, '\n', res.encoding, encoding, url)
return num, []
except KeyError as e:
LogControl.err(e, url)
return num, []
else:
return num, []
def del_res(self):
"""
delete some cmds.
"""
while 1:
c = dinput("[l:list all res, q:exit, s:search cmd, d xxx :delete xxx comba cmd] \n>", default='q')
if c == 'q':
LogControl.i('~ by')
break
elif c == 'l':
for res in self.get_res():
LogControl.i(res[0], res[1:])
else:
if c[0] == 's':
cc = ''
try:
cc = c.split()[1]
except Exception as e:
LogControl.err(e)
continue
for res in self.get_res(grep=cc):
LogControl.i(res[0], res[1:])
elif c[0] == 'd':
cc = ''
try:
cc = c.split()[1]
except Exception as e:
LogControl.err(e)
continue
self.db.delete(self.table, id=cc)
LogControl.ok("delete it.")
else:
continue
def __call__(self, *args, **kargs):
if isinstance(self._res, bs4.element.Tag):
return ShowTags(self._res(*args, **kargs))
else:
L.err("not supported this search function")
return None
def search(self, search_str, key='search'):
link = None
form = None
method = None
id = None
for i, f in enumerate(self.forms):
if not hasattr(f, 'method'):
f.method = 'get' # just for js's search form
if f.action.find(key) != -1:
link = f.action
form = f
id = i
break
# find GET form as search form
for i, f in enumerate(self.forms):
if f.method.lower() == 'get':
link = f.action
form = f
id = i
break
if not link:
L.err("no search form action be found !", txt_color='red')
L.i('type search("xxx", key="[ here]")',
tag="only found 'key' :\n")
for i, v in enumerate(self.forms):
L.i(v.action, tag=i)
return None
method = form.method.lower()
f_table = {}
for name in form.names():
i = form[name]['value']
if not i:
f_table[name] = search_str
else:
f_table[name] = i
if method == 'get':
link += "?%s" % urlencode(f_table)
return Analyze(urljoin(self.url, link),
show_process=self.show_process,
session=self.session,
method=method)
else:
res = to(link, method=method, session=self.session, data=f_table)
if res.headers['Content-Type'].find("json") != -1:
return res.json()
elif res.headers['Content-Type'].find('ml') != -1:
return Analyze(res,
session=self.session,
show_process=self.show_process)
else:
L.err("return res is not json or xml",
res.headers['Content-Type'])
return None
def __init__(self,
url,
show_process=True,
cookie=True,
session=None,
**kargs):
self.session = session
if show_process:
with show_pro("Geting", str(url) + " | " + str(kargs)):
if isinstance(url, requests.models.Response):
self.raw_response = url
else:
if cookie:
if not self.session:
self.session, self.raw_response = to(url,
cookie=True,
**kargs)
else:
_, self.raw_response = to(url,
cookie=True,
session=self.session,
**kargs)
else:
if self.session:
self.raw_response = to(url,
session=self.session,
**kargs)
else:
self.raw_response = to(url, **kargs)
else:
if isinstance(url, requests.models.Response):
self.raw_response = url
else:
if cookie:
self.session, self.raw_response = to(url,
cookie=True,
**kargs)
else:
if self.session:
self.raw_response = to(url,
session=self.session,
**kargs)
else:
self.raw_response = to(url, **kargs)
self.show_process = show_process
self.url = self.raw_response.url
self.encoding = self.raw_response.encoding.lower()
# L.err(self.encoding)
# check if encoding is iso-8859-1
self.encoding = ERR_ENCODING_2_RIGHT.get(self.encoding, self.encoding)
try:
self.content = self.raw_response.content.decode(self.encoding)
except UnicodeDecodeError as e:
L.err(e)
L.title(self.encoding + "error", "try another charset")
self._decode_raw()
self.content = re.sub(r'(\n)+', '\n', self.content)
self.Soup = BeautifulSoup(self.content, "html.parser")
self.Base = copy.copy(self.Soup)