def execute(cmd, help=False, console=False, **args):
LogControl.ok('\n')
t_dir = dir_gen()
try:
os.makedirs(t_dir)
except Exception as e:
pass
DB_Handler = SqlEngine(database=DB_PATH)
cmds = []
options = set()
for i in DB_Handler.select("templates", 'cmd', 'keys', 'output', group_key=cmd):
cmds.append([i[0], i[2]])
[ options.add(m) for m in i[1].split()]
if help:
LogControl.i("need to set options: ", options)
for cmd, out in cmds:
LogControl.i(cmd, ' -> ', out, txt_color='yellow')
return True
else:
for cmd, out in cmds:
try:
rrun(cmd.format(**args), t_dir, out)
except Exception as e:
LogControl.err(e,cmd)
LogControl.i("need to set options: ", options)
continue
if console:
try:
os.system('sleep 2 && tail -f %s/*' % t_dir)
except KeyboardInterrupt as e:
LogControl.info("~bye")
def list_cmd():
"""
list all load in redis's cmd
"""
cmd_groups = redis.keys()
for group in cmd_groups:
LogControl.info(group, '\n\t', redis.redis.hgetall(group))
def create_single_template(cmd_str, debug=False, **args_map):
"""
create single-cmd templates
example:
@cmd_str: 'ls some | grep some'
@args_map: { 'dir': 1}
will make a : 'ls {dir} | grep some' str to save to db.
then will save data:
{
cmd : 'ls {dir} | grep some' ,
keys: 'dir'
}
"""
cmd_str = cmd_str.decode('utf-8') if isinstance(cmd_str, bytes) else cmd_str
cmd_args = cmd_str.split()
for arg in args_map:
# if cmd_str.find(arg) == -1:
# LogControl.err("No such args can be replace")
# raise CmdMakeException("No arg %s can be replace in %s" %(arg, cmd_str))
try:
cmd_args[args_map[arg]] = '{%s}' % arg
except IndexError as e:
LogControl.err("No such args can be replace")
raise CmdMakeException("No arg %s can be replace in %s" %(arg, cmd_str))
return False
replaced_cmd = ' '.join(cmd_args)
keys = ' '.join(list(args_map.keys()))
LogControl.info(replaced_cmd, keys, txt_color='white') if debug else ''
return replaced_cmd, keys, '%s.log' % cmd_args[0]
def search(self, query, type='web', url=None):
"""
supported web , news, video, images
"""
def get_vqd(query):
vqd_url = 'https://duckduckgo.com/?q={query}&t=h_&ia={type}'.format(
query=query, type=type)
LogControl.info(vqd_url) if self.debug else ''
sss = self.proxy_to(vqd_url).content.decode('utf8')
return sss[sss.rfind("vqd"):].split("&").pop(0).split("=").pop()
if url is None:
url_query = '+'.join(query.split())
args = self.search_args
vqd = get_vqd(url_query)
url = self.search_url.format(use_js=self.search_pools[type],
query=quote(query),
options=args,
vqd=vqd) + '&sp=1&yhs=1'
LogControl.info(url) if self.debug else ''
response = self.proxy_to(url, headers={'cookie': 'ak=-1'})
if response.status_code / 100 == 2:
self.last_search_type = type # record successful request's type
json_content = ''
try:
json_content = response.json()
except Exception:
LogControl.err(response.content)
sys.exit(0)
self.result = json_content.get('results')
self.deepanswer = json_content.get('deep_answers')
self.next_url = self.result[-1].get('n')
if self.use_db:
self.db.insert(self.search_name, ['query', 'content', 'type'],
query, json.dumps(json_content), type)
return json_content.get('results')
else:
LogControl.err(response.status_code, 'err')
return ''
def __init__(self,
ssl=True,
asyn=True,
debug=False,
db=False,
database=None,
proxy=False):
self.url_pre = 'https://www.' if ssl else 'https//www.'
self.search_name = self.__class__.__name__.lower()
self.host = self.url_pre + self.search_name + '.com'
self.agent = random_choice(AGS)
self.asyn = None
self.db = None
self.debug = debug
self.proxy = None
self.grep_html_content = dict()
if asyn:
self.asyn = Exe(20)
if proxy:
LogControl.info("loading proxy..", end='')
self.proxy = proxy #Mongo('local').find("setting")[0].get("proxy")
if self.proxy:
LogControl.ok("")
if db:
self.use_db = db
db_path = os.path.join(
os.getenv("HOME"),
'.Search-engine-sqlite3-db.db') if not database else database
self.db_path = db_path
self.db = SqlEngine(database=db_path)
if not self.db.table_list():
self.db.create(self.search_name,
query=str,
content=str,
type='web')
def xpath(self, html, *tags, exclude=None):
xhtml = HTML(html)
exclude = '[not(name()={})]'.format(exclude) if exclude else ''
LogControl.info("//" + "//".join(tags) + exclude) if self.debug else ''
for item in xhtml.xpath("//" + "//".join(tags) + exclude):
yield item
def get_vqd(query):
vqd_url = 'https://duckduckgo.com/?q={query}&t=h_&ia={type}'.format(
query=query, type=type)
LogControl.info(vqd_url) if self.debug else ''
sss = self.proxy_to(vqd_url).content.decode('utf8')
return sss[sss.rfind("vqd"):].split("&").pop(0).split("=").pop()
def single_display(val, key=None, level=1):
if key:
LogControl.ok('[' + key + ']', '\t', val)
else:
LogControl.info('\t\t' + ' ' * level, val)
