def recv(url, resp):
global STA
content = resp.content
sql_fi = '%s/res_db/%s.db' % (ROOT, url)
db_init = SqlEngine(database=sql_fi)
c = content.decode('utf8', 'ignore')
o = demjson.decode(c)
# path = ROOT+"/cdn_cache/" + url
# with open(path,'w') as fp:
# json.dump(o, fp)
da = o['freshdata']
if not da:
STA[url] = 're-try'
show("try - again")
return
for k in da:
if not db_init.first('cdn', sid=k):
db_init.insert(
'cdn',
['url', 'avgtime', 'srcip', 'srcname', 'isp', 'name', 'view'],
url,
da[k]['Avg'],
da[k]['SrcIP']['srcip'],
da[k]['SrcIP']['ipfrom'],
da[k]['isp'],
da[k]['name'],
da[k]['view'],
)
STA[url] = False
class ExpDBHandler:
"""
can ues functions:
get(*args, **kargs)
to_db(*values)
can use args:
self.table
self.db
self.columns
"""
def __init__(self, table_name):
self.table = table_name
self.db = SqlEngine(database=DB_PATH)
if not (table_name,) in self.db.table_list():
ex_values = {}
while 1:
name = dinput("[q or None exit] name>", None)
if name in ['q', None]:
break
value = dinput("\r[q or None exit] value>", None)
if value in ['q', None]:
break
if value is "int":
ex_values[name] = int
elif value is 'str':
ex_values[name] = str
elif value is "time":
ex_values[name] = time
elif value in digits:
ex_values[name] = int(value)
else:
ex_values[name] = value
self.db.create(table_name, payload='some payload',**ex_values)
self.columns = tuple([i[0] for i in self.db.check_table(self.table)][2:])
def get(self, *columns, **kargs):
for item in self.db.select(self.table, *columns, **kargs):
yield item
def to_db(self, *value):
try:
self.db.insert(self.table, self.columns, *value)
except Exception as e:
return False
else:
return True
def on_result(self, t, v):
if t == 'new':
self.id = v['taskid']
db = SqlEngine(**self.connect_options)
if db is not None:
db.insert("Task", ['task_id', 'url', 'status'], self.id, 'None', 'not running')
db.close()
elif t == 'data':
if self.verbose:
info(v)
db = SqlEngine(**self.connect_options)
if v['data']:
if db:
upadte_set = {
'url': self.test_url,
'status': 'injectable',
'data': str(v['data']),
'ftime': datetime.datetime.now()
}
db.update("Task", upadte_set, task_id=self.id)
else:
if self.verbose:
ok(v['data'])
self.injectable = True
else:
upadte_set = {
'url': self.test_url,
'status': 'failed',
'ftime': datetime.datetime.now()
}
db.update("Task", upadte_set, task_id=self.id)
db.close()
# when scan stoped , to delete task;
self.task_cmd("delete")
elif t == "status":
try:
if self.verbose:
info(v[u'status'])
if v[u'status'] == 'terminated':
self.terminated = True
self.result()
else:
if not self.terminated:
self.exe.timmer(5, self.status)
except KeyError:
pass
elif t == 'start':
if self.verbose:
info(v['success'])
elif t == 'set':
if self.verbose:
ok('\ninit options')
elif t == 'kill':
if self.verbose:
fail(v)
elif t == 'stop':
if self.verbose:
wrn(v)
elif t == 'list':
for k in v['options']:
if self.verbose:
ok(k, v['options'][k])
elif t == 'task':
if self.verbose:
info(v)
elif t == 'log':
for msg in v['log']:
# if self.log:
info(msg)
class Social(cmd.Cmd):
intro = colored("Social DB", "red")
prompt = colored("<Social>", "red")
def __init__(self):
super().__init__()
self.db = SqlEngine(database=SOCIAL_DB_PATH)
for k in SOCIAL:
if (k,) not in self.db.table_list():
self.db.create(k, **SOCIAL[k])
self.social = SOCIAL
def do_add(self, name):
m = SOCIAL[name]
res = dict_cmd(m)
res = file_check(res)
self.db.insert(name, list(res.keys()), *res.values())
def do_change(self, name):
self.do_search(name)
tm1 = {
'title': str,
'new':str,
'id': str,
}
tm1 = dict_cmd(tm1)
self.db.update(name, {tm1['title']: tm1['new']}, id=int(tm1['id']))
def do_setting(self, args):
for k in SOCIAL:
LogControl.i(k, txt_color='red')
m = dict_cmd({"which table":None})
for k in SOCIAL[m["which table"]]:
LogControl.i(k, txt_color='red', tag=m["which table"])
res = dict_cmd({
'Title': None,
'Value': None,
})
new_cols = {res['Title']: res['Value']}
self.db.alter(m["which table"], **new_cols)
def do_clear(self, arg):
os.system("tput cl")
def do_vi_setting(self,arg):
os.system("vim " + TEMPLATE_PATH)
def do_search(self, name):
data_tmp = SOCIAL[name]
tmp = {
"set search key": str,
}
tmp = dict_cmd(tmp)
keys = dict.fromkeys(tmp['set search key'].split(',') if tmp['set search key'].find(",") != -1 else tmp['set search key'].split() )
keys = dict_cmd(keys)
cols = input_default("which interesting?\n %s\n>" % ' '.join([colored(i, attrs=['underline']) for i in data_tmp.keys() ]) )
if 'all' in cols:
cols = ' '.join(list(data_tmp.keys()))
cols = cols.split(",") if cols.find(",") != -1 else cols.split()
res = self.db.search(name, *cols, **keys)
for i in res:
v = dict(zip(cols, i))
for k in v:
print("\t", end='')
v[k] = v[k][:90] + '...' if len(v[k]) > 90 else v[k]
LogControl.i(v[k], tag=k, txt_color='yellow')
def do_list(self, name):
data_tmp = SOCIAL[name]
cols = list(data_tmp.keys())
res = self.db.search(name, *cols)
for i in res:
v = dict(zip(cols, i))
for k in v:
print("\t", end='')
v[k] = v[k][:90] + '...' if len(v[k]) > 90 else v[k]
LogControl.i(v[k], tag=k, txt_color='yellow')
print( "---------------------")
def export(self, name):
tmp = SOCIAL[name]
keys = list(tmp.keys())
data = self.db.select(name, *keys)
for item in data:
for i,v in enumerate(item):
if v.startswith('[*b64*]'):
v = b642file(v[7:])
r = v.find("]") + 1
v_name = v[1: r-1]
v = v[r:]
str_tmp = "%s: %s" % (keys[i] , v)
yield str_tmp
yield "--------------------------------------------------------"
def do_export(self, name):
data = self.export(name)
output = dict_cmd({'output[path to save]': "./pdf.pdf"})
fpath = output['output[path to save]']
LogControl.title('to',output_to_pdf(data, fpath))
def do_quit(self, some):
return True