def post(self, *args, **kwargs):
paraments = {}
post_data = self.request.arguments
post_data = {
x: post_data.get(x)[0].decode("utf-8")
for x in post_data.keys()
}
if not post_data:
post_data = self.request.body.decode('utf-8')
post_data = json.loads(post_data)
print(post_data)
paraments['account_id'] = post_data.get("account_id", None)
paraments['user_name'] = post_data.get("user_name", None)
paraments['password'] = post_data.get("password", None)
paraments['email'] = post_data.get("email", None)
if (self.check(paraments) == 0):
self.write(self.error_code)
return
sql_connector = mysqlConnector()
sql = 'insert into user (account_id,user_name,password,email) values (%s,%s,%s,%s);'
values = [
paraments['account_id'], paraments['user_name'],
paraments['password'], paraments['email']
]
res = sql_connector.executeSql(sql, values)
if (res == None):
self.write("OK")
else:
res = str(res)
self.write(res)
def post(self, *args, **kwargs):
paraments = {}
post_data = self.request.arguments
post_data = {
x: post_data.get(x)[0].decode("utf-8")
for x in post_data.keys()
}
if not post_data:
post_data = self.request.body.decode('utf-8')
post_data = json.loads(post_data)
paraments['user_id'] = post_data.get('user_id', '')
paraments['urls'] = post_data.get('urls', [])
paraments['header'] = post_data.get('header', {})
paraments['antiminer'] = post_data.get('antiminer', False)
paraments['miner_param'] = post_data.get('miner_param', {})
paraments['timing'] = post_data.get('timing', '')
if (paraments['timing'] == "instant"):
paraments['start_time'] = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
print(paraments)
get = miner.finalgrab.spider(paraments['user_id'], paraments['urls'],
paraments['header'],
paraments['antiminer'],
paraments['miner_param'],
paraments['timing'],
paraments['start_time'])
print(type(get))
print(json.dumps(get))
s = []
for i in get:
s.append({'url': i, 'data': get[i]})
sql_connector = mysqlConnector()
sql = 'insert into data (user_id,data,start_time,url) values (%s,%s,%s,%s);'
if (paraments['timing'] == 'instant'):
paraments['start_time'] = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
else:
t = post_data.get('start_time', '')
t = t[:-6]
a = t.split("T")
paraments['start_time'] = a[0] + " " + a[1]
flag = 0
print(s)
for i in s:
values = [
paraments['user_id'],
json.dumps(i['data']), paraments['start_time'], i['url']
]
res = sql_connector.executeSql(sql, values)
if (res != None):
flag = 1
err = res
if flag == 0:
self.write("OK")
else:
self.write(str(err))
'''values = [paraments['user_id'], paraments['start_time']]
def test1():
connection = sql.mysqlConnector(host='localhost',
user='******',
password='******',
database='rail_db')
# cursor = connection.setConnection()
#mysqlConnection = connection.__getConnection__()
process = analytics(connection, rail_id="RSK17CS036")
process.generateAnalytics('login')
connection.commitChanges()
def post(self, *args, **kwargs):
paraments = {}
post_data = self.request.arguments
post_data = {
x: post_data.get(x)[0].decode("utf-8")
for x in post_data.keys()
}
if not post_data:
post_data = self.request.body.decode('utf-8')
post_data = json.loads(post_data)
#paraments['account_id']=post_data.get('account_id',None)
'''paraments['time']=post_data.get('time',None)
if(paraments['time']==0):
write0()
if(paraments['time']==1):
write1()
if(paraments['time']==2):
write2()
if(paraments['time']>=3):
miner.grabid.run('a','b')
self.write('1111')
return'''
paraments['url'] = post_data.get('url', None)
paraments['request_method'] = post_data.get('request_method', None)
paraments['antiminer'] = post_data.get('antiminer', None)
if (paraments['antiminer'] == 'True'):
paraments['header'] = post_data.get('header', None)
paraments['miner_method'] = post_data.get('miner_method', None)
if (paraments['miner_method'] == 'content'):
write1()
return
paraments['content'] = post_data.get('content', None)
elif (paraments['miner_method'] == 'style'):
write2()
return
paraments['get_by_id'] = post_data.get('get_by_id', None)
paraments['get_by_table'] = post_data.get('get_by_table', None)
paraments['get_by_pic'] = post_data.get('get_by_pic', None)
paraments['timing'] = post_data.get('timing', None)
if (paraments['timing'] != 'instant'):
paraments['start_time'] = post_data.get('start_time', None)
paraments['interval'] = post_data.get('interval', None)
#if(self.check(paraments)==0):
#self.write(self.error_code)
sql_connector = mysqlConnector()
'''sql='select * from user where account_id = %s and password = %s;'
def setConnection(self, connection=None):
"""
Sets the connection the database
"""
if (connection == None):
logging.info("Creating a new connection to the database")
self.mysqlConnection = sql.mysqlConnector(
option_files=(PATH + ".config/mysql.cnf"))
logging.info("Connection successful")
else:
logging.info(
"This connection in inherited from the calling function/script."
)
self.mysqlConnection = connection
try:
self.mysqlConnection.use(self.database)
except AttributeError:
logging.debug(
"Failed to connect to the database, this is when you are using persistent connections"
)
def test_update():
link = mysqlConnector()
connect = link.openDb('192.168.31.160', 'root', '', 'Spider')
test = GrabSite(connect)
time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
#time = datetime.datetime.now()
items = {}
items['siteDomain'] = 'www.hello.com'
items['siteName'] = 'hello'
items['webPageCount'] = 6
items['totalOutLinkCuont'] = 8
items['siteStatus'] = 'FINI'
items['siteType'] = 'seed'
items['createTime'] = time
items['startGrabTime'] = time
items['endGrabTime'] = time
test.update(items)
print 'update successed'
test.update_count(items)
print 'update count successed'
def main(jsonCnf):
"""
Creates a database with tables, procedures and triggers.
"""
db = sql.mysqlConnector(option_files=PATH + ".config/mysql.cnf")
try:
createDatabase(db, jsonCnf["databases"])
createProcedure(db, jsonCnf["procedures"])
createTrigger(db, jsonCnf["triggers"])
createView(db, jsonCnf["views"])
insertInitialData(db, jsonCnf["initial_data"])
except KeyError:
logging.critical(
"The config file is either corrupted or not configured properly - MISSING KEY FIELDS"
)
except Exception as e:
logging.critical(
"An error raised, could be an ERROR that breaks the program:" +
(str(e)))
pass # Look at what error needs to be given
else:
db.commitChanges()
def post(self, *args, **kwargs):
paraments = {}
post_data = self.request.arguments
post_data = {
x: post_data.get(x)[0].decode("utf-8")
for x in post_data.keys()
}
if not post_data:
post_data = self.request.body.decode('utf-8')
post_data = json.loads(post_data)
paraments['account_id'] = post_data.get('account_id', None)
paraments['password'] = post_data.get('password', None)
if (self.check(paraments) == 0):
self.write(self.error_code)
sql_connector = mysqlConnector()
sql = 'select * from user where account_id = %s and password = %s;'
values = [paraments['account_id'], paraments['password']]
res = sql_connector.query(sql, values)
if (type(res) == tuple and len(res) == 1):
self.write("OK")
else:
res = '输入的账号或密码不存在'
self.write(res)
def post(self, *args, **kwargs):
paraments = {}
post_data = self.request.arguments
post_data = {
x: post_data.get(x)[0].decode("utf-8")
for x in post_data.keys()
}
if not post_data:
post_data = self.request.body.decode('utf-8')
post_data = json.loads(post_data)
paraments['account_id'] = post_data.get("account_id", None)
paraments['new_password'] = post_data.get("new_password", None)
if (self.check(paraments) == 0):
self.write(self.error_code)
sql_connector = mysqlConnector()
sql = 'update user set password = %s where account_id= %s'
values = [paraments['new_password'], paraments['account_id']]
res = sql_connector.executeSql(sql, values)
if (res == None):
self.write("OK")
else:
res = str(res)
self.write(res)
def test_query():
link = mysqlConnector()
connect = link.openDb('127.0.0.1', 'root', 'mysql', 'Freebuf_Secpulse')
test = GrabSite(connect)
time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
#time = datetime.datetime.now()
items = {}
items['siteDomain'] = 'www.hello.com'
items['siteName'] = 'aa'
items['webPageCount'] = 0
items['totalOutLinkCuont'] = 0
items['siteStatus'] = 'NEW'
items['siteType'] = 'seed'
items['createTime'] = time
items['startGrabTime'] = time
items['endGrabTime'] = time
result = test.query_grab_site_by_status(items)
print 'query_grab_site_by_status = '
print result
result = test.query_grab_site_by_domain(items)
print 'query_grab_site_by_status = '
print result
def post(self, *args, **kwargs):
paraments = {}
post_data = self.request.arguments
post_data = {
x: post_data.get(x)[0].decode("utf-8")
for x in post_data.keys()
}
if not post_data:
post_data = self.request.body.decode('utf-8')
post_data = json.loads(post_data)
print(post_data)
paraments['user_id'] = post_data.get("user_id", None)
paraments['start_time'] = post_data.get("start_time", None) + ':00'
paraments['url'] = post_data.get("url", None)
print(paraments['start_time'])
sql_connector = mysqlConnector()
sql = 'delete from data where user_id = %s and start_time between %s and %s and url = %s'
values = [
paraments['user_id'], paraments['start_time'],
post_data.get("start_time", None) + ':59', paraments['url']
]
res = sql_connector.executeSql(sql, values)
print(res)
self.write("OK")
def test_query_by_domain_and_status(siteGbHis):
items = {}
items['siteDomain'] = 'www.baidu.com'
items['grabStatus'] = 'NEW'
print siteGbHis.query_by_domain_and_status(items)
def test_update(siteGbHis):
items = {}
items['siteDomain'] = 'www.baidu.com'
items['url'] = 'http://www.baidu.com'
items['grabStatus'] = 'FINSIH'
items['innerPageCount'] = 1000
items['outPageCount'] = 1000
items['lastUpdateTime'] = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
siteGbHis.update(items)
if __name__ == '__main__':
link = mysqlConnector()
connect = link.openDb('127.0.0.1', 'root', '', 'Freebuf_Secpulse')
siteGbHis = SiteGrabHistory(connect)
#test_insert_one(siteGbHis)
test_query_by_domain_and_status(siteGbHis)
test_update(siteGbHis)
def post(self, *args, **kwargs):
paraments = {}
post_data = self.request.arguments
post_data = {
x: post_data.get(x)[0].decode("utf-8")
for x in post_data.keys()
}
if not post_data:
post_data = self.request.body.decode('utf-8')
post_data = json.loads(post_data)
#print(post_data)
paraments['user_id'] = post_data.get("user_id", None)
paraments['start_time'] = post_data.get("start_time", None)
paraments['end_time'] = post_data.get("start_time", None)
paraments['url'] = post_data.get("url", None)
sql_connector = mysqlConnector()
if paraments['start_time'] is None:
if paraments['url'] == '':
sql = 'select * from data where user_id = %s'
values = [paraments['user_id']]
res = sql_connector.query(sql, values)
else:
sql = 'select * from data where user_id = %s and url = %s'
values = [paraments['user_id'], paraments['url']]
res = sql_connector.query(sql, values)
else:
t = post_data.get('start_time', '')
t = t[:-6]
a = t.split("T")
paraments['start_time'] = a[0] + " " + a[1]
t = post_data.get('end_time', '')
t = t[:-6]
a = t.split("T")
paraments['end_time'] = a[0] + " " + a[1]
if paraments['url'] == '':
#print(1)
sql = 'select * from data where user_id = %s and start_time between %s and %s'
values = [
paraments['user_id'], paraments['start_time'],
paraments['end_time']
]
res = sql_connector.query(sql, values)
else:
sql = 'select * from data where user_id = %s and start_time between %s and %s and url = %s'
values = [
paraments['user_id'], paraments['start_time'],
paraments['end_time'], paraments['url']
]
res = sql_connector.query(sql, values)
#print(res)
reslist = []
for i in res:
js = {}
js['data'] = json.loads(i[1])
js['start_time'] = i[2].strftime('%Y-%m-%d %H:%M')
js['url'] = i[3]
reslist.append(js)
#print(reslist)
fin = {}
fin['data'] = reslist
self.write(fin)