def query_create(self):
TABLES = {}
result = DB_Helper()
TABLES['tables'] = ("CREATE TABLE `tables`("
" `g_id` int NOT NULL AUTO_INCREMENT,"
" `title` VARCHAR (50),"
"PRIMARY KEY (`g_id`)"
")ENGINE=InnoDB")
TABLES['community'] = (
"CREATE TABLE `community`("
" `c_id` int NOT NULL AUTO_INCREMENT,"
" `community_title` VARCHAR (50),"
" `url` VARCHAR (255),"
" `g_id` int NOT NULL,"
" FOREIGN KEY (`g_id`) REFERENCES tables(`g_id`),"
"PRIMARY KEY (`c_id`)"
")ENGINE=InnoDB")
TABLES['users'] = ("CREATE TABLE `users`("
" `u_id` int NOT NULL AUTO_INCREMENT,"
" `url` VARCHAR (255),"
"PRIMARY KEY (`u_id`)"
")ENGINE=InnoDB")
TABLES['sub_community'] = (
"CREATE TABLE `sub_community`("
" `s_id` int NOT NULL AUTO_INCREMENT,"
" `community` VARCHAR (50),"
" `title` VARCHAR (50),"
" `url` VARCHAR (255),"
" `text` VARCHAR (255),"
" `u_id` int,"
" FOREIGN KEY (`u_id`) REFERENCES users(`u_id`),"
" `c_id` int,"
" FOREIGN KEY (`c_id`) REFERENCES community(`c_id`),"
"PRIMARY KEY (`s_id`)"
")ENGINE=InnoDB")
result.create_table(TABLES)
def entry_camera_data():
""" entry camera module API.
"""
module = 'Inserted sucessfully Entry Camera Module Data'
engine = DB_Helper().db_connect_sql_alchemy()
Session = sessionmaker(bind=engine)
session = Session()
EntryCameraModule().entry_camera(session)
session.close()
return module
class Query:
def __init__(self, data):
self.data = data
self.db_helper = DB_Helper()
def tables(self):
add_tables = ("INSERT INTO tables " "(title) " "VALUES (%s)")
self.insert(add_tables)
def community(self):
add_community = (
"INSERT INTO community "
"(community_title, url, g_id) "
"VALUES ((%s, %s, select last_insert_id() from table)")
# g_id = "INSERT INTO tab_student (name_student, id_teacher_fk) VALUES ('rich man', LAST_INSERT_ID())"
self.insert(add_community)
def user(self):
add_users = ("INSERT INTO users " "(url) " "VALUES (%s)")
self.insert(add_users)
def sub_community(self):
add_sub_community = (
"INSERT INTO sub_community "
"(community, title, url, text,u_id, c_id) "
"VALUES ( %s,%s, %s, %s, select last_insert_id(),"
" from user, select last_insert_id() from community)")
self.insert(add_sub_community)
def insert(self, add_query):
self.db_helper.insert_into_table(add_query, self.data)
def update(self, add_query):
sql1 = "update post set date=%s"
self.db_helper.updating_rows(add_query, self.data)
def get_animal_data():
""" Get Animal DB Data API.
"""
module = 'Retrive Data from the DB'
engine = DB_Helper().db_connect_sql_alchemy()
Session = sessionmaker(bind=engine)
session = Session()
# try :
AnimalInfo().get_info(session)
# except :
# module = 'Not Able To Retrive Data'
session.close()
return module
#print(res)
#print(res.json())
#print(res.json()["data"])
res.close()
s.close()
return res.json()
def test_conn(self, url_token):
res = requests.get("https://www.zhihu.com/people/" + url_token,
headers=self.header)
return res.status_code
getmsg = GetMSG()
cnt = 0
db_helper = DB_Helper()
crawling_user = None
if __name__ == '__main__':
while True:
try:
[uid, url_token, insert_time] = db_helper.find_user_to_crawl()
while uid is not None:
with open("log_%d.txt" % os.getpid(), "a+") as f:
f.write("crawling %s %s %s \n" %
(uid, url_token, insert_time))
crawling_user = [uid, url_token]
print("crawling %s %s" % (uid, url_token))
end = False
data = None
# python3 script to initialize the Entry Camera Module table.
from sqlalchemy import Column, Integer, String, create_engine, Date, Boolean, LargeBinary, FLOAT
from sqlalchemy.ext.declarative import declarative_base
# from sqlalchemy.types import TypeDecorator, CHAR
from sqlalchemy.dialects.postgresql import DOUBLE_PRECISION
from psycopg2.extensions import adapt, register_adapter, AsIs
import rootpath
import sys
path = rootpath.detect()
sys.path.append(path)
from db_helper import DB_Helper
import datetime
engine = DB_Helper().db_connect_sql_alchemy()
print("engine", engine)
Base = declarative_base(bind=engine)
class EntryCameraTable(Base):
""" Declaring store_entry_table table
"""
__tablename__ = 'entry_table2'
__table_args__ = ({"schema": 'public'})
animal_name = Column(String)
animal_image = Column(LargeBinary)
total_time = Column(DOUBLE_PRECISION)
#print(res.text)
return res.json()
def test_conn(self, url_token, i):
item = cookies[i]
self.header['Cookie'] = item[0]
self.header['User-Agent'] = item[1]
res = requests.get("https://www.zhihu.com/people/" + url_token,
headers=self.header)
return res.status_code
ga = GetActivity()
cnt = 0
db_helper = DB_Helper()
crawling_user = None
if __name__ == '__main__':
while True:
try:
[uid, url_token,
insert_time] = db_helper.find_user_to_crawl_activity()
while uid is not None:
with open("logs/log_%d.txt" % os.getpid(), "a+") as f:
f.write("crawling %s %s %s \n" %
(uid, url_token, insert_time))
crawling_user = [uid, url_token]
print("crawling %s %s" % (uid, url_token))
from db_helper import DB_Helper
from export_emercoin import Export_Emercoin
from export_namecoin import Export_Namecoin
import sqlite3
print("Initialising DB")
db_handler = DB_Helper()
db_handler.reset_database()
print("Fetching Emercoin domain history")
export_e = Export_Emercoin()
export_data_e = export_e.download_all()
print("Fetching Namecoin domain history")
export_n = Export_Namecoin()
export_data_n = export_n.download_all()
print("Exported {} Namecoin domain entries".format(len(export_data_n)))
print("Adding entries to the database")
for entry in export_data_e + export_data_n:
domain = entry["name"]
current_ip = entry["current_ip"]
past_ips = entry["historical_ips"]
if current_ip:
try:
db_handler.add_domain_ip_current_relation(domain, current_ip)
except sqlite3.InterfaceError:
def __init__(self, data):
self.data = data
self.db_helper = DB_Helper()