def create_tables():
database.connect()
try:
User.create_table()
except: # OperationalError
pass # database already exists. skip creating tables.
database.close()
def create_db():
global con
con = psycopg2.connect(database='postgres', user='******', password='******')
con.set_isolation_level(0)
database.create_database(con, test_db)
con.close()
con = database.connect(dbname=test_db)
database.create_tables(con)
def __init__(self):
config_parser = ConfigParser()
config_parser.read('config.cfg')
data_set = config_parser.get('system-config', 'data_set')
data_set_abs_path = join(abspath(dirname(__file__)), data_set)
lars_path = config_parser.get('system-config', 'lars')
self.db = database.connect()
from QuadTree import QuadTree
self.quad_tree = QuadTree.import_tree(join(data_set_abs_path, abspath(lars_path)))
class Spider(scrapy.Spider):
print("news parser")
db = database.connect()
name = 'news'
allowed_domains = ['fakty.ua']
page = 1
base_url = "https://fakty.ua"
url = 'https://fakty.ua/news?newswidget-main2_page='
start_urls = ["{}{}".format(url, page)]
def start_requests(self):
yield scrapy.Request(url=self.start_urls[0], callback=self.parse)
def parse(self, response):
print('parse')
news = response.css('div.news-block')
for item in news:
title = item.xpath(".//h2//text()").extract_first()
if title is not None:
text = item.xpath(".//p//text()").extract_first()
link = item.xpath(".//a//@href").extract_first()
link = "{}{}".format(self.base_url, link)
time = item.xpath(
".//span[@class='time']//text()").extract_first()
date = item.xpath(
".//span[@class='g-gate']/text()").extract()[2]
if (date == "сегодня"):
date = datetime.datetime.today()
else:
date = datetime.datetime.strptime(date, '%d.%m.%Y')
hour = int(time.split(":")[0])
minute = int(time.split(":")[1])
date = date.replace(hour=hour, minute=minute)
# print(title)
# print(text)
# print(link)
# print(date)
self.db["news"].insert_one({
"title": title,
"text": text,
"link": link,
"date": date
})
self.page += 1
print("parsing page : ", self.page)
link = self.get_link()
print("link : ", link)
yield response.follow(link, self.parse)
def get_link(self):
return "{}{}".format(self.url, self.page)
def getContent():
content = {}
conn = db.connect()
c = conn.cursor()
for j, i in enumerate(source):
z = c.execute("Select * from content_agg where source='{}' order by rowid desc;".format(i))
content[name[j]] = z.fetchall()
if not content[name[j]]:
conn.close()
return None
conn.close()
return content
def getContentForSource(s):
if s in source:
i = source.index(s)
content = {}
conn = db.connect()
c = conn.cursor()
z = c.execute("Select * from content_agg where source='{}' order by rowid desc;".format(s))
content[name[i]] = z.fetchall()
if not content[name[i]]:
conn.close()
return None
conn.close()
return content
else:
return None
def __init__(self, logger_name, db=True, email=True):
self.logger = logging.getLogger(logger_name)
if db:
self.db_manager = database.connect(logger_name, Constants.DB_HOST,
Constants.DB_USERNAME,
Constants.DB_PASSWORD,
Constants.DB_NAME)
else:
self.db_manager = None
if email:
self.email_manager = EmailDataManager(
logger_name, Constants.EMAIL_HOST, Constants.EMAIL_USERNAME,
Constants.EMAIL_PASSWORD, Constants.EMAIL_SOURCE_MAILBOX,
Constants.EMAIL_DEST_MAILBOX, Constants.EMAIL_ERROR_MAILBOX)
else:
self.email_manager = None
async def start():
if MySQL.is_closed():
MySQL.connect()
# create tables for database
MySQL.create_tables([Dogs])
from database import database
db = database.connect()
def get_all_titles():
return db.news.distinct("title")
def get_dates():
return db.news.aggregate([{
"$match": {
"date": {
"$exists": True
}
}
}, {
"$project": {
"_id": 0,
"date": 1
}
}])
def get_all_news_dates():
return db.news.aggregate([{
"$match": {
"date": {
"$exists": True
}
}
def startup():
if connection.is_closed():
connection.connect()
connection.create_tables([User])
def __init__(self, *args, **keywords):
super().__init__(*args, **keywords)
self.db = database.connect()
def startup():
if connection.is_closed():
connection.connect()
# Antes de comenzar el servidor:
connection.create_tables([User
]) #Crea todas las tablas de nuestro proyecto