def __init__(self, db_url=environ.get("DB_URL"), new=False, echo=False):
self.engine = create_engine(db_url, echo=echo)
self.session_maker = sessionmaker(bind=self.engine)
self.metadata = MetaData(bind=self.engine)
spider_loader = spiderloader.SpiderLoader.from_settings(settings())
s_names = spider_loader.list()
self.spiders = tuple(spider_loader.load(name) for name in s_names)
# todo consider wrapping sqlalchemy.exc.OperationalError instead of using new parameter
if new:
create_database(self.engine.url)
self.create_tables(Base)
else:
self.metadata.reflect()
from sqlalchemy_utils.functions import create_database, drop_database
from contextlib import contextmanager
from itertools import product
from pytz import timezone
import datetime
from pathlib import Path
import csv
from .orm import CurrencyCode, Rate, Provider, Base
environ['SCRAPY_SETTINGS_MODULE'] = "cardRatesUpdater.settings"
std_date_fmt = settings().get('STD_DATE_FMT')
def strpdate(date, fmt=std_date_fmt):
return datetime.datetime.strptime(date, fmt).date()
class DbClient:
def __init__(self, db_url=environ.get("DB_URL"), new=False, echo=False):
self.engine = create_engine(db_url, echo=echo)
self.session_maker = sessionmaker(bind=self.engine)
self.metadata = MetaData(bind=self.engine)
spider_loader = spiderloader.SpiderLoader.from_settings(settings())
s_names = spider_loader.list()
def __init__(self):
super(GanjiSpider, self).__init__()
self.cnx = sqlite3.connect(settings().get("STORE_DATABASE"))
self.cursor = self.cnx.cursor()
self.start_urls = self._get_urls()
from scrapy.utils.project import get_project_settings as settings
from yt.logging import logging_slow_query
from sqlalchemy import create_engine, insert, update, Column, CHAR, Date, DateTime, Float, ForeignKey, MetaData, String, Table, Text, text
from sqlalchemy.dialects.mysql import INTEGER, MEDIUMINT, SMALLINT, TINYINT
if settings().get("DB_DEBUG"):
logging_slow_query(settings().get('DB_LOGGER'))
engine = create_engine(settings().get("CONNECTION_STRING"),
echo=settings().get("DB_ECHO"))
metadata = MetaData(bind=engine)
connect = engine.connect()
print('\x1b[2j\033c\x1bc')
class Main(object):
def insert(self, data):
try:
insert(self._table).values(data).execute()
except Exception as e:
raise e
def update(self, data={}):
try:
update(self._table).values(**data).execute()
except Exception as e:
raise e
def execute(self, sql):
return connect.execute(text(sql)).fetchone()
def __init__(self):
connection = pymongo.MongoClient(settings().get('MONGODB_SERVER'),
settings().get('MONGODB_PORT'))
db = connection[settings().get('MONGODB_DB')]
self.collection = db[settings().get('MONGODB_COLLECTION')]
def test_settings():
assert isinstance(settings().get("CONNECTION_STRING"), str)
def open_spider(self, spider):
self.cnx = sqlite3.connect(settings().get("STORE_DATABASE"))
self.cursor = self.cnx.cursor()
self.cursor.execute("PRAGMA JOURNAL_MODE=WAL")
self.logger = logging.Logger(__file__)