def main():
parser = ArgumentParser(description='Scrape WMATA SmarTrip usage data.')
parser.add_argument('username', metavar='USER', help='SmarTrip username')
parser.add_argument('--password', dest='password', metavar='PASSWORD', help='SmarTrip password')
parser.add_argument('--card-id', dest='cardID', type=int, help='card ID to fetch, defaults to first card')
parser.add_argument('--db', dest='outFile', default="smartrip.sqlite", help='SQLite output file')
parser.add_argument('--list', dest='list', default=False,
action='store_true', help='List available SmarTrip cards')
args = vars(parser.parse_args())
if args['password'] is None:
args['password'] = getpass.getpass('Enter SmarTrip password: '******'username'], args['password'])
cards = getCards(response)
if args['list']:
listCards(cards)
else:
engine = create_engine("sqlite:///" + args['outFile'])
Session = sessionmaker(bind=engine)
session = Session()
metadata.bind = engine
metadata.create_all()
if args['cardID'] is not None:
scrapeCard(br, session, str(args['cardID']), cards[str(args['cardID'])]['cardSerial'])
else:
cardID = cards.keys()[0]
scrapeCard(br, session, str(cardID), cards[cardID]['cardSerial'])
def database(self) -> sessionmaker:
"""
constructs the database connection and fills it with data
"""
logger.info('Starting database connection')
dns = self.settings['dns']
if dns.startswith("sqlite"):
engine = create_engine(dns, connect_args={'isolation_level': None})
else:
engine = create_engine(dns)
engine.connect()
metadata.bind = engine
metadata.create_all(engine)
factory = sessionmaker(bind=engine, autoflush=True)
scope = scoped_session(factory)
session = scope()
for char in yaml.load(open('data/characters.yaml')):
logger.debug(char)
char = character.DbCharacter(**char)
session.merge(char)
session.commit()
session.close()
return scope
def main():
parse_command_line()
global engine
engine = sqlalchemy.create_engine(options.db_url, echo=options.debug, convert_unicode=True)
if options.init_db:
metadata.create_all(engine)
with engine.connect() as conn:
conn.execute(users.insert(), [
{'name': 'tester1', 'password': '******'},
{'name': 'tester2', 'password': '******'},
])
return
app = Application(
(('/', RootHandler),
('/whoami', WhoAmIHandler),
('/login', LoginHandler),
('/games', Games),
(r'/games/(\d+)', Game),
(r'/games/(\d+)/moves', Moves),
(r'/games/(\d+)/wait', Wait)),
static_path=os.path.join(os.path.dirname(__file__), 'static'),
debug=options.debug,
cookie_secret='42',
)
app.listen(options.port)
IOLoop.current().start()
def init_db(metadata):
import sqlalchemy
# init table
engine = sqlalchemy.create_engine(
DATABASE_URL, connect_args={"check_same_thread": False})
metadata.create_all(bind=engine)
engine.dispose()
def actionCreateDB(self):
from models import metadata, engine
from alembic.config import Config
from alembic import command
metadata.create_all(engine)
alembic_cfg = Config()
alembic_cfg.set_main_option("script_location", "alembic")
alembic_cfg.set_main_option("url", config.DB_URL)
command.stamp(alembic_cfg, "head")
def get_engine(path, echo=False):
"""Returns a connection to the database.
If the database path does not exist a new blank database
is created using defined metadata.
"""
engine = create_engine('sqlite:///%s' % (path, ), echo=echo)
if not os.path.exists(path):
metadata.create_all(engine)
return engine
def get_engine(path, echo=False):
"""Returns a connection to the database.
If the database path does not exist a new blank database
is created using defined metadata.
"""
engine = create_engine('sqlite:///%s' % (path, ), echo=echo)
if not os.path.exists(path):
metadata.create_all(engine)
return engine
def setUp(self):
engine = create_engine("sqlite:///:memory:")
metadata.bind = engine
metadata.create_all()
self.conn = engine.connect()
self.db_handler = BaseDBHandler(self.conn)
self.uuid1 = str(uuid.uuid4())
test_user = users.insert().values(user_uuid = self.uuid1, username = u"konrad", password = "******", email = "*****@*****.**")
self.conn.execute(test_user)
self.uuid2 = str(uuid.uuid4())
test_user2 = users.insert().values(user_uuid = self.uuid2, username = u"malgosia", password = "******", email = "*****@*****.**")
self.conn.execute(test_user2)
def before(self):
super(TestAnswer, self).before()
self.db = create_engine('sqlite://')
metadata.create_all(self.db)
self.resource = Answer(self.db)
self.api.add_route('/answer/{game_id}/{question_id}', self.resource)
self.api.req_options.auto_parse_form_urlencoded = True
# test data
self.test_run_id = self.getUniqueString('run_id')
self.test_score = self.getUniqueInteger()
self.test_uid = self.getUniqueInteger()
self.test_selected = ','.join([str(self.getUniqueInteger()) for i in range(3)])
def before(self):
super(TestAnswer, self).before()
self.db = create_engine('sqlite://')
metadata.create_all(self.db)
self.resource = Answer(self.db)
self.api.add_route('/answer/{game_id}/{question_id}', self.resource)
self.api.req_options.auto_parse_form_urlencoded = True
# test data
self.test_run_id = self.getUniqueString('run_id')
self.test_score = self.getUniqueInteger()
self.test_uid = self.getUniqueInteger()
self.test_selected = ','.join(
[str(self.getUniqueInteger()) for i in range(3)])
def setUp(self):
engine = create_engine("sqlite:///:memory:")
metadata.bind = engine
metadata.create_all()
self.conn = engine.connect()
self.db_handler = BaseDBHandler(self.conn)
self.uuid1 = str(uuid.uuid4())
test_user = users.insert().values(user_uuid=self.uuid1,
username=u"konrad",
password="******",
email="*****@*****.**")
self.conn.execute(test_user)
self.uuid2 = str(uuid.uuid4())
test_user2 = users.insert().values(user_uuid=self.uuid2,
username=u"malgosia",
password="******",
email="*****@*****.**")
self.conn.execute(test_user2)
def setUp(self):
engine = create_engine("sqlite:///:memory:")
metadata.bind = engine
metadata.create_all()
self.conn = engine.connect()
self.db_handler = BaseDBHandler(self.conn)
self.uuid1 = str(uuid.uuid4())
test_user = users.insert().values(user_uuid = self.uuid1, username = u"konrad", password = "******", email = "*****@*****.**")
self.conn.execute(test_user)
self.uuid2 = str(uuid.uuid4())
test_user2 = users.insert().values(user_uuid = self.uuid2, username = u"malgosia", password = "******", email = "*****@*****.**")
self.conn.execute(test_user2)
self.uuid3 = str(uuid.uuid4())
test_user3 = users.insert().values(user_uuid = self.uuid3, username = u"kuba", password = "******", email = "*****@*****.**")
self.conn.execute(test_user3)
self.product_uuid1 = str(uuid.uuid4())
self.product_uuid2 = str(uuid.uuid4())
self.product_uuid3 = str(uuid.uuid4())
product_1 = products.insert().values(product_uuid = self.product_uuid1, product_name = u"wiertarka", product_desc = u"test" )
self.conn.execute(product_1)
product_2 = products.insert().values(product_uuid = self.product_uuid2, product_name = u"suszarka", product_desc = u"test" )
self.conn.execute(product_2)
product_3 = products.insert().values(product_uuid = self.product_uuid3, product_name = u"pralka", product_desc = u"test" )
self.conn.execute(product_3)
buyout1 = bought_products.insert().values(quantity = 10, user_id = 1, product_id = 1)
self.conn.execute(buyout1)
buyout2 = bought_products.insert().values(quantity = 5, user_id = 1, product_id = 2)
self.conn.execute(buyout2)
buyout3 = bought_products.insert().values(quantity = 1, user_id = 1, product_id = 3)
self.conn.execute(buyout3)
buyout4 = bought_products.insert().values(quantity = 11, user_id = 2, product_id = 1)
self.conn.execute(buyout4)
buyout5 = bought_products.insert().values(quantity = 1, user_id = 2, product_id = 2)
self.conn.execute(buyout5)
buyout6 = bought_products.insert().values(quantity = 2, user_id = 3, product_id = 1)
self.conn.execute(buyout6)
def create_table(self):
metadata.create_all(self.engine)
from db import create_engine
from models import metadata
from configuration import config
# ### 初始化redis客户端
redis_config = config['REDIS']
# 当decode_responses参数为True时redis-py会自动将redis返回的字符菜转换为unicode。
# 默认不这么做的原因是：如果需要转换结果为数字，先转换为unicode就造成了额外花销。
# 节点本身不会从redis读取大量数字，但在归档数据时可能需要注意。
redis_config.update(decode_responses=True)
r = redis.StrictRedis(**redis_config)
# ###初始化数据库
db = create_engine(config['DATABASE_URL'])
# 程序在创建表格之前会检测是否存在
metadata.create_all(db)
"""
### API接口
按falcon的设计模式，每条url规则对应一个[资源]
* /api/start/{game_id} - [开始游戏]
* /api/end/{game_id} - [结束游戏]
* /api/answer/{game_id}/{question_id} - [记录答题选择]
[资源]: resources.html
[开始游戏]: resources.html#Start
[结束游戏]: resources.html#End
[记录答题选择]: resources.html#Answer
"""
api = falcon.API()
def init_db():
with app.app_context():
engine = get_engine()
metadata.create_all(engine)
from fixture import DataTestCase, SQLAlchemyFixture
from sqlalchemy import create_engine, text
from models import metadata, UserTable, RegionTable
from fixtures import RegionData, UserData
# import logging
# logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
engine = create_engine('postgresql://postgres:@localhost/sqlagg_test')
metadata.bind = engine
metadata.create_all()
engine.execute(
text('CREATE OR REPLACE VIEW "user_view" as SELECT * from user_table'))
class BaseTest(DataTestCase):
datasets = [UserData, RegionData]
fixture = SQLAlchemyFixture(engine=metadata.bind,
env={
"UserData": UserTable,
"RegionData": RegionTable
})
@classmethod
def metadata(cls):
return metadata
def get_app(self):
engine = create_engine("sqlite:///:memory:")
metadata.bind = engine
self.conn = engine.connect()
metadata.create_all()
return Application(self.conn)
class UFC():
"ULL Flow Control"
# Únicamente tenemos en cuenta correos cuyo remitente sea de nuestro dominio
only_from_domain = True
# Lista de usuarios a los que no se le aplica este filtro
whitelist = []
# Mensajes de respuesta
ans = 'El usuario ha superado el limite de correos enviados por minuto'
hold = "HOLD %s" % ans
reject = "REJECT %s" % ans
dunno = "DUNNO"
# Configuración del correo
smtp_server = 'localhost'
recipients = [
'root',
]
fqdn = socket.getfqdn()
tls_required = True
def __init__(self, verbose=False, listen_tcp=True):
if verbose:
log.setLevel(logging.DEBUG)
self.listen_tcp = listen_tcp
self.configure()
def configure(self):
config = ConfigParser.ConfigParser()
try:
config.read(config_path)
except IOError:
log.error('Error reading configuration from %s' % config_path)
return False
log.info("Reading configuration from %s" % config_path)
self.smtp_server = self.get_config(config, 'smtp', 'server',
self.smtp_server)
self.recipients = self.get_config(config, 'smtp', 'recipients',
self.recipients)
if type(self.recipients) == str:
self.recipients = str_to_list(self.recipients)
# Limits configuration
self.max_time = int(self.get_config(config, 'limits', 'max_time'))
self.max_email = int(self.get_config(config, 'limits', 'max_email'))
self.domain = self.get_config(config, 'limits', 'domain')
if self.domain is None:
try:
self.domain = self.fqdn.split('.', 1)[1]
except IndexError:
self.domain = 'localdomain'
log.info("Domain: %s " % self.domain)
self.whitelist = self.get_config(config, 'limits', 'whitelist',
self.whitelist)
if type(self.whitelist) == str:
self.whitelist = str_to_list(self.whitelist)
log.info("Whitelist: %s " % self.whitelist)
connection_string = self.get_config(config, 'database',
'connection_string')
try:
connection = create_engine(connection_string,
pool_size=25,
max_overflow=25,
pool_recycle=30)
except Exception, e:
log.error(
'Database access error, connection string used: %s. Error: %s'
% (connection_string, e))
return False
metadata.create_all(connection, checkfirst=True)
self.session = sessionmaker(bind=connection)
return True
def setUp(self):
engine = create_engine("sqlite:///:memory:")
metadata.bind = engine
metadata.create_all()
self.conn = engine.connect()
def get_app(self):
engine = create_engine("sqlite:///:memory:")
metadata.bind = engine
self.conn = engine.connect()
metadata.create_all()
return Application(self.conn)
def setUp(self):
engine = create_engine("sqlite:///:memory:")
metadata.bind = engine
metadata.create_all()
self.conn = engine.connect()
self.db_handler = BaseDBHandler(self.conn)
self.uuid1 = str(uuid.uuid4())
test_user = users.insert().values(user_uuid=self.uuid1,
username=u"konrad",
password="******",
email="*****@*****.**")
self.conn.execute(test_user)
self.uuid2 = str(uuid.uuid4())
test_user2 = users.insert().values(user_uuid=self.uuid2,
username=u"malgosia",
password="******",
email="*****@*****.**")
self.conn.execute(test_user2)
self.uuid3 = str(uuid.uuid4())
test_user3 = users.insert().values(user_uuid=self.uuid3,
username=u"kuba",
password="******",
email="*****@*****.**")
self.conn.execute(test_user3)
self.product_uuid1 = str(uuid.uuid4())
self.product_uuid2 = str(uuid.uuid4())
self.product_uuid3 = str(uuid.uuid4())
product_1 = products.insert().values(product_uuid=self.product_uuid1,
product_name=u"wiertarka",
product_desc=u"test")
self.conn.execute(product_1)
product_2 = products.insert().values(product_uuid=self.product_uuid2,
product_name=u"suszarka",
product_desc=u"test")
self.conn.execute(product_2)
product_3 = products.insert().values(product_uuid=self.product_uuid3,
product_name=u"pralka",
product_desc=u"test")
self.conn.execute(product_3)
buyout1 = bought_products.insert().values(quantity=10,
user_id=1,
product_id=1)
self.conn.execute(buyout1)
buyout2 = bought_products.insert().values(quantity=5,
user_id=1,
product_id=2)
self.conn.execute(buyout2)
buyout3 = bought_products.insert().values(quantity=1,
user_id=1,
product_id=3)
self.conn.execute(buyout3)
buyout4 = bought_products.insert().values(quantity=11,
user_id=2,
product_id=1)
self.conn.execute(buyout4)
buyout5 = bought_products.insert().values(quantity=1,
user_id=2,
product_id=2)
self.conn.execute(buyout5)
buyout6 = bought_products.insert().values(quantity=2,
user_id=3,
product_id=1)
self.conn.execute(buyout6)
from db import create_engine
from models import metadata
from configuration import config
# ### 初始化redis客户端
redis_config = config['REDIS']
# 当decode_responses参数为True时redis-py会自动将redis返回的字符菜转换为unicode。
# 默认不这么做的原因是：如果需要转换结果为数字，先转换为unicode就造成了额外花销。
# 节点本身不会从redis读取大量数字，但在归档数据时可能需要注意。
redis_config.update(decode_responses=True)
r = redis.StrictRedis(**redis_config)
# ###初始化数据库
db = create_engine(config['DATABASE_URL'])
# 程序在创建表格之前会检测是否存在
metadata.create_all(db)
"""
### API接口
按falcon的设计模式，每条url规则对应一个[资源]
* /api/start/{game_id} - [开始游戏]
* /api/end/{game_id} - [结束游戏]
* /api/answer/{game_id}/{question_id} - [记录答题选择]
[资源]: resources.html
[开始游戏]: resources.html#Start
[结束游戏]: resources.html#End
[记录答题选择]: resources.html#Answer
"""
from models import (
DBSession,
metadata
)
from sqlalchemy import create_engine
import settings
engine = create_engine('mysql+mysqldb://'+settings.db_user+':'+settings.db_password+'@'+settings.db_host+'/'+settings.db_name)
DBSession.configure(bind=engine)
metadata.bind = engine
import models
metadata.drop_all()
DBSession.flush()
metadata.create_all()
from business.game import Game
game = Game('The game you just lost')
DBSession.add(game)
from business.game.player import PlayerGame
player = PlayerGame('Me')
DBSession.add(player)
player.game = game
from business.army.army import Army
from models import checkin, metadata
from sqlalchemy import create_engine, MetaData
# Generates the variables used later
engine = create_engine('sqlite:///checkins.db')
#TODO: get the name associated with cardnumber from API
def returnName(numb):
#Temp name until API is in place
name = "Onliner"
return name
# Connects and safely creates the DB if it doesn't exist
try:
conn = engine.connect()
metadata.create_all(engine)
except:
print("Oooops. Something went wrong while connecting to the database, is it up?")
number = ""
# Main program that loops
while (number != 'exit'):
print ("Scan Card:")
# Gets input from card-reader
number = str(input())
dateNow = datetime.datetime.now()
firstWatch = dateNow.replace(hour=8, minute=0, second=0, microsecond=0)
from typing import Any
from sqlalchemy import create_engine, Table, and_
from models import metadata
_USER = '******'
_PASSWORD = '******'
_HOST = 'localhost:5432'
_DBNAME = 'combat_sports_db'
_ENGINE = create_engine(
f"postgresql+psycopg2://{_USER}:{_PASSWORD}@{_HOST}/{_DBNAME}")
metadata.create_all(_ENGINE)
def get_entity(table: Table, conditions: dict[str, Any]) -> dict:
with _ENGINE.connect() as connection:
return connection.execute(table.select().where(
and_(*[
getattr(table.c, column_name) == column_value
for column_name, column_value in conditions.items()
]))).fetchone()
def create_entity(table: Table, values: dict) -> dict:
with _ENGINE.connect() as connection:
return connection.execute(
table.insert().values(**values).returning(*table.c)).fetchone()
def setUp(self):
engine = create_engine("sqlite:///:memory:")
metadata.bind = engine
metadata.create_all()
self.conn = engine.connect()
self.product_handler = ProductDatabaseHandler(self.conn)
def init(engine: Engine) -> None:
""" init tables"""
metadata.create_all(bind=engine)
def create_test_database():
engine = sqlalchemy.create_engine(DATABASE_URL)
metadata.create_all(engine)
yield
metadata.drop_all(engine)
def create_db():
engine = get_engine()
metadata.create_all(engine)
pymysql.install_as_MySQLdb()
from gevent.queue import Queue
from gevent.queue import Empty
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, scoped_session
from models import metadata, Region, Xiaoqu, Chengjiao
from LianJiaSpider import do_xiaoqu_spider, xiaoqu_chengjiao_spider
from optparse import OptionParser
engine = create_engine('mysql://root@localhost/lianjia?charset=utf8',
echo=False)
xiaoqu_queue = Queue(maxsize=20)
default_xiaoqu_thread = 10
metadata.create_all(engine)
Session = sessionmaker(bind=engine, autoflush=True, autocommit=False)
# Session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=False))
def xiaoqu_callback(res):
session = Session()
for item in res:
xiaoqu = session.query(Xiaoqu).filter(Xiaoqu.href == item.href).first()
if not xiaoqu:
session.add(item)
session.commit()
def get_all_xiaoqu():
session = Session()
def setUp(self):
engine = create_engine("sqlite:///:memory:")
metadata.bind = engine
metadata.create_all()
self.conn = engine.connect()
self.product_handler = ProductDatabaseHandler(self.conn)
