def test_post_valid_record():
queries = pugsql.module("tests/webapi/queries")
queries.connect(V.PLAY_DB_URL)
for record in V.valid_records:
original_publication = queries.get_publication_by_id(
publication_id=record["publication_id"])
r = requests.post(api_url + "playground/add_record", json=record)
assert r.status_code == 200
assert r.json()["record"]
inserted_record = queries.get_record(
publication_id=record["publication_id"],
play_at=r.json()["record"]["play_at"],
)
assert inserted_record
assert json.loads(inserted_record["content"]) == record["tokens"]
updated_publication = queries.get_publication_by_id(
publication_id=record["publication_id"])
assert updated_publication["last_play_at"] > (
original_publication["last_play_at"] or 0)
assert (updated_publication["play_count"] ==
original_publication["play_count"] + 1)
queries.disconnect()
def init():
handler = pugsql.module('assets/sql')
handler.connect('sqlite:///memory')
settings.sql = handler
# Create table if they don't exist
handler.org_create()
handler.user_create()
handler.prof_create()
handler.fund_create()
handler.stat_create()
handler.value_create()
handler.role_create()
handler.role_user_create()
# Add test rows if they don't exist
org = handler.org_find(id=1)
if (org == None):
org = Organization.add('Test Company')
admin = User.add('admin', 'test123')
guest = User.add('guest', 'test123')
admin_role = Role.add('admin')
admin.add_to_role(role_id=admin_role.id)
profile = Profile.add('Test Profile', org.id)
fund = Fund.add('Test Fund', 'Test Manager', 2000, 0.00, 0.00,
profile.id)
# return the queries handler
return handler
def connect(connection_string):
queries = pugsql.module("db/")
queries.connect(connection_string)
queries.create_table_package()
queries.create_table_package_event()
queries.create_table_package_subscription()
return queries
def populate_db():
db = pugsql.module('tests/test_fixtures/queries')
db.connect('mysql+pymysql://widgets:password@localhost/widgets')
db.add_widgets()
db.add_orders()
db.populate_widgets()
db.populate_orders()
db.disconnect()
def main(args):
queries = pugsql.module('queries')
queries.connect(os.getenv("DB_URL"))
date_start_unix, date_end_unix = date_to_unix(args.date)
for stats in queries.count_site_stats(time_start=date_start_unix,
time_end=date_end_unix):
queries.upsert_stats({**stats, 'date': args.date})
def get_db():
if 'db' not in g:
try:
logger.debug('Establishing new connection to DB')
g.db = pugsql.module('api/db/queries/')
g.db.connect(f'mysql+pymysql://{current_app.config["DB_USER"]}:{current_app.config["DB_PASS"]}@{current_app.config["DB_HOST"]}/{current_app.config["DB_NAME"]}')
except Exception as err:
raise exceptions.DBError(err)
return g.db
def startpy():
# Create a module of database functions from a set of sql files on disk.
queries = pugsql.module('resources/sql')
# Point the module at your database.
queries.connect('sqlite:///test.db')
# Invoke parameterized queries, receive dicts!
city = queries.get_city(name='Theni')
print(city)
def __init__(self, configFile, configPath, parameters, htmlBody):
self.configFile = configFile
self.configPath = configPath
self.htmlBody = htmlBody
self.parameters = parameters
modPath = os.path.join(configPath, self.configFile["sqlfolder"])
self.queries = pugsql.module(modPath)
conn = self.configFile["connection"]
if "<absolutePath>" in conn:
conn.replace("<absolutePath>", configPath)
self.queries.connect(conn)
return
def download(stub):
queries = pugsql.module(os.environ['QUERIES_PATH'])
queries.connect(os.environ['DB_CONNECTION_URL'])
s3 = get_s3()
bucket = os.environ['S3_BUCKET']
result = queries.key_from_stub(stub = stub)
if result is None:
return 'invalid'
key = result['key']
tmp_file = '/tmp/{}'.format(uuid.uuid4())
with open(tmp_file, 'wb') as data:
s3.download_fileobj(bucket, key, data)
return send_file(tmp_file, attachment_filename = key)
def upload():
if 'file' not in request.files:
return 'no file'
upload_file = request.files['file']
if upload_file.filename == '':
return 'no selected file'
key = secure_filename(upload_file.filename)
stub = generate_stub()
queries = pugsql.module(os.environ['QUERIES_PATH'])
queries.connect(os.environ['DB_CONNECTION_URL'])
s3 = get_s3()
s3.upload_fileobj(upload_file, os.environ['S3_BUCKET'], key)
queries.insert_stub(stub = stub, key = key)
return stub
def __init__(self, configFile, configPath, parameters=None):
self.configFile = configFile
self.configPath = configPath
self.parameters = parameters
modPath = os.path.join(configPath, self.configFile["sqlfolder"])
self.queries = pugsql.module(modPath)
conn = self.configFile["connection"]
if "<absolutePath>" in conn:
conn = conn.replace("<absolutePath>", configPath)
self.queries.connect(conn)
self.templateDir = os.path.join(os.path.dirname(
os.path.abspath(__file__)), "templates")
if "templates" in self.configFile and \
"css" in self.configFile["templates"]:
with open(os.path.join(configPath,
self.configFile["templates"]
["css"]), "r") as f:
self.baseCss = f.read()
else:
with open(os.path.join(self.templateDir,
"base_styles.css"), "r") as f:
self.baseCss = f.read()
if "templates" in self.configFile and \
"html" in self.configFile["templates"]:
self.templateDir = os.path.join(configPath,
self.configFile["templates"]
["html"])
self.env = Environment(
loader=FileSystemLoader(self.templateDir),
autoescape=select_autoescape(['html', 'xml'])
)
def module(*args, **kwargs):
queries = pugsql.module(*args, **kwargs)
queries.add_query = add_query.__get__(queries)
queries.add_method = add_method.__get__(queries)
return queries
def setup_db():
# todo: figure out pooling with pug
queries = pugsql.module('./sql/queries')
queries.connect(db.db_url())
flask.current_app.queries = queries
logging.info('ok pugsql')
def run(runner, site_id, args=None):
queries = pugsql.module("./queries")
queries.connect(os.getenv("DB_URL"))
site_info = queries.get_site_by_id(site_id=site_id)
dedup_limit = args["dedup_limit"] or 500
recent_articles = queries.get_recent_articles_by_site(site_id=site_id,
limit=dedup_limit)
queries.disconnect()
site_conf = SiteConfig.default()
site_conf.update(json.loads(site_info["config"]))
site_conf["url"] = site_info["url"]
site_conf["type"] = site_info["type"]
if args is not None:
site_conf.update(args)
settings = {
**get_project_settings(),
"DEPTH_LIMIT": site_conf["depth"],
"DOWNLOAD_DELAY": site_conf["delay"],
"USER_AGENT": site_conf["ua"],
}
if "appledaily" in site_conf["url"]:
site_conf["selenium"] = True
if "dcard" in site_conf["url"]:
crawler = Crawler(DcardDiscoverSpider, settings)
crawler.stats.set_value("site_id", site_id)
runner.crawl(
crawler,
site_id=site_id,
site_url=site_conf["url"],
article_url_excludes=[a["url"] for a in recent_articles],
)
elif "toutiao" in site_conf["url"]:
crawler = Crawler(ToutiaoDiscoverSpider, settings)
crawler.stats.set_value("site_id", site_id)
runner.crawl(
crawler,
site_id=site_id,
site_url=site_conf["url"],
article_url_excludes=[a["url"] for a in recent_articles],
)
elif "ptt.cc" in site_conf["url"]:
ptt.DiscoverSite(site_info).run(depth=site_conf["depth"])
else:
crawler = Crawler(BasicDiscoverSpider, settings)
crawler.stats.set_value("site_id", site_id)
runner.crawl(
crawler,
site_id=site_id,
site_url=site_conf["url"],
article_url_patterns=site_conf["article"],
following_url_patterns=site_conf["following"],
article_url_excludes=[a["url"] for a in recent_articles],
selenium=site_conf.get("selenium", False),
)
import flask_api
from flask import request, jsonify
from flask_api import status, exceptions
import uuid
import pugsql
import sqlite3
app = flask_api.FlaskAPI(__name__)
app.config.from_envvar('APP_CONFIG')
queries = [
pugsql.module('queries/tracks1'),
pugsql.module('queries/tracks2'),
pugsql.module('queries/tracks3')
]
for i in range(len(queries)):
queries[i].connect(app.config[f'TRACKS{i+1}_DATABASE_URL'])
sqlite3.register_converter('GUID', lambda b: uuid.UUID(bytes_le=b))
sqlite3.register_adapter(uuid.UUID, lambda u: u.bytes_le)
@app.route('/tracks/<uuid:id>', methods=['GET', 'DELETE', 'PATCH'])
def track(id):
if request.method == 'GET':
return get_track(id)
elif request.method == 'DELETE':
return delete_track(id)
elif request.method == 'PATCH':
return update_track(id, request.data)
def init():
self.fixtures = pugsql.module('tests/sql/fixtures')
self.fixtures.connect('sqlite:///./tests/data/fixtures.sqlite3')
def test_module():
compiler.modules.clear()
assert pugsql.module('tests/sql').sqlpath == 'tests/sql'
def test_transaction_not_connected(self):
fixtures = pugsql.module('tests/sql/fixtures')
fixtures.disconnect()
with pytest.raises(exceptions.NoConnectionError):
with fixtures.transaction():
pass
import pugsql
import sys, os
from libs import load as pcld, send as pcsend
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
# Load all of the *sql files in the queries/ directory into a single module.
queries = pugsql.module('sample/sql/')
queries.connect('sqlite:///sample/sample.db')
'''
print("Connected")
most_sold = [i for i in getattr(queries, "most_sold")()]
print("most_sold \n", most_sold)
pugsql.get_modules().clear()
'''
# test my loader
parameters = dict({"kind": "audio"})
pc = pcld('sample/sample-config.json', parameters)
html = pc.build()
f = open("test.html", "w")
f.write(html)
f.close()
pcsend(pc)
def test_bad_path(self):
with pytest.raises(ValueError,
match='Directory not found: does/not/exist'):
pugsql.module('does/not/exist')
def setUp(self):
pugsql.get_modules().clear()
self.fixtures = pugsql.module('tests/sql/fixtures')
self.fixtures.connect('sqlite:///./tests/data/fixtures.sqlite3')
'supply':
u.supply,
'is_worker':
u.is_worker,
'is_army':
u.is_army,
'is_building':
u.is_building,
'race':
u.race,
})
sc2reader.engine.register_plugin(UnitCollector())
queries = pugsql.module('sql/')
queries.connect('postgresql+pg8000://%[email protected]:5432/sc2' % getuser())
sc2dir = os.path.expanduser(
'~/Library/Application Support/Blizzard/Starcraft II')
replay_files = Path(sc2dir).rglob('*.SC2Replay')
replays = (sc2reader.load_replay(str(p), load_level=4) for p in replay_files)
def one_replay():
for r in replays:
return r
def parse(r):
print(f'Parsing {r.filename}')
def module(module_path):
return pugsql.module("queries")
import flask_api
from flask import request
from flask_api import status, exceptions
import pugsql
app = flask_api.FlaskAPI(__name__)
app.config.from_envvar('APP_CONFIG')
queries = pugsql.module('queries/descriptions')
queries.connect(app.config['MUSIC_DATABASE_URL'])
@app.route('/descriptions/<uuid:id>', methods=['GET'])
def description(id):
description = queries.description_by_track_id(id=str(id))
if description:
return description
else:
raise exceptions.NotFound()
@app.route('/descriptions', methods=['POST'])
def descriptions():
if request.method == 'POST':
return insert_description(request.data)
def insert_description(description):
required_fields = ['id', 'user_name', 'description']
if not all([field in description for field in required_fields]):
raise exceptions.ParseError()
def __init__(self, site_info):
self.queries = pugsql.module("./queries")
self.queries.connect(os.getenv("DB_URL"))
self.site_url = site_info["url"]
self.site_id = site_info["site_id"]
def test_not_connected(self):
fixtures = pugsql.module('tests/sql/fixtures')
fixtures.disconnect()
with pytest.raises(exceptions.NoConnectionError):
fixtures.user_for_id(user_id=1)
#
# * New API calls:
# - GET /api/v1/resources/books/{id} to retrieve a specific book
# - POST /api/v1/resources/books to create a new book
#
import sys
import flask_api
from flask import request
from flask_api import status, exceptions
import pugsql
app = flask_api.FlaskAPI(__name__)
app.config.from_envvar('APP_CONFIG')
queries = pugsql.module('queries/')
queries.connect(app.config['DATABASE_URL'])
@app.cli.command('init')
def init_db():
with app.app_context():
db = queries._engine.raw_connection()
with app.open_resource('books.sql', mode='r') as f:
db.cursor().executescript(f.read())
db.commit()
@app.route('/', methods=['GET'])
def home():
return '''<h1>Distant Reading Archive</h1>
import pugsql
from psycopg2.extensions import register_adapter, AsIs
# register_converter('GUID', lambda b: uuid.UUID(bytes_le=b))
register_adapter(uuid.UUID, lambda u: u.bytes_le)
# # def addapt_numpy_float64(numpy_float64):
# # return AsIs(numpy_float64)
# register_adapter(numpy.float64, addapt_numpy_float64)
queries = pugsql.module( os.path.abspath(os.path.dirname(__file__)) + '/queries/')
# queries.connect("sqlite:///main.db")
# conn = sqlite3.connect('test.db', detect_types=sqlite3.PARSE_DECLTYPES)
# c = conn.cursor()
# c.execute('CREATE TABLE test (guid GUID PRIMARY KEY, name TEXT)')
data = (uuid.uuid4(), 'foo')
print ('Input Data:', data)
c.execute('INSERT INTO test VALUES (?,?)', data)
c.execute('SELECT * FROM test')
print ('Result Data:', c.fetchone())
#!/usr/bin/env python3
import argparse
import pugsql
import os
from dotenv import load_dotenv
load_dotenv()
import datetime
from tabulate import tabulate
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from newsSpiders.runner import discover, update
queries = pugsql.module("./queries")
queries.connect(os.getenv("DB_URL"))
def main(args):
if args.command == "discover":
process = CrawlerProcess(get_project_settings())
discover.run(process, args.id, vars(args))
process.start()
elif args.command == "update":
print("updating articles from site " + str(args.id))
process = CrawlerProcess(get_project_settings())
update.run(process, args.id, vars(args))
process.start()
import sys
import flask_api
import pugsql
from flask import request, jsonify, Response
from flask_api import status, exceptions
from werkzeug.security import generate_password_hash, check_password_hash
app = flask_api.FlaskAPI(__name__)
app.config.from_envvar('APP_CONFIG')
queries = pugsql.module('queries/user')
queries.connect(app.config['DATABASE_URL'])
def validContentType(request, type='application/json'):
if request.headers.has_key('Content-Type'):
if request.headers['Content-Type'] == type:
return True
return { 'Error':'Unsupported Media Type', 'Support-Content-Type':type}, status.HTTP_415_UNSUPPORTED_MEDIA_TYPE
@app.route('/', methods=['GET'])
def home():
return '''<h1>Cloue-Works-SERVICE</h1>'''
@app.route('/api/v1/cloud/users/register', methods=['POST', 'GET'])
def register():
if request.method=='GET':
all_users=queries.all_users()
data = list(all_users)
return data, status.HTTP_200_OK
elif request.method=='POST':
