def get_html(url):
fpath = Path(__file__).change(
new_basename="{}.html".format(fingerprint.of_text(url)))
if fpath.exists():
html = fpath.read_text(encoding="utf-8")
else:
# i am lazy, I don't want to login, session_id is the key
headers = {
"user-agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
"accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"accept-encoding": "gzip, deflate, br",
"accept-language":
"en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,ja;q=0.5",
"sec-fetch-mode": "navigate",
"sec-fetch-site": "none",
# "cookie": "csrftoken=9999JP9mz49NwmqfykyrMupzywy3XZNGgq7Rly23j0u2QuEHdDcOreAnWIjGIhtj; _ga=GA1.2.1853091755.1574187377; _gid=GA1.2.909819738.1574187377; intercom-id-aaaa73km=aaaa1111-18b1-48af-849d-94ad2564a3bc; ga_cid=1853091755.1574187377; sessionid=t2xtsqy6pkf3mkndvd8oljs102ffp6bc; intercom-session-aaaa73km=bXI5eG01b1pJdHlmSk9mYU1jSzZPNGVpWng0KzR6Snk3MngwUjJtNVRzWHhzSHlEenBqTXYyNDRwMWZaekxydC0tUmtTUVo1bjlaNmo3SDVIVFVhcGpCZz09--d927dd0ff0f890889144d645e77525943c851cf5"
}
res = requests.get(url, headers=headers)
html = res.text
fpath.write_text(html, encoding="utf-8")
return html
# -*- coding: utf-8 -*-
from jinja2 import Template
from pathlib_mate import Path
from create_app import app
HERE = Path(__file__).parent
# jinja2 Template creation reference: https://jinja.palletsprojects.com/en/2.11.x/api/#jinja2.Template
circleci_single_template = Template(
Path(HERE, ".circleci", "config-single.yml").read_text(encoding="utf-8")
)
circleci_parallel_template = Template(
Path(HERE, ".circleci", "config-parallel.yml").read_text(encoding="utf-8")
)
app.plan()
circleci_config_yml = Path(HERE, ".circleci", "config.yml")
circleci_config_yml.write_text(
circleci_parallel_template.render(app=app)
)
dir_here = Path(__file__).absolute().parent
if __name__ == "__main__":
# read config from cookiecutter-pygitrepo.json
path_cookiecutter_pygitrepo_json = Path(dir_here,
"cookiecutter-pygitrepo.json")
data = json.loads(
strip_comments(path_cookiecutter_pygitrepo_json.read_text()))
del data["_please_ignore_this"]
config = Config(**data)
config.path_cookiecutter_pygitrepo_json = path_cookiecutter_pygitrepo_json.abspath
# dump context data to cookiecutter.json
path_cookiecutter_json = Path(dir_here, "cookiecutter.json")
path_cookiecutter_json.write_text(
json.dumps(config.to_context_data(), indent=4))
# clean up existing environment
dir_output = Path(dir_here, "tmp")
dir_output_project_root = Path(dir_output, config.repo_name)
if dir_output_project_root.exists():
shutil.rmtree(dir_output_project_root.abspath)
# create project skeleton
cookiecutter(
template=dir_here.abspath,
output_dir=dir_output.abspath,
no_input=True,
)
username = getpass.getuser()
# by default, username is your $USER, and there's no password
conn_str = "postgresql+psycopg2://{}:@localhost:5432/upaya_development".format(
username)
engine = sqlalchemy.create_engine(conn_str)
metadata = sqlalchemy.MetaData()
metadata.reflect(engine)
l = list()
for tname, t in sorted(metadata.tables.items(), key=lambda x: x[0]):
title = "{table_name}".format(table_name=tname)
h = Header(title=title, header_level=2)
lt_data = [("name", "fullname", "type")]
for cname, c in sorted(t.columns.items(), key=lambda x: x[0]):
lt_data.append(
(CodeBlockSQL.from_string(cname),
CodeBlockSQL.from_string(tname + "." + cname), str(c.type)))
lt = ListTable(data=lt_data,
title="columns",
index=False,
header=True,
class_="sortable")
l.append(h)
l.append(lt)
content = "\n\n".join([obj.render() for obj in l])
p = Path(__file__).change(new_basename="IDP-DB-Schema.rst")
p.write_text(content, encoding="utf-8")
import sqlalchemy_mate as sam
from uszipcode.model import SimpleZipcode, ComprehensiveZipcode
db_file_path = "/Users/sanhehu/.crawl_uszipcode/comprehensive.sqlite"
engine = sam.EngineCreator().create_sqlite(path=db_file_path)
Zipcode = ComprehensiveZipcode
with orm.Session(engine) as ses:
stmt = sa.select(
Zipcode.zipcode,
Zipcode.major_city,
Zipcode.state,
Zipcode.population,
).where(Zipcode.population.between(10000, 50000)).limit(10)
res = ses.execute(stmt)
print(sam.pt.from_result(res))
# stmt = sa.select(Zipcode).limit(20)
# res = ses.execute(stmt)
# sam.pt.from_everything(Zipcode, ses)
# n = Zipcode.count_all(ses)
# print(n)
import json
from pathlib_mate import Path
l = sam.selecting.select_single_column(engine, Zipcode.__table__.c.zipcode)
p = Path(__file__).change(new_basename="data.json")
p.write_text(json.dumps(l))
