def get_html(url): fpath = Path(__file__).change( new_basename="{}.html".format(fingerprint.of_text(url))) if fpath.exists(): html = fpath.read_text(encoding="utf-8") else: # i am lazy, I don't want to login, session_id is the key headers = { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36", "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", "accept-encoding": "gzip, deflate, br", "accept-language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,ja;q=0.5", "sec-fetch-mode": "navigate", "sec-fetch-site": "none", # "cookie": "csrftoken=9999JP9mz49NwmqfykyrMupzywy3XZNGgq7Rly23j0u2QuEHdDcOreAnWIjGIhtj; _ga=GA1.2.1853091755.1574187377; _gid=GA1.2.909819738.1574187377; intercom-id-aaaa73km=aaaa1111-18b1-48af-849d-94ad2564a3bc; ga_cid=1853091755.1574187377; sessionid=t2xtsqy6pkf3mkndvd8oljs102ffp6bc; intercom-session-aaaa73km=bXI5eG01b1pJdHlmSk9mYU1jSzZPNGVpWng0KzR6Snk3MngwUjJtNVRzWHhzSHlEenBqTXYyNDRwMWZaekxydC0tUmtTUVo1bjlaNmo3SDVIVFVhcGpCZz09--d927dd0ff0f890889144d645e77525943c851cf5" } res = requests.get(url, headers=headers) html = res.text fpath.write_text(html, encoding="utf-8") return html
# -*- coding: utf-8 -*- from jinja2 import Template from pathlib_mate import Path from create_app import app HERE = Path(__file__).parent # jinja2 Template creation reference: https://jinja.palletsprojects.com/en/2.11.x/api/#jinja2.Template circleci_single_template = Template( Path(HERE, ".circleci", "config-single.yml").read_text(encoding="utf-8") ) circleci_parallel_template = Template( Path(HERE, ".circleci", "config-parallel.yml").read_text(encoding="utf-8") ) app.plan() circleci_config_yml = Path(HERE, ".circleci", "config.yml") circleci_config_yml.write_text( circleci_parallel_template.render(app=app) )
dir_here = Path(__file__).absolute().parent if __name__ == "__main__": # read config from cookiecutter-pygitrepo.json path_cookiecutter_pygitrepo_json = Path(dir_here, "cookiecutter-pygitrepo.json") data = json.loads( strip_comments(path_cookiecutter_pygitrepo_json.read_text())) del data["_please_ignore_this"] config = Config(**data) config.path_cookiecutter_pygitrepo_json = path_cookiecutter_pygitrepo_json.abspath # dump context data to cookiecutter.json path_cookiecutter_json = Path(dir_here, "cookiecutter.json") path_cookiecutter_json.write_text( json.dumps(config.to_context_data(), indent=4)) # clean up existing environment dir_output = Path(dir_here, "tmp") dir_output_project_root = Path(dir_output, config.repo_name) if dir_output_project_root.exists(): shutil.rmtree(dir_output_project_root.abspath) # create project skeleton cookiecutter( template=dir_here.abspath, output_dir=dir_output.abspath, no_input=True, )
username = getpass.getuser() # by default, username is your $USER, and there's no password conn_str = "postgresql+psycopg2://{}:@localhost:5432/upaya_development".format( username) engine = sqlalchemy.create_engine(conn_str) metadata = sqlalchemy.MetaData() metadata.reflect(engine) l = list() for tname, t in sorted(metadata.tables.items(), key=lambda x: x[0]): title = "{table_name}".format(table_name=tname) h = Header(title=title, header_level=2) lt_data = [("name", "fullname", "type")] for cname, c in sorted(t.columns.items(), key=lambda x: x[0]): lt_data.append( (CodeBlockSQL.from_string(cname), CodeBlockSQL.from_string(tname + "." + cname), str(c.type))) lt = ListTable(data=lt_data, title="columns", index=False, header=True, class_="sortable") l.append(h) l.append(lt) content = "\n\n".join([obj.render() for obj in l]) p = Path(__file__).change(new_basename="IDP-DB-Schema.rst") p.write_text(content, encoding="utf-8")
import sqlalchemy_mate as sam from uszipcode.model import SimpleZipcode, ComprehensiveZipcode db_file_path = "/Users/sanhehu/.crawl_uszipcode/comprehensive.sqlite" engine = sam.EngineCreator().create_sqlite(path=db_file_path) Zipcode = ComprehensiveZipcode with orm.Session(engine) as ses: stmt = sa.select( Zipcode.zipcode, Zipcode.major_city, Zipcode.state, Zipcode.population, ).where(Zipcode.population.between(10000, 50000)).limit(10) res = ses.execute(stmt) print(sam.pt.from_result(res)) # stmt = sa.select(Zipcode).limit(20) # res = ses.execute(stmt) # sam.pt.from_everything(Zipcode, ses) # n = Zipcode.count_all(ses) # print(n) import json from pathlib_mate import Path l = sam.selecting.select_single_column(engine, Zipcode.__table__.c.zipcode) p = Path(__file__).change(new_basename="data.json") p.write_text(json.dumps(l))