def fixcode(**kwargs):
"""
auto pep8 format all python file in ``source code`` and ``tests`` dir.
"""
# repository direcotry
repo_dir = Path(__file__).parent.absolute()
# source code directory
source_dir = Path(repo_dir, package.__name__)
if source_dir.exists():
print("Source code locate at: '%s'." % source_dir)
print("Auto pep8 all python file ...")
source_dir.autopep8(**kwargs)
else:
print("Source code directory not found!")
# unittest code directory
unittest_dir = Path(repo_dir, "tests")
if unittest_dir.exists():
print("Unittest code locate at: '%s'." % unittest_dir)
print("Auto pep8 all python file ...")
unittest_dir.autopep8(**kwargs)
else:
print("Unittest code directory not found!")
print("Complete!")
def reformat(**kwargs):
"""
auto pep8 format all python file in ``source code`` and ``tests`` dir.
"""
# repository direcotry
repo_dir = Path(__file__).parent.absolute()
# source code directory
source_dir = Path(repo_dir, package.__name__)
if source_dir.exists():
print("Source code locate at: '%s'." % source_dir)
print("Auto pep8 all python file ...")
source_dir.autopep8(**kwargs)
else:
print("Source code directory not found!")
# unittest code directory
unittest_dir = Path(repo_dir, "tests")
if unittest_dir.exists():
print("Unittest code locate at: '%s'." % unittest_dir)
print("Auto pep8 all python file ...")
unittest_dir.autopep8(**kwargs)
else:
print("Unittest code directory not found!")
print("Complete!")
def get_testdata():
"""
**中文文档**
下载测试数据。
"""
for page, state, county, zipcode, street in testdata:
url = urlencoder.browse_home_listpage_url(state, county, zipcode,
street)
filepath = Path("testdata", "%s.html" % page)
if not filepath.exists():
html = spider.get_html(url, encoding="utf-8")
textfile.write(html, filepath.abspath)
for href in zillow_house_url_list:
url = urlencoder.url_join(href)
zid = href.split("/")[-2]
filepath = Path("testdata", "%s.html" % zid)
if not filepath.exists():
html = spider.get_html(url, encoding="utf-8")
textfile.write(html, filepath.abspath)
def fixcode():
# repository direcotry
repo_dir = Path(__file__).absolute().parent.parent
# source code directory
source_dir = Path(repo_dir, repo_dir.basename.replace("-project", ""))
if source_dir.exists():
print("Source code locate at: '%s'." % source_dir)
print("Auto pep8 all python file ...")
source_dir.autopep8()
else:
print("Source code directory not found!")
# unittest code directory
unittest_dir = Path(repo_dir, "tests")
if unittest_dir.exists():
print("Unittest code locate at: '%s'." % unittest_dir)
print("Auto pep8 all python file ...")
unittest_dir.autopep8()
else:
print("Unittest code directory not found!")
print("Complete!")
def __init__(self, name, path=None, parent=None, is_single_file=None):
assert_is_valid_name(name)
self.name = name
self.parent = parent
self.is_single_file = is_single_file
if path is None:
chain = self.name.split(".") # "a.b.c" -> ["a", "b", "c"]
root = chain[0]
# test if using .egg-link
p = Path(SP_DIR, root.replace("_", "-") + ".egg-link")
if p.is_file() and p.exists():
with open(p.abspath, "rb") as f:
sp_dir = f.readline().decode("utf-8").strip()
else:
sp_dir = SP_DIR
# is single file package
p = Path(Path(sp_dir, *chain).abspath + ".py")
if p.is_file() and p.exists():
self.path = p
self.is_single_file = True
return
# then has to be a directory having __init__.py file
p = Path(sp_dir, *chain)
if p.is_dir() and p.exists() and Path(p, "__init__.py").exists():
self.path = Path(sp_dir, *chain)
self.is_single_file = False
return
raise ValueError("Can't found '%s'!" % self.name)
else:
self.path = path
def run(self):
node = nodes.Element()
node.document = self.state.document
current_file = self.state.document.current_source
current_dir = Path(current_file).parent
image_dir = Path(Path(current_file).parent, self.arguments[0])
n_columns = self.options.get("n_columns", 3)
if image_dir.exists():
output_rst = derive_rst(
current_dir=current_dir,
image_dir=image_dir,
n_columns=n_columns,
)
else:
output_rst = ""
view_list = StringList(output_rst.splitlines(), source='')
sphinx.util.nested_parse_with_titles(self.state, view_list, node)
return node.children
def make_gzip(this_file, filename):
"""
将文件名为 <filename> 的 ``.tsv`` 文件 (后缀名必须是tsv, 否则出粗) 压缩成 ``.tsv.gz``
的压缩包文件. 执行该操作的 python 脚本必须和 ``.tsv`` 文件在同一目录下.
:type this_file: str
:param this_file: 当前 python 脚本的文件全路径
:type filename: str
:param filename: ``.tsv`` 文件名
"""
if not filename.endswith(".tsv"):
raise ValueError
dst = Path(this_file).change(new_basename=filename + ".gz")
if not dst.exists():
with open(Path(this_file).change(new_basename=filename).abspath, "rb") as f:
b = f.read()
with gzip.open(dst.abspath, 'wb') as f:
f.write(b)
def get_html(url):
fpath = Path(__file__).change(
new_basename="{}.html".format(fingerprint.of_text(url)))
if fpath.exists():
html = fpath.read_text(encoding="utf-8")
else:
# i am lazy, I don't want to login, session_id is the key
headers = {
"user-agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
"accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"accept-encoding": "gzip, deflate, br",
"accept-language":
"en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7,zh-TW;q=0.6,ja;q=0.5",
"sec-fetch-mode": "navigate",
"sec-fetch-site": "none",
# "cookie": "csrftoken=9999JP9mz49NwmqfykyrMupzywy3XZNGgq7Rly23j0u2QuEHdDcOreAnWIjGIhtj; _ga=GA1.2.1853091755.1574187377; _gid=GA1.2.909819738.1574187377; intercom-id-aaaa73km=aaaa1111-18b1-48af-849d-94ad2564a3bc; ga_cid=1853091755.1574187377; sessionid=t2xtsqy6pkf3mkndvd8oljs102ffp6bc; intercom-session-aaaa73km=bXI5eG01b1pJdHlmSk9mYU1jSzZPNGVpWng0KzR6Snk3MngwUjJtNVRzWHhzSHlEenBqTXYyNDRwMWZaekxydC0tUmtTUVo1bjlaNmo3SDVIVFVhcGpCZz09--d927dd0ff0f890889144d645e77525943c851cf5"
}
res = requests.get(url, headers=headers)
html = res.text
fpath.write_text(html, encoding="utf-8")
return html
def _download_db_file_if_not_exists(self):
if self.db_file_path is None:
self.db_file_path = self._default_db_file_path_mapper[
self.simple_or_comprehensive]
if self.download_url is None:
self.download_url = self._default_download_url_mapper[
self.simple_or_comprehensive]
p = Path(self.db_file_path)
if not p.exists():
if self.simple_or_comprehensive is self.SimpleOrComprehensiveArgEnum.simple:
download_db_file(
db_file_path=self.db_file_path,
download_url=self.download_url,
chunk_size=1024 * 1024,
progress_size=1024 * 1024,
)
elif self.simple_or_comprehensive is self.SimpleOrComprehensiveArgEnum.comprehensive:
download_db_file(
db_file_path=self.db_file_path,
download_url=self.download_url,
chunk_size=1024 * 1024,
progress_size=50 * 1024 * 1024,
)
def fly(self):
"""
Generate doc tree.
"""
dst_dir = Path(self.conf_file).parent.abspath
package_dir = Path(dst_dir, self.package.shortname)
# delete existing api document
try:
if package_dir.exists():
shutil.rmtree(package_dir.abspath)
except Exception as e:
print("'%s' can't be removed! Error: %s" % (package_dir, e))
# create .rst files
for pkg, parent, sub_packages, sub_modules in self.package.walk():
if not is_ignored(pkg, self.ignored_package):
dir_path = Path(*([
dst_dir,
] + pkg.fullname.split(".")))
init_path = Path(dir_path, "__init__.rst")
make_dir(dir_path.abspath)
make_file(
init_path.abspath,
self.generate_package_content(pkg),
)
for mod in sub_modules:
if not is_ignored(mod, self.ignored_package):
module_path = Path(dir_path, mod.shortname + ".rst")
make_file(
module_path.abspath,
self.generate_module_content(mod),
)
dir_here = Path(__file__).absolute().parent
if __name__ == "__main__":
# read config from cookiecutter-pygitrepo.json
path_cookiecutter_pygitrepo_json = Path(dir_here,
"cookiecutter-pygitrepo.json")
data = json.loads(
strip_comments(path_cookiecutter_pygitrepo_json.read_text()))
del data["_please_ignore_this"]
config = Config(**data)
config.path_cookiecutter_pygitrepo_json = path_cookiecutter_pygitrepo_json.abspath
# dump context data to cookiecutter.json
path_cookiecutter_json = Path(dir_here, "cookiecutter.json")
path_cookiecutter_json.write_text(
json.dumps(config.to_context_data(), indent=4))
# clean up existing environment
dir_output = Path(dir_here, "tmp")
dir_output_project_root = Path(dir_output, config.repo_name)
if dir_output_project_root.exists():
shutil.rmtree(dir_output_project_root.abspath)
# create project skeleton
cookiecutter(
template=dir_here.abspath,
output_dir=dir_output.abspath,
no_input=True,
)
