def test_get_path():
assert Config.get_datastore_path("choka.csv")
assert Config.get_download_path("choka_daikoku_2021_04_001.html")
assert Config.test_resource("daikoku1.html")
assert Config.get_url("daikoku")
assert Config.get_download_file("daikoku", 2021, 4)
assert Config.get_db_path()
assert Config.get_config_path("config.toml")
def load_config(self, config_path=Config.get_config_path()):
"""
設定ファイルを読み、ホームページダウンロードパラメータを登録します
"""
config_toml = toml.load(open(config_path, encoding='utf-8'))
if 'area' in config_toml:
self.areas = config_toml.get('area')
if 'interval' in config_toml:
self.crawl_interval = config_toml['interval']
if 'max_page' in config_toml:
self.max_page = config_toml['max_page']
return self
def test_toml_multibyte():
config_path = Config.get_config_path("config.toml")
config_toml = toml.load(open(config_path, encoding='utf-8'))
df = pd.DataFrame(columns=['Target', 'Species'])
if 'target' in config_toml:
targets = config_toml['target']
for target in targets:
target_name = target['name']
for species in target['species']:
values = {'Target': target_name, 'Species': species}
df = df.append(values, ignore_index=True)
print(df)
def parser(self):
"""
コマンド実行オプションの解析
"""
parser = argparse.ArgumentParser(description=Description)
parser.add_argument("-c",
"--config",
type=str,
default=Config.get_config_path(),
help="<path>\\config.toml")
parser.add_argument("-m",
"--month",
type=int,
default=0,
help="last n month before downloading")
parser.add_argument("-p",
"--page",
type=int,
default=config.MaxPage,
help="max number of pages to visit the homepage")
parser.add_argument("-i",
"--init",
action="store_true",
help="initialize database")
parser.add_argument("-k",
"--keep",
action="store_true",
help="keep old download files")
parser.add_argument("-l",
"--log",
action="store_true",
help="write log to file")
parser.add_argument("-s",
"--show",
action="store_true",
help="show config parameter")
parser.add_argument("-e",
"--export",
action="store_true",
help="export csv data")
parser.add_argument("--loadmaster",
action="store_true",
help="import master data")
parser.add_argument("-t",
"--time",
type=str,
default="1day",
help="time period to export")
return parser.parse_args()
def load_config(self, config_path=Config.get_config_path()):
"""
設定ファイルを読み、ホームページダウンロードパラメータを登録します
"""
config_toml = toml.load(open(config_path, encoding='utf-8'))
"""魚種ターゲットの読み込み"""
if 'target' in config_toml:
targets = config_toml['target']
for target in targets:
target_name = target['name']
for species in target['species']:
values = {'Target': target_name, 'Species': species}
self.target = self.target.append(values, ignore_index=True)
"""魚種ターゲットの読み込み"""
if 'area' in config_toml:
areas = config_toml['area']
for area in areas:
values = {'Point': area['name'], 'PointName': area['label']}
self.area = self.area.append(values, ignore_index=True)
return self