def test_sweep_scheduled_sweep():
ct.place_config_file("", {"pattern": "*", "ttl": 2})
conf = config.create_config_map(ct.ROOT_DIR)
schedule.every(1).seconds.do(lambda: sweeper.sweep(conf))
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
file_path_old = ct.write_csv("", "test_old.csv", [("hello", "world"), (1, 2), (3, 4)])
schedule.run_pending()
w.wait(2)
file_path_new = ct.write_csv("", "test_new.csv", [("hello", "world"), (1, 2), (3, 4)])
schedule.run_pending()
w.wait(1)
schedule.run_pending()
w.stop()
dir_db = ct.read_db_file(ct.ROOT_DIR)
dir_conf = conf[ct.ROOT_DIR]
assert not Path(file_path_old).exists()
assert (Path(ct.ROOT_DIR) / dir_conf["archive"]["name"] / Path(file_path_old).name).exists()
assert Path(file_path_new).exists()
dir_db = ct.read_db_file(ct.ROOT_DIR)
assert Path(file_path_new).name in dir_db["watching"].keys()
assert Path(file_path_old).name not in dir_db["watching"].keys()
def test_sweep_directory_files_into_archive_zip():
ct.place_config_file("", {"pattern": "*", "archive": {"type": "zip", "name": "_archive.zip"}})
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
file_path_old = ct.write_csv("", "test_old.csv", [("hello", "world"), (1, 2), (3, 4)])
file_path_new = ct.write_csv("", "test_new.csv", [("hello", "world"), (1, 2), (3, 4)])
w.wait(1)
w.stop()
archive_target_files = [Path(file_path_old)]
dir_db = ct.read_db_file(ct.ROOT_DIR)
dir_conf = conf[ct.ROOT_DIR]
sweeper._sweep_directory_files(Path(ct.ROOT_DIR), archive_target_files, dir_db, dir_conf)
assert Path(file_path_new).exists()
assert not Path(file_path_old).exists()
zip_file = Path(ct.ROOT_DIR) / dir_conf["archive"]["name"]
assert zip_file.exists()
with zipfile.ZipFile(str(zip_file)) as existing_zip:
assert Path(file_path_old).name in existing_zip.namelist()
def test_sweep_directory():
ct.place_config_file("", {"pattern": "*", "ttl": 10})
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
file_path_old = ct.write_csv("", "test_old.csv", [("hello", "world"), (1, 2), (3, 4)])
file_path_new = ct.write_csv("", "test_new.csv", [("hello", "world"), (1, 2), (3, 4)])
w.wait(1)
w.stop()
# modify last_detected for debugging
last_detected = datetime(2020, 7, 19, 0, 0, 0, 0, timezone(timedelta(hours=0)))
ct.mod_last_detected(file_path_old, last_detected)
ct.mod_last_detected(file_path_new, last_detected + timedelta(seconds=10))
current_time = last_detected + timedelta(seconds=10)
sweeper.sweep(conf, current_time)
dir_db = ct.read_db_file(ct.ROOT_DIR)
dir_conf = conf[ct.ROOT_DIR]
assert not Path(file_path_old).exists()
assert (Path(ct.ROOT_DIR) / dir_conf["archive"]["name"] / Path(file_path_old).name).exists()
assert Path(file_path_new).exists()
dir_db = ct.read_db_file(ct.ROOT_DIR)
assert Path(file_path_new).name in dir_db["watching"].keys()
assert Path(file_path_old).name not in dir_db["watching"].keys()
def test_sweep_directory_list_return_expired_file_list():
ct.place_config_file("", {"pattern": "*", "ttl": 10})
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
file_path_old = ct.write_csv("", "test_old.csv", [("hello", "world"), (1, 2), (3, 4)])
file_path_new = ct.write_csv("", "test_new.csv", [("hello", "world"), (1, 2), (3, 4)])
w.wait(1)
w.stop()
# modify last_detected for debugging
last_detected = datetime(2020, 7, 19, 0, 0, 0, 0, timezone(timedelta(hours=0)))
ct.mod_last_detected(file_path_old, last_detected)
ct.mod_last_detected(file_path_new, last_detected + timedelta(seconds=10))
db = ct.read_db_file(ct.ROOT_DIR)
current_time = last_detected + timedelta(seconds=10)
conf_dir = conf[ct.ROOT_DIR]
actual_sweep_targets = sweeper._sweep_directory_list_target(Path(ct.ROOT_DIR), db, conf_dir, current_time)
expected_sweep_targets = [file_path_old]
assert expected_sweep_targets == actual_sweep_targets
def test_sweep_directory_files_into_archive_already_exists_save_with_datetime_postfix():
ct.place_config_file("", {"pattern": "*"})
conf = config.create_config_map(ct.ROOT_DIR)
dir_conf = conf[ct.ROOT_DIR]
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
file_path_old = ct.write_csv("", "test.csv", [("hello", "world"), (1, 2), (3, 4)])
w.wait(1)
w.stop()
dir_db = ct.read_db_file(ct.ROOT_DIR)
expected_content_old = file_path_old.read_text()
sweeper._sweep_directory_files(Path(ct.ROOT_DIR), [Path(file_path_old)], dir_db, dir_conf)
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
file_path_new = ct.write_csv("", "test.csv", [("hello", "world"), (5, 6), (7, 8)])
w.wait(1)
w.stop()
dir_db = ct.read_db_file(ct.ROOT_DIR)
expected_content_new = file_path_new.read_text()
now = datetime(2019, 8, 15, 1, 39, 0, 0 * 1000, timezone(timedelta(hours=-6)))
sweeper._sweep_directory_files(Path(ct.ROOT_DIR), [Path(file_path_new)], dir_db, dir_conf, now)
assert not Path(file_path_old).exists()
archived_file_old = Path(ct.ROOT_DIR) / dir_conf["archive"]["name"] / Path(file_path_old).name
assert archived_file_old.exists()
actual_content_old = archived_file_old.read_text()
assert expected_content_old == actual_content_old
assert not Path(file_path_new).exists()
archived_file_new = Path(ct.ROOT_DIR) / dir_conf["archive"]["name"] / "test_20190815013900.csv"
assert archived_file_new.exists()
actual_content_new = archived_file_new.read_text()
assert expected_content_new == actual_content_new
def test_create_db_file_when_receive_file_event():
ct.place_config_file("", {"pattern": "*"})
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
ct.place_file("", "sample-file.csv", "hoge")
w.wait(1)
w.stop()
db = ct.read_db_file(ct.ROOT_DIR)
keys = db["watching"].keys()
assert "sample-file.csv" in keys
def test_sweep_directory_files_into_archive_zip_already_exists_save_with_datetime_postfix():
ct.place_config_file("", {"pattern": "*", "archive": {"type": "zip", "name": "_archive.zip"}})
conf = config.create_config_map(ct.ROOT_DIR)
dir_conf = conf[ct.ROOT_DIR]
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
file_path_old = ct.write_csv("", "test.csv", [("hello", "world"), (1, 2), (3, 4)])
w.wait(1)
w.stop()
dir_db = ct.read_db_file(ct.ROOT_DIR)
sweeper._sweep_directory_files(Path(ct.ROOT_DIR), [Path(file_path_old)], dir_db, dir_conf)
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
file_path_new = ct.write_csv("", "test.csv", [("hello", "world"), (5, 6), (7, 8)])
w.wait(1)
w.stop()
dir_db = ct.read_db_file(ct.ROOT_DIR)
now = datetime(2019, 8, 15, 1, 39, 0, 0 * 1000, timezone(timedelta(hours=-6)))
sweeper._sweep_directory_files(Path(ct.ROOT_DIR), [Path(file_path_new)], dir_db, dir_conf, now)
zip_file_path = Path(ct.ROOT_DIR) / dir_conf["archive"]["name"]
with zipfile.ZipFile(zip_file_path) as zip_file:
files_in_zip = zip_file.namelist()
assert not Path(file_path_old).exists()
archived_file_old = Path(ct.ROOT_DIR) / dir_conf["archive"]["name"] / Path(file_path_old).name
assert archived_file_old.name in files_in_zip
assert not Path(file_path_new).exists()
archived_file_new = Path(ct.ROOT_DIR) / dir_conf["archive"]["name"] / "test_20190815013900.csv"
assert archived_file_new.name in files_in_zip
def test_return_create_notify():
ct.place_config_file("sub", {"pattern": "*.csv"})
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
ct.place_file("sub", f"sample.csv", "hello,world1")
w.wait(1)
w.stop()
expected = 1
actual = len(events)
assert expected == actual
def test_inject_on_start(self):
ct.place_config_file("", {"pattern": "*.csv"})
conf = config.create_config_map(ct.ROOT_DIR)
ct.place_file("", "sample.csv", "hello,world")
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf,
lambda ev: events.append(ev))
w.start()
w.wait(1)
w.stop()
assert 1 == len(events)
ev = events[0]
assert ev.src_path == "/".join([ct.ROOT_DIR, "sample.csv"])
def test_ignore_db_file():
ct.place_config_file("c", {"pattern": "*"})
ct.place_config_file("c/j", {"pattern": "*", "db_file": "cache-file.yml"})
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
ct.place_file("c", "hoge.csv", "hoge")
ct.place_file("c/j", "hoge.csv", "hoge")
w.wait(1)
w.stop()
expected = [f"{ct.ROOT_DIR}/c/hoge.csv", f"{ct.ROOT_DIR}/c/j/hoge.csv"]
actual = [e.src_path for e in events]
assert expected == actual
def test_return_delayed_events():
ct.place_config_file("", {
"pattern": "*.csv",
"callback_delay": 2
}) # root configuration
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
ct.place_file("", f"hoge.csv", "hello,csv")
w.wait(1)
assert 0 == len(events)
w.wait(2)
assert 1 == len(events)
w.stop()
def test_sweep_directory_files_into_archive_dir():
ct.place_config_file("", {"pattern": "*"})
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
file_path_old = ct.write_csv("", "test_old.csv", [("hello", "world"), (1, 2), (3, 4)])
file_path_new = ct.write_csv("", "test_new.csv", [("hello", "world"), (1, 2), (3, 4)])
w.wait(1)
w.stop()
archive_target_files = [Path(file_path_old)]
dir_db = ct.read_db_file(ct.ROOT_DIR)
dir_conf = conf[ct.ROOT_DIR]
sweeper._sweep_directory_files(Path(ct.ROOT_DIR), archive_target_files, dir_db, dir_conf)
assert not Path(file_path_old).exists()
assert (Path(ct.ROOT_DIR) / dir_conf["archive"]["name"] / Path(file_path_old).name).exists()
assert Path(file_path_new).exists()
def test_ignore_duplicated_events_in_callback_delay_multi_dir():
ct.place_config_file("mario", {"pattern": "*.csv", "callback_delay": 2})
ct.place_config_file("luigi", {"pattern": "*.txt", "callback_delay": 1})
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
ct.place_file("mario", f"hoge.csv", "hello,world")
ct.place_file("luigi", f"hoge.txt", "hello,world")
w.wait(3) # 1event
w.stop()
expected = [
f"{ct.ROOT_DIR}/luigi/hoge.txt",
f"{ct.ROOT_DIR}/mario/hoge.csv",
]
actual = [e.src_path for e in events]
assert expected == actual
def test_return_ignored_duplicated_events():
ct.place_config_file("", {
"min_mod_interval": 10,
"pattern": "*.csv"
}) # root configuration
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
ct.place_file("", f"hoge.csv", "hello,csv")
ct.place_file("", f"fuga.csv", "hello,csv")
ct.place_file("", f"hoge.csv", "hello,csv") # should be ignore
ct.place_file("", f"fuga.csv", "hello,csv") # should be ignore
ct.place_file("", f"hoge.csv", "hello,csv") # should be ignore
w.wait(1)
w.stop()
expected = [f"{ct.ROOT_DIR}/hoge.csv", f"{ct.ROOT_DIR}/fuga.csv"]
actual = [e.src_path for e in events]
assert expected == actual
def test_return_specified_pattern():
ct.place_config_file("c", {"pattern": "*.csv"})
ct.place_config_file("c/j", {"pattern": "*.json"})
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
ct.place_file("c", f"sample.csv", "hello,csv")
ct.place_file("c", f"sample.json", '{"hello":"json"}')
ct.place_file("c/j", f"sample.csv", "hello,csv")
ct.place_file("c/j", f"sample.json", '{"hello":"json"}')
w.wait(1)
w.stop()
expected = [
f"{ct.ROOT_DIR}/c/sample.csv", f"{ct.ROOT_DIR}/c/j/sample.json"
]
actual = [e.src_path for e in events]
assert expected == actual
def test_ignore_duplicated_events_in_callback_delay():
ct.place_config_file("", {
"pattern": "*.csv",
"min_mod_interval": 0,
"callback_delay": 2
}) # root configuration
conf = config.create_config_map(ct.ROOT_DIR)
events = []
w = watcher.NamazuWatcher(ct.ROOT_DIR, conf, lambda ev: events.append(ev))
w.start()
ct.place_file("", "hoge.csv", "hello,csv")
w.wait(1) # will be ignored
ct.place_file("", "hoge.csv", "hello,csv")
w.wait(1) # will be ignored
ct.place_file("", "hoge.csv", "hello,csv")
w.wait(1) # will be ignored
ct.place_file("", "hoge.csv", "hello,csv")
w.wait(1)
w.wait(3) # 1event
w.stop()
assert 1 == len(events)
import time
from onamazu import config
from onamazu import watcher
events = []
ROOT_DIR = 'sample'
conf = config.create_config_map(ROOT_DIR)
w = watcher.NamazuWatcher(ROOT_DIR, conf, lambda ev: events.append(ev))
try:
w.start()
while w.isAlive():
w.wait(1)
print(w.isAlive())
except KeyboardInterrupt:
w.stop()