def init_autoindex(self, auto_interval):
if not auto_interval:
return
from pywb.manager.autoindex import AutoIndexer
colls_dir = self.warcserver.root_dir if self.warcserver.root_dir else None
indexer = AutoIndexer(colls_dir=colls_dir, interval=int(auto_interval))
if not os.path.isdir(indexer.root_path):
msg = 'No managed directory "{0}" for auto-indexing'
logging.error(msg.format(indexer.root_path))
import sys
sys.exit(2)
msg = 'Auto-Indexing Enabled on "{0}", checking every {1} secs'
logging.info(msg.format(indexer.root_path, auto_interval))
indexer.start()
def test_auto_index(self):
main(['init', 'auto'])
auto_dir = os.path.join(self.root_dir, COLLECTIONS, 'auto')
archive_dir = os.path.join(auto_dir, ARCHIVE_DIR)
archive_sub_dir = os.path.join(archive_dir, 'sub')
os.makedirs(archive_sub_dir)
def do_copy():
try:
time.sleep(1.0)
shutil.copy(self._get_sample_warc('example.warc.gz'),
archive_dir)
shutil.copy(self._get_sample_warc('example-extra.warc'),
archive_sub_dir)
time.sleep(1.0)
finally:
indexer.interval = 0
indexer = AutoIndexer(interval=0.25)
indexer.start()
ge = gevent.spawn(do_copy)
ge.join()
index_file = os.path.join(auto_dir, INDEX_DIR, AUTOINDEX_FILE)
assert os.path.isfile(index_file)
with open(index_file, 'r') as fh:
index = fh.read()
assert '"example.warc.gz' in index, index
assert '"sub/example-extra.warc' in index, index
mtime = os.path.getmtime(index_file)
# Update
indexer.interval = 0.25
indexer.start()
os.remove(index_file)
#thread = threading.Thread(target=do_copy)
#thread.daemon = True
#thread.start()
ge = gevent.spawn(do_copy)
#wayback(['-p', '0', '-a', '--auto-interval', '0.25'])
#thread.join()
ge.join()
# assert file was update
assert os.path.getmtime(index_file) > mtime
def init_autoindex(self, auto_interval):
"""Initialize and start the auto-indexing of the collections. If auto_interval is None this is a no op.
:param str|int auto_interval: The auto-indexing interval from the configuration file or CLI argument
"""
if not auto_interval:
return
from pywb.manager.autoindex import AutoIndexer
colls_dir = self.warcserver.root_dir if self.warcserver.root_dir else None
indexer = AutoIndexer(colls_dir=colls_dir, interval=int(auto_interval))
if not os.path.isdir(indexer.root_path):
msg = 'No managed directory "{0}" for auto-indexing'
logging.error(msg.format(indexer.root_path))
import sys
sys.exit(2)
msg = 'Auto-Indexing Enabled on "{0}", checking every {1} secs'
logging.info(msg.format(indexer.root_path, auto_interval))
indexer.start()
def init_autoindex(self, auto_interval):
"""Initialize and start the auto-indexing of the collections. If auto_interval is None this is a no op.
:param str|int auto_interval: The auto-indexing interval from the configuration file or CLI argument
"""
if not auto_interval:
return
from pywb.manager.autoindex import AutoIndexer
colls_dir = self.warcserver.root_dir if self.warcserver.root_dir else None
indexer = AutoIndexer(colls_dir=colls_dir, interval=int(auto_interval))
if not os.path.isdir(indexer.root_path):
msg = 'No managed directory "{0}" for auto-indexing'
logging.error(msg.format(indexer.root_path))
import sys
sys.exit(2)
msg = 'Auto-Indexing Enabled on "{0}", checking every {1} secs'
logging.info(msg.format(indexer.root_path, auto_interval))
indexer.start()
def test_auto_index(self):
main(['init', 'auto'])
auto_dir = os.path.join(self.root_dir, COLLECTIONS, 'auto')
archive_dir = os.path.join(auto_dir, ARCHIVE_DIR)
archive_sub_dir = os.path.join(archive_dir, 'sub')
os.makedirs(archive_sub_dir)
def do_copy():
try:
time.sleep(1.0)
shutil.copy(self._get_sample_warc('example.warc.gz'), archive_dir)
shutil.copy(self._get_sample_warc('example-extra.warc'), archive_sub_dir)
time.sleep(1.0)
finally:
indexer.interval = 0
indexer = AutoIndexer(interval=0.25)
indexer.start()
ge = gevent.spawn(do_copy)
ge.join()
index_file = os.path.join(auto_dir, INDEX_DIR, AUTOINDEX_FILE)
assert os.path.isfile(index_file)
with open(index_file, 'r') as fh:
index = fh.read()
assert '"example.warc.gz' in index, index
assert '"sub/example-extra.warc' in index, index
mtime = os.path.getmtime(index_file)
# Update
indexer.interval = 0.25
indexer.start()
os.remove(index_file)
#thread = threading.Thread(target=do_copy)
#thread.daemon = True
#thread.start()
ge = gevent.spawn(do_copy)
#wayback(['-p', '0', '-a', '--auto-interval', '0.25'])
#thread.join()
ge.join()
# assert file was update
assert os.path.getmtime(index_file) > mtime
def setup_class(cls):
super(TestRecordReplay, cls).setup_class('config_test_record.yaml')
cls.indexer = AutoIndexer(interval=0.1)
cls.indexer.start()