def lock(self, lock_slug):
lock_filename = self.lock_filename(lock_slug)
if exists(lock_filename):
raise StorageError(f"Can't lock: Storage is already locked: "
f'"{lock_filename}"')
write(lock_filename, dt())
self.locked = True
def _all(self, limit=None):
iterator = storage.iterate_pages(silent=True, limit=limit)
for i, (title, page) in enumerate(iterator):
self._debug_title(i, title)
if not i % 100000:
print(dt(), i)
self.update_page(page)
self.convert_entries()
self.export_entries('.current')
def update_page(self, title, content, edited, redirect):
old_content, old_info = self.storage.get(title, silent=True)
edited_str = dt(edited, utc=True)
info = f"{edited_str}, {'R' if redirect else 'A'}"
if content != old_content or info != old_info:
self.storage.update(title, content=content, info=info)
self.log_hour('changed', f'<{info}> - {title}')
self.log_day('titles_changed', title)
def wrapped(*args, **kwargs):
lock_file = join(conf.root_path, 'sys', 'lock', slug)
if exists(lock_file):
print(dt(), f'Already locked: `{slug}`')
return
write(lock_file, '')
try:
return func(*args, **kwargs)
finally:
os.remove(lock_file)
def lock(self, lock_slug):
lock_filename = self.lock_filename(lock_slug)
if exists(lock_filename):
storages = {
'/home/vitalik/storages/wiktionary/storage'
'/authors/sys/lock_recent':
'*authors*',
}
storage_name = storages.get(lock_filename, lock_filename)
post_to_slack(
'recent-errors', f':lock: `{Logger.slug}` '
f'Storage is locked: {storage_name}')
raise StorageAlreadyLocked(
f'Can\'t lock: Storage is already locked: "{lock_filename}"')
write(lock_filename, dt())
self.locked = True
def update_page(self, page): # todo: rename to `update_page`
title = page.title
if title in self.storage.titles_set:
# Информация об авторе этой статьи уже сохранена
self._debug_skipped()
return False
# get oldest revision
oldest = next(
get_page(title).revisions(reverseOrder=True, total=1,
content=True))
created_at = dt(oldest.timestamp, utc=True)
created_lang = self.get_created_lang(title, oldest.text)
created_author = oldest.user
created = f"{created_at}, {created_lang}, {created_author}"
self._debug_info(created)
self.storage.update(title, created=created)
self.log_hour('saved', f'<{created}> - {title}')
self.log_day('titles_saved', title)
self._debug_processed()
return True
def lock_file(
filename
): # todo: возможно попробовать сделать context manager `with locked(...):`
if is_locked(filename):
raise LockError(filename)
write(filename + b'.lock', dt())
def update_page(self, title, content, edited, redirect):
edited_str = dt(edited, utc=True)
info = f"{edited_str}, {'R' if redirect else 'A'}"
self.storage.update(title, content=content, info=info)
self.log_hour('changed', f'<{info}> - {title}')
self.log_day('titles_changed', title)
def _debug_title(self, title):
print(dt(), title)
def latest_updated(self, value):
write(self.latest_updated_filename, dt(value, utc=True))
def _debug_title(self, i, title):
print(dt(), i, title, ' -- ', end='')
def _debug_titles_saved(self):
print(dt(), '## Titles saved!')