def on_moved(self, event):
if event.src_path and match_path(event.src_path,
included_patterns=self.patterns,
excluded_patterns=self.ignore_patterns,
case_sensitive=self.case_sensitive):
self._deletes.append(event.src_path)
if event.dest_path and match_path(event.dest_path,
included_patterns=self.patterns,
excluded_patterns=self.ignore_patterns,
case_sensitive=self.case_sensitive):
self._inserts.append(event.dest_path)
self.flush()
def file_changes (self): modified_files = [] path = "/".join([self.connection["store"], "themes", self.connection["theme"]]) rs_keys = self.connection["bucket"].list(prefix=path) # Go though each file in s3 and compare it with the local file system for key_val in rs_keys: loc_path = key_val.name.replace(path+'/', '') filepath = os.path.join(self.config.watch_dir, self.__normalize_path(loc_path)) # Only remove files that are on the watch list if not match_path(filepath, included_patterns=self.config.file_patterns, excluded_patterns=self.config.ignore_patterns, case_sensitive=True): continue if not os.path.isfile(filepath): modified_files.append(loc_path) else: md5hash = hashlib.md5(open(filepath, 'rb').read()).hexdigest() # The etag is quoted, so they need to be removed to compare if not (md5hash == key_val.etag[1:-1]): modified_files.append(loc_path) return modified_files
def on_deleted(self, event):
borg = ConfBorg()
self.conf = borg.conf
self.dao = LycheeDAO(self.conf)
if event.is_directory:
album = getAlbum(self, event.src_path)
if album['id'] is not None:
filelist = self.dao.eraseAlbum(album['id'])
deleteFiles(self, filelist)
logger.info("Deleted album: %s.", album['name'])
assert self.dao.dropAlbum(album['id'])
else:
logger.error("Tried to delete album: %s, but it wasn't present in the DB", album['name'])
return
else:
if match_path(event.src_path,
included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
excluded_patterns=None,
case_sensitive=False):
dirs = event.src_path.split(os.sep)
albDir = os.sep.join(dirs[:-1])
album = getAlbum(self, albDir)
album['path'] = albDir
dbPhoto = self.dao.get_photo_light(album['id'], os.sep.join(dirs[-1:]), "")
if dbPhoto is not None:
delete = [dbPhoto]
deletePhotos(self, delete)
logger.info("Deleted Photo: %s.", os.sep.join(dirs[-1:]))
else:
logger.info("Tried to delete Photo: %s, but it wasn't in the database.", os.sep.join(dirs[-1:]))
return
def on_moved(self, event):
"""
Called when a file or a directory is moved or renamed.
Many editors don't directly change a file, instead they make a
transitional file like ``*.part`` then move it to the final filename.
Args:
event: Watchdog event, either ``watchdog.events.DirMovedEvent`` or
``watchdog.events.FileModifiedEvent``.
"""
if not self._event_error:
# We are only interested for final file, not transitional file
# from editors (like *.part)
pathtools_options = {
'included_patterns': self.patterns,
'excluded_patterns': self.ignore_patterns,
'case_sensitive': self.case_sensitive,
}
# Apply pathtool matching on destination since Watchdog only
# automatically apply it on source
if match_path(event.dest_path, **pathtools_options):
self.logger.info(u"Change detected from a move on: %s",
event.dest_path)
self.compile_dependencies(event.dest_path)
def sync (self, files):
for f in files:
path = os.path.join(self.config.watch_dir, f)
if not match_path(path,
included_patterns=self.config.file_patterns,
excluded_patterns=self.config.ignore_patterns,
case_sensitive=True):
continue
key = "/".join([self.connection["store"], "themes", self.connection["theme"], f.replace('\\', '/')])
# Split the path into the directory and the filename
file_dir, file_name = os.path.split(path)
# Check to see if the file path exists, and create the needed
# directories if it does not
if not os.path.exists(file_dir):
os.makedirs(file_dir, 0o755)
if not os.path.isdir(path):
# This will create the file if it does not exist
fi = open(path, 'wb+')
k = self.connection["bucket"].new_key(key)
k.get_contents_to_file(fi)
print(Fore.GREEN + '[' + time.strftime("%c") + '] Successfully downloaded ' + f + Style.RESET_ALL)
return
def on_moved(self, event):
borg = ConfBorg()
self.conf = borg.conf
self.dao = LycheeDAO(self.conf)
if event.is_directory:
albSrc = getAlbum(self, event.src_path)
albDest = getAlbum(self, event.dest_path)
logger.info("%s Album moved to %s. ", event.src_path, event.dest_path)
self.dao.setAlbumParentAndTitle(albDest['name'], albDest['parent'], albSrc['id'])
return
else:
if match_path(event.src_path,
included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
excluded_patterns=None,
case_sensitive=False):
dirs = event.src_path.split(os.sep)
albDir = os.sep.join(dirs[:-1])
dirs2 = event.dest_path.split(os.sep)
albDir2 = os.sep.join(dirs2[:-1])
album = getAlbum(self, albDir)
if album['id'] == None:
album = getAlbum(self, albDir2)
dbPhoto = self.dao.get_photo_light(album['id'], os.sep.join(dirs[-1:]), "")
album2 = getAlbum(self, albDir2)
logger.info("%s Photo moved to %s. ", event.src_path, event.dest_path)
self.dao.setPhotoAlbumAndTitle(os.sep.join(dirs2[-1:]), album2['id'], dbPhoto['id'])
return
def on_moved(self, event):
# We are only interested for the destination
if match_path(event.dest_path,
included_patterns=self.patterns,
excluded_patterns=self.ignore_patterns,
case_sensitive=self.case_sensitive):
self.logger.debug("Change detected from a move on: %s", event.dest_path)
self.build_for_item(event.dest_path)
def enum_files(start_dir, patterns=None, ignore_patterns=None, case_sensitive=False):
test = (lambda x : True) if patterns is None else (lambda x: match_path(x,
included_patterns=patterns,
excluded_patterns=ignore_patterns,
case_sensitive=case_sensitive)
)
for (path, dirs, files) in os_walk(start_dir):
for file in files:
if not test(file):
continue
yield pjoin(path, file)
def process_templates(source_path, dest_path, config):
for found in path.walk(path.absolute_path(source_path)):
for file_name in found[2]:
real_file_name = "%s/%s" % (found[0], file_name)
if patterns.match_path(real_file_name, ["*.tpl.html"]):
template_processor = JinjaTemplateProcessor()
output_dir = template_processor.mkdirs(
real_file_name, path.absolute_path(dest_path),
path.absolute_path(source_path))
output_file = template_processor.write_templates(
real_file_name, output_dir, config=config)
print("jinja wrote file {0}/{1}".format(
output_dir, output_file))
def on_any_event(self, event):
if not event.is_directory \
and match_path(event.src_path,
included_patterns=self.include_pattern,
excluded_patterns=self.exclude_pattern,
case_sensitive=self.case_sensitive) \
and event.event_type == EVENT_TYPE_CREATED:
try:
file_path = event.src_path.decode('utf-8')
media_file = self.add_to_processing_queue(file_path)
if media_file:
logger.info("File [{}] added to processing queue".format(
media_file.identifier))
logger.debug(media_file)
except Exception:
logger.exception(
"An error occurred during adding of [{}] to processing queue"
.format(file_path))
def on_modified(self, event):
borg = ConfBorg()
self.conf = borg.conf
self.dao = LycheeDAO(self.conf)
if event.is_directory:
return
else:
if match_path(event.src_path,
included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
excluded_patterns=None,
case_sensitive=False):
dirs = event.src_path.split(os.sep)
albDir = os.sep.join(dirs[:-1])
album = getAlbum(self, albDir)
album['path'] = albDir
photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
dbPhoto = self.dao.get_photo(photo)
if dbPhoto is not None:
delete = [dbPhoto]
deletePhotos(self, delete)
dirs = event.src_path.split(os.sep)
albDir = os.sep.join(dirs[:-1])
album = getAlbum(self, albDir)
album['path'] = albDir
photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
if not (self.dao.photoExists(photo)):
res = copyFileToLychee(self, photo)
adjustRotation(self, photo)
makeThumbnail(self, photo)
res = self.dao.addFileToAlbum(photo)
logger.info("Modified Photo: %s.", photo.srcfullpath)
# increment counter
if not res:
logger.error(
"while adding to album: %s photo: %s",
album['name'],
photo.srcfullpath)
else:
logger.error(
"photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
photo.srcfullpath)
return
def on_modified(self, event):
borg = ConfBorg()
self.conf = borg.conf
self.dao = LycheeDAO(self.conf)
if event.is_directory:
return
else:
if match_path(
event.src_path,
included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
excluded_patterns=None,
case_sensitive=False):
dirs = event.src_path.split(os.sep)
albDir = os.sep.join(dirs[:-1])
album = getAlbum(self, albDir)
album['path'] = albDir
photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
dbPhoto = self.dao.get_photo(photo)
if dbPhoto is not None:
delete = [dbPhoto]
deletePhotos(self, delete)
dirs = event.src_path.split(os.sep)
albDir = os.sep.join(dirs[:-1])
album = getAlbum(self, albDir)
album['path'] = albDir
photo = LycheePhoto(self.conf, os.sep.join(dirs[-1:]), album)
if not (self.dao.photoExists(photo)):
res = copyFileToLychee(self, photo)
adjustRotation(self, photo)
makeThumbnail(self, photo)
res = self.dao.addFileToAlbum(photo)
logger.info("Modified Photo: %s.", photo.srcfullpath)
# increment counter
if not res:
logger.error("while adding to album: %s photo: %s",
album['name'], photo.srcfullpath)
else:
logger.error(
"photo already exists in this album with same name or same checksum: %s it won't be added to lychee",
photo.srcfullpath)
return
def load(file, chat, patterns):
file.seek(0)
lines = iter(file)
ver, oldchat = next(lines, chat.id).rstrip('\n').split(' ')
if int(ver) != 1:
raise RuntimeError("Version mismatch")
if oldchat != str(chat.id):
raise RuntimeError('Chat ID mismatch')
files = dict()
for line in lines:
msgid, digest, name = line.rstrip('\n').split('\t', maxsplit=2)
if not match_path(name, patterns, case_sensitive=True):
continue
assert (msgid == TOMBSTONE) == (digest == TOMBSTONE)
if msgid == TOMBSTONE:
files.pop(name, None)
else:
files[name] = File(int(msgid), digest)
return files
def on_moved(self, event):
"""
Called when a file or a directory is moved or renamed.
Many editors don't directly change a file, instead they make a
transitional file like ``*.part`` then move it to the final filename.
Arguments:
event: Watchdog event, either ``watchdog.events.DirMovedEvent`` or
``watchdog.events.FileModifiedEvent``.
"""
# We are only interested for destination
if match_path(event.dest_path,
included_patterns=self.patterns,
excluded_patterns=self.ignore_patterns,
case_sensitive=self.case_sensitive):
msg = "Change detected from a move on: {}"
self.logger.debug(msg.format(event.dest_path))
return self.build_for_item(event.dest_path)
return []
def on_moved(self, event):
borg = ConfBorg()
self.conf = borg.conf
self.dao = LycheeDAO(self.conf)
if event.is_directory:
albSrc = getAlbum(self, event.src_path)
albDest = getAlbum(self, event.dest_path)
logger.info("%s Album moved to %s. ", event.src_path,
event.dest_path)
self.dao.setAlbumParentAndTitle(albDest['name'], albDest['parent'],
albSrc['id'])
return
else:
if match_path(
event.src_path,
included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
excluded_patterns=None,
case_sensitive=False):
dirs = event.src_path.split(os.sep)
albDir = os.sep.join(dirs[:-1])
dirs2 = event.dest_path.split(os.sep)
albDir2 = os.sep.join(dirs2[:-1])
album = getAlbum(self, albDir)
if album['id'] == None:
album = getAlbum(self, albDir2)
dbPhoto = self.dao.get_photo_light(album['id'],
os.sep.join(dirs[-1:]), "")
album2 = getAlbum(self, albDir2)
logger.info("%s Photo moved to %s. ", event.src_path,
event.dest_path)
self.dao.setPhotoAlbumAndTitle(os.sep.join(dirs2[-1:]),
album2['id'], dbPhoto['id'])
return
def on_deleted(self, event):
borg = ConfBorg()
self.conf = borg.conf
self.dao = LycheeDAO(self.conf)
if event.is_directory:
album = getAlbum(self, event.src_path)
if album['id'] is not None:
filelist = self.dao.eraseAlbum(album['id'])
deleteFiles(self, filelist)
logger.info("Deleted album: %s.", album['name'])
assert self.dao.dropAlbum(album['id'])
else:
logger.error(
"Tried to delete album: %s, but it wasn't present in the DB",
album['name'])
return
else:
if match_path(
event.src_path,
included_patterns=['*.jpg', '*.jpeg', '*.gif', '*.png'],
excluded_patterns=None,
case_sensitive=False):
dirs = event.src_path.split(os.sep)
albDir = os.sep.join(dirs[:-1])
album = getAlbum(self, albDir)
album['path'] = albDir
dbPhoto = self.dao.get_photo_light(album['id'],
os.sep.join(dirs[-1:]), "")
if dbPhoto is not None:
delete = [dbPhoto]
deletePhotos(self, delete)
logger.info("Deleted Photo: %s.", os.sep.join(dirs[-1:]))
else:
logger.info(
"Tried to delete Photo: %s, but it wasn't in the database.",
os.sep.join(dirs[-1:]))
return
def remove_local_files(self, directory):
for the_file in os.listdir(directory):
file_path = os.path.join(directory, the_file)
# Only remove files that are on the watch list
if not match_path(file_path,
included_patterns=self.config.file_patterns,
excluded_patterns=self.config.ignore_patterns,
case_sensitive=True):
continue
# Just in case the user did not add .git to their list of ignore
if the_file.startswith(".git"):
continue
try:
if os.path.isfile(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
print(e)
return
def remove_local_files (self, directory):
for the_file in os.listdir(directory):
file_path = os.path.join(directory, the_file)
# Only remove files that are on the watch list
if not match_path(file_path,
included_patterns=self.config.file_patterns,
excluded_patterns=self.config.ignore_patterns,
case_sensitive=True):
continue
# Just in case the user did not add .git to their list of ignore
if the_file.startswith(".git"):
continue
try:
if os.path.isfile(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
print(e)
return
def match(path):
return match_path(path,
included_patterns=self.patterns,
excluded_patterns=self.ignore_patterns,
case_sensitive=self.case_sensitive)
def reset_remote (self):
print(Back.RED + Fore.WHITE + 'Are you sure you want to permanently overwrite your remote theme with the contents of ' + self.config.watch_dir + ' ?' + Style.RESET_ALL)
print(Fore.RED + 'Type [Y] to overwrite your remote theme or [q] to quit. Any other key will result in no action being taken.' + Style.RESET_ALL)
if version_info[0] > 2:
response = input(": ")
else:
response = raw_input(": ")
if not response == "Y":
sys.exit(0)
self.remove_remote_files()
# Get the filenames we about to push to s3
uploads = []
keynames = []
for (source, dirname, filenames) in os.walk(self.config.watch_dir):
for name in filenames:
uploads.append(os.path.join(source, name))
# Upload the new files
for filename in uploads:
keypath = filename.replace(self.config.watch_dir, '')
keyname = "/".join([self.connection["store"], "themes", self.connection["theme"], keypath.replace('\\', '/')])
# Only remove files that are on the watch list
if not match_path(filename,
included_patterns=self.config.file_patterns,
excluded_patterns=self.config.ignore_patterns,
case_sensitive=True):
continue
expires = int(time.time())
headers = {
'Cache-Control': "max-age=" + str(expires) + ", public",
'Expires': expires
}
try:
k = self.connection["bucket"].new_key(keyname)
k.set_contents_from_filename(filename, headers=headers)
keynames.append(keypath.replace('\\', '/'))
print(Fore.GREEN + '[' + time.strftime("%c") + '] Successfully uploaded ' + keypath + Style.RESET_ALL)
except:
print(Fore.RED + '[' + time.strftime("%c") + '] Failed to upload ' + keypath + Style.RESET_ALL)
# Notify LS2 that the files have changed
try:
data = { 'keys': keynames }
# Update the resource with LemonStand
res = requests.put(
self.config.api_host + '/api/v2/resource/touch',
headers = {
'Content-Type': 'application/json',
'Authorization': 'Bearer ' + self.config.api_access
},
data=json.dumps(data),
allow_redirects=False,
verify=False
)
if res.status_code != 200:
raise Exception()
except:
print(Fore.RED + '[' + time.strftime("%c") + '] Failed to register local files with LemonStand!' + Style.RESET_ALL)
return
def reset_remote(self):
print(
Back.RED + Fore.WHITE +
'Are you sure you want to permanently overwrite your remote theme with the contents of '
+ self.config.watch_dir + ' ?' + Style.RESET_ALL)
print(
Fore.RED +
'Type [Y] to overwrite your remote theme or [q] to quit. Any other key will result in no action being taken.'
+ Style.RESET_ALL)
if version_info[0] > 2:
response = input(": ")
else:
response = raw_input(": ")
if not response == "Y":
sys.exit(0)
self.remove_remote_files()
# Get the filenames we about to push to s3
uploads = []
keynames = []
for (source, dirname, filenames) in os.walk(self.config.watch_dir):
for name in filenames:
uploads.append(os.path.join(source, name))
# Upload the new files
for filename in uploads:
keypath = filename.replace(self.config.watch_dir, '')
keyname = "/".join([
self.connection["store"], "themes", self.connection["theme"],
keypath.replace('\\', '/')
])
# Only remove files that are on the watch list
if not match_path(filename,
included_patterns=self.config.file_patterns,
excluded_patterns=self.config.ignore_patterns,
case_sensitive=True):
continue
expires = int(time.time())
headers = {
'Cache-Control': "max-age=" + str(expires) + ", public",
'Expires': expires
}
try:
k = self.connection["bucket"].new_key(keyname)
k.set_contents_from_filename(filename, headers=headers)
keynames.append(keypath.replace('\\', '/'))
print(Fore.GREEN + '[' + time.strftime("%c") +
'] Successfully uploaded ' + keypath + Style.RESET_ALL)
except:
print(Fore.RED + '[' + time.strftime("%c") +
'] Failed to upload ' + keypath + Style.RESET_ALL)
# Notify LS2 that the files have changed
try:
data = {'keys': keynames}
# Update the resource with LemonStand
res = requests.put(self.config.api_host + '/api/v2/resource/touch',
headers={
'Content-Type':
'application/json',
'Authorization':
'Bearer ' + self.config.api_access
},
data=json.dumps(data),
allow_redirects=False,
verify=False)
if res.status_code != 200:
raise Exception()
except:
print(Fore.RED + '[' + time.strftime("%c") +
'] Failed to register local files with LemonStand!' +
Style.RESET_ALL)
return
def push(chat, patterns):
with Lock(open(JOURNAL, 'a+')) as journal:
if journal.tell() == 0:
init(journal, chat)
files = load(journal, chat, patterns)
queue = Queue()
handler = Handler(queue, patterns)
observer = Observer()
done = False
def interrupt(signum, frame):
nonlocal done
done = True
queue.put((CREATE, JOURNAL)) # dummy iteration
sigint = None
observer.schedule(handler, '.', recursive=False)
try:
observer.start()
actual = dict()
mtimes = dict()
for entry in scandir():
if (not entry.is_file() or not match_path(
entry.name, patterns, case_sensitive=True)
or entry.name == JOURNAL):
continue
mtimes[entry.name] = entry.stat().st_mtime_ns
actual[entry.name] = readdigest(entry.name)
oldnames = set(files.keys())
newnames = set(actual.keys())
old = oldnames - newnames
new = newnames - oldnames
for name in oldnames & newnames:
if files[name].digest != actual[name]:
old.add(name)
new.add(name)
# order doesn't matter for old
new = sorted(new, key=mtimes.__getitem__, reverse=True)
sigint = signal(SIGINT, interrupt)
while not done:
if old:
type, name = DELETE, old.pop()
elif new:
type, name = CREATE, new.pop()
else:
type, name = queue.get()
name = basename(name)
if name == JOURNAL:
continue
if name in files:
save(journal, name, None)
delete(chat, name, files[name])
del files[name]
if type is CREATE:
file = create(chat, name)
save(journal, name, file)
if file is not None:
files[name] = file
finally:
observer.stop()
if sigint is not None:
signal(SIGINT, sigint)
observer.join()
sigint(SIGINT, currentframe())
