def detect_changed_files(self, force_reimport):
"""
Walk through all entries and verify if files are changed.
self.model is expected to have File foreign key called file.
Return tuple of unmodified, modified and removed files.
Returned value representation:
- unmodified files: set of absolute paths,
- modified files: {absolute_path: digest},
- removed files: {digest: absolute_path}
"""
unmodified = set()
modified = {}
removed = {}
# iterate through all files of self.model
# is JOIN without WHERE clause possible with django queries?
cond = {self.model._meta.object_name.lower() + '__pk__isnull': False}
for file in models.File.objects.filter(**cond):
if os.path.exists(file.path):
digest = utils.calc_digest(file.path)
if digest != file.digest or force_reimport:
modified[file.path] = digest
else:
unmodified.add(file.path)
else:
removed[file.digest] = file.path
return unmodified, modified, removed
def detect_changed_files(self, force_reimport):
"""
Walk through all entries and verify if files are changed.
self.model is expected to have File foreign key called file.
Return tuple of unmodified, modified and removed files.
Returned value representation:
- unmodified files: set of absolute paths,
- modified files: {absolute_path: digest},
- removed files: {digest: absolute_path}
"""
unmodified = set()
modified = {}
removed = {}
# iterate through all files of self.model
# is JOIN without WHERE clause possible with django queries?
cond = {self.model._meta.object_name.lower() + '__pk__isnull': False}
for file in models.File.objects.filter(**cond):
if os.path.exists(file.path):
digest = utils.calc_digest(file.path)
if digest != file.digest or force_reimport:
modified[file.path] = digest
else:
unmodified.add(file.path)
else:
removed[file.digest] = file.path
return unmodified, modified, removed
def import_all(self, force_reimport=False):
"""
Walk through base directory and populate database with entries.
"""
unmodified, modified, removed = \
self.detect_changed_files(force_reimport)
renamed = set() # files renamed among 'removed' files
n_new = n_skipped = 0 # new/skipped posts number
for root, dirs, files in os.walk(self.base_dir):
conf_path = os.path.join(root, self.conf_loader.filename)
if not os.path.exists(conf_path):
logger.info("no %s in %s, directory skipped",
self.conf_loader.filename, root)
continue
try:
conf = self.conf_loader.load(conf_path)
except ConfLoaderError, e:
logger.error(unicode(e))
logger.info("directory %s skipped", root)
continue
# search for all markdown files
for filename in filter(
lambda s: s.endswith('.markdown') or s.endswith('.md'),
files):
path = os.path.join(root, filename)
# skip untouched files
if path in unmodified:
continue
# reimport modified files
elif path in modified:
digest = modified[path]
# new file?
else:
digest = utils.calc_digest(path)
# file renamed
if digest in removed:
old_path = removed[digest]
self.rename_entry(old_path, path, conf)
renamed.add(digest)
logger.info("%s renamed to %s", old_path, path)
continue
# new file
else:
n_new += 1
# import a new file, or reimport existing file
ok = self.import_entries(path, digest, conf)
if not ok:
logger.info("%s skipped", path)
n_skipped += 1
else:
logger.info('%s imported', path)
def import_all(self, force_reimport=False):
"""
Walk through base directory and populate database with entries.
"""
unmodified, modified, removed = \
self.detect_changed_files(force_reimport)
renamed = set() # files renamed among 'removed' files
n_new = n_skipped = 0 # new/skipped posts number
for root, dirs, files in os.walk(self.base_dir):
conf_path = os.path.join(root, self.conf_loader.filename)
if not os.path.exists(conf_path):
logger.info("no %s in %s, directory skipped",
self.conf_loader.filename, root)
continue
try:
conf = self.conf_loader.load(conf_path)
except ConfLoaderError, e:
logger.error(unicode(e))
logger.info("directory %s skipped", root)
continue
# search for all markdown files
for filename in filter(lambda s: s.endswith('.markdown') or
s.endswith('.md'), files):
path = os.path.join(root, filename)
# skip untouched files
if path in unmodified:
continue
# reimport modified files
elif path in modified:
digest = modified[path]
# new file?
else:
digest = utils.calc_digest(path)
# file renamed
if digest in removed:
old_path = removed[digest]
self.rename_entry(old_path, path, conf)
renamed.add(digest)
logger.info("%s renamed to %s", old_path, path)
continue
# new file
else:
n_new += 1
# import a new file, or reimport existing file
ok = self.import_entries(path, digest, conf)
if not ok:
logger.info("%s skipped", path)
n_skipped += 1
else:
logger.info('%s imported', path)
