def scan_photos(user, job_id):
if LongRunningJob.objects.filter(job_id=job_id).exists():
lrj = LongRunningJob.objects.get(job_id=job_id)
lrj.started_at = datetime.datetime.now().replace(tzinfo=pytz.utc)
lrj.save()
else:
lrj = LongRunningJob.objects.create(
started_by=user,
job_id=job_id,
queued_at=datetime.datetime.now().replace(tzinfo=pytz.utc),
started_at=datetime.datetime.now().replace(tzinfo=pytz.utc),
job_type=LongRunningJob.JOB_SCAN_PHOTOS,
)
lrj.save()
photo_count_before = Photo.objects.count()
try:
fc = file_counter() # first walk and count sum of files
walk_directory(user.scan_directory, fc)
files_found = fc.counter
ps = photo_scanner(user, lrj, job_id, files_found)
walk_directory(user.scan_directory,
ps) # now walk with photo-scannning
util.logger.info("Scanned {} files in : {}".format(
files_found, user.scan_directory))
build_image_similarity_index(user)
except Exception:
util.logger.exception("An error occured:")
lrj.failed = True
added_photo_count = Photo.objects.count() - photo_count_before
util.logger.info("Added {} photos".format(added_photo_count))
lrj.finished = True
lrj.finished_at = datetime.datetime.now().replace(tzinfo=pytz.utc)
lrj.result["new_photo_count"] = added_photo_count
lrj.save()
return {
"new_photo_count": added_photo_count,
"status": lrj.failed == False
}
def scan_photos(user, job_id):
if LongRunningJob.objects.filter(job_id=job_id).exists():
lrj = LongRunningJob.objects.get(job_id=job_id)
lrj.started_at = datetime.datetime.now().replace(tzinfo=pytz.utc)
lrj.save()
else:
lrj = LongRunningJob.objects.create(
started_by=user,
job_id=job_id,
queued_at=datetime.datetime.now().replace(tzinfo=pytz.utc),
started_at=datetime.datetime.now().replace(tzinfo=pytz.utc),
job_type=LongRunningJob.JOB_SCAN_PHOTOS)
lrj.save()
nc = nextcloud.Client(user.nextcloud_server_address)
nc.login(user.nextcloud_username, user.nextcloud_app_password)
photos = []
image_paths = []
collect_photos(nc, user.nextcloud_scan_directory, photos)
for photo in photos:
local_dir = os.path.join(settings.DATA_ROOT, 'nextcloud_media',
user.username,
os.path.dirname(photo)[1:])
local_path = os.path.join(settings.DATA_ROOT, 'nextcloud_media',
user.username, photo[1:])
image_paths.append(local_path)
if not os.path.exists(local_dir):
pathlib.Path(local_dir).mkdir(parents=True, exist_ok=True)
if not os.path.exists(local_path):
nc.get_file(photo, local_path)
util.logger.info('Downloaded photo from nextcloud to ' + local_path)
try:
image_paths.sort()
existing_hashes = [p.image_hash for p in Photo.objects.all()]
image_paths_to_add = []
for image_path in image_paths:
if not Photo.objects.filter(image_path=image_path).exists():
image_paths_to_add.append(image_path)
added_photo_count = 0
already_existing_photo = 0
to_add_count = len(image_paths_to_add)
for idx, image_path in enumerate(image_paths_to_add):
util.logger.info('begin handling of photo %d/%d' %
(idx + 1, to_add_count))
handle_new_image(user, image_path, job_id)
lrj.result = {
'progress': {
"current": idx + 1,
"target": to_add_count
}
}
lrj.save()
util.logger.info("Added {} photos".format(len(image_paths_to_add)))
build_image_similarity_index(user)
lrj = LongRunningJob.objects.get(job_id=job_id)
lrj.finished = True
lrj.finished_at = datetime.datetime.now().replace(tzinfo=pytz.utc)
prev_result = lrj.result
next_result = prev_result
next_result['new_photo_count'] = added_photo_count
lrj.result = next_result
lrj.save()
except Exception as e:
util.logger.exception(str(e))
lrj = LongRunningJob.objects.get(job_id=job_id)
lrj.finished = True
lrj.failed = True
lrj.finished_at = datetime.datetime.now().replace(tzinfo=pytz.utc)
prev_result = lrj.result
next_result = prev_result
next_result['new_photo_count'] = 0
lrj.result = next_result
lrj.save()
return {"new_photo_count": added_photo_count, "status": True}
def scan_photos(user):
job_id = rq.get_current_job().id
if LongRunningJob.objects.filter(job_id=job_id).exists():
lrj = LongRunningJob.objects.get(job_id=job_id)
lrj.started_at = datetime.datetime.now().replace(tzinfo=pytz.utc)
lrj.save()
else:
lrj = LongRunningJob.objects.create(
started_by=user,
job_id=job_id,
queued_at=datetime.datetime.now().replace(tzinfo=pytz.utc),
started_at=datetime.datetime.now().replace(tzinfo=pytz.utc),
job_type=LongRunningJob.JOB_SCAN_PHOTOS)
lrj.save()
added_photo_count = 0
already_existing_photo = 0
try:
image_paths = []
image_paths.extend([
os.path.join(dp, f) for dp, dn, fn in os.walk(user.scan_directory)
for f in fn
])
image_paths = [
p for p in image_paths
if p.lower().endswith('.jpg') and 'thumb' not in p.lower()
]
image_paths.sort()
existing_hashes = [p.image_hash for p in Photo.objects.all()]
# Create a list with all images whose hash is new or they do not exist in the db
image_paths_to_add = []
for image_path in image_paths:
if not Photo.objects.filter(image_path=image_path).exists():
image_paths_to_add.append(image_path)
to_add_count = len(image_paths_to_add)
for idx, image_path in enumerate(image_paths_to_add):
handle_new_image(user, image_path, job_id)
lrj.result = {
'progress': {
"current": idx + 1,
"target": to_add_count
}
}
lrj.save()
'''
image_paths_to_add = Parallel(n_jobs=multiprocessing.cpu_count(), backend="multiprocessing")(delayed(is_new_image)(existing_hashes, image_path) for image_path in tqdm(image_paths))
image_paths_to_add = filter(None, image_paths_to_add)
Parallel(n_jobs=multiprocessing.cpu_count(), backend="multiprocessing")(delayed(handle_new_image)(user, image_path) for image_path in tqdm(image_paths_to_add))
'''
util.logger.info("Added {} photos".format(len(image_paths_to_add)))
build_image_similarity_index(user)
lrj = LongRunningJob.objects.get(job_id=rq.get_current_job().id)
lrj.finished = True
lrj.finished_at = datetime.datetime.now().replace(tzinfo=pytz.utc)
prev_result = lrj.result
next_result = prev_result
next_result['new_photo_count'] = added_photo_count
lrj.result = next_result
lrj.save()
except Exception as e:
util.logger.error(str(e))
lrj = LongRunningJob.objects.get(job_id=rq.get_current_job().id)
lrj.finished = True
lrj.failed = True
lrj.finished_at = datetime.datetime.now().replace(tzinfo=pytz.utc)
prev_result = lrj.result
next_result = prev_result
next_result['new_photo_count'] = 0
lrj.result = next_result
lrj.save()
return {"new_photo_count": added_photo_count, "status": True}
def handle(self, *args, **kwargs):
for user in User.objects.all():
build_image_similarity_index(user)