def processfile(pathfile):
global total_size
global FileCount
global target
global FilesCount
global log
log = open("output.log", "a")
if accessories.IsIgnoredFile(pathfile):
return
file = accessories.File(pathfile)
FileCount += 1
#print(file.HashTime)
if not file.AllreadyInDB():
CreationDate = file.creationDate
if file.Make == "Other" and file.Model == "Other":
Camera = "Other"
elif file.Make == "Other" and file.Model != "Other":
Camera = file.Model
elif file.Make != "Other" and file.Model == "Other":
Camera = file.Make
elif str(file.Make) in str(file.Model):
Camera = file.Model
else:
Camera = str(file.Make) + " " + str(file.Model)
FileDestination = os.path.join(target, CreationDate[0],
CreationDate[1], CreationDate[2],
Camera)
if accessories.CreateDestionation(FileDestination):
success, error, time = file.CopyTo(FileDestination)
if not success:
print("%s : %s" % (error, file.Filename))
log.write("%s : %s" % (error, file.Filename))
file.WriteErrorToDB("%s : %s" % (error, file.Filename))
else:
print("%s/%s, %s, %s, %s, %s, %s, %s" %
(FileCount, FilesCount, file.Filename, file.Make,
file.Model, round(file.Size / 1024 / 1024,
2), round(file.HashTime, 2),
round((file.Size / 1024 / 1024) / time, 2)))
log.write("%s/%s, %s, %s, %s, %s, %s, %s\n" %
(FileCount, FilesCount, file.Filename, file.Make,
file.Model, round(file.Size / 1024 / 1024,
2), round(file.HashTime, 2),
round((file.Size / 1024 / 1024) / time, 2)))
file.WriteFiletoDB()
else:
print("Could not create destination %s" % (FileDestination))
log.write("Could not create destination %s\n" % (FileDestination))
file.WriteErrorToDB("Could not create destination %s\n" %
(FileDestination))
else:
print("Allready in database %s" % str(file.Filename))
log.write("Allready in database %s\n" % str(file.Filename))
file.WriteErrorToDB("Allready in database %s" % str(file.Filename))
def processfile(pathfile):
log = open("output.log", "a")
ProcessStart = datetime.now()
with job_lock:
global ProcessedFiles
LocalProcessedFiles = ProcessedFiles
LocalProcessedFiles += 1
ProcessedFiles = LocalProcessedFiles
file = accessories.File(pathfile)
with job_lock:
FileInDB = file.AllreadyInDB(mydb)
if not FileInDB:
CreationDate = file.creationDate
if file.Make == "Other" and file.Model == "Other":
Camera = "Other"
elif file.Make == "Other" and file.Model != "Other":
Camera = file.Model
elif file.Make != "Other" and file.Model == "Other":
Camera = file.Make
elif str(file.Make) in str(file.Model):
Camera = file.Model
else:
Camera = str(file.Make) + " " + str(file.Model)
FileDestination = os.path.join(target, CreationDate[0],
CreationDate[1], CreationDate[2],
Camera)
if accessories.CreateDestionation(FileDestination):
newFileName = Path(file.Filename)
#print(FileDestination)
while Path.exists(newFileName):
#print("before : %s" % newFileName)
fullpath, filename = os.path.split(newFileName)
currentfile = Path(filename)
nameNoSuffix = currentfile.with_suffix('')
namewithSuffix = str(
nameNoSuffix) + "-dup" + currentfile.suffix
newFileName = os.path.join(FileDestination, namewithSuffix)
newFileName = Path(newFileName)
#print("after : %s" % newFileName)
success, error, time = file.CopyTo(newFileName)
if not success:
print("%s/%s %s : %s : %s" %
(LocalProcessedFiles, FilesCount,
threading.current_thread().name, error, file.Filename))
log.write(
"%s/%s %s : %s : %s\n" %
(LocalProcessedFiles, FilesCount,
threading.current_thread().name, error, file.Filename))
with job_lock:
file.WriteErrorToDB(("%s : %s" % (error, file.Filename)),
mydb)
else:
print(
"%s/%s %s : %s, %s, %s, %s, %s, %s" %
(LocalProcessedFiles, FilesCount,
threading.current_thread().name, file.Filename, file.Make,
file.Model, round(file.Size / 1024 / 1024,
2), round(file.HashTime, 2),
round((file.Size / 1024 / 1024) / time, 2)))
log.write(
"%s/%s, %s, %s, %s, %s, %s, %s\n" %
(LocalProcessedFiles, FilesCount, file.Filename, file.Make,
file.Model, round(file.Size / 1024 / 1024,
2), round(file.HashTime, 2),
round((file.Size / 1024 / 1024) / time, 2)))
with job_lock:
file.WriteFiletoDB(mydb)
else:
print("Could not create destination %s" % (FileDestination))
log.write("Could not create destination %s\n" % (FileDestination))
with job_lock:
file.WriteErrorToDB(
("Could not create destination %s\n" % (FileDestination)),
mydb)
else:
with job_lock:
file.WriteErrorToDB(
("Allready in database %s" % str(file.Filename)), mydb)
ProcessingTime = (datetime.now() - ProcessStart).total_seconds()
print("%s/%s %s : Allready in database %s, %s, %s" %
(LocalProcessedFiles, FilesCount,
threading.current_thread().name, str(file.Filename),
round(file.HashTime, 2), round(ProcessingTime, 2)))
log.write("Allready in database %s\n" % str(file.Filename))
#file ="/mnt/target/2006/12/08/CYBERSHOT/DSC08820.JPG"
#fileobj = accessories.File(file)
print("-----------------------------------")
#print(accessories.GetFileDate(file))
#print(accessories.GetCameraModel(file))
#print(accessories.GetGPS(file))
#accessories.CreateDestionation("/mnt/source/USA 2015/gps3/1/2/3")
#accessories.CreateDestionation("/mnt/source/USA 2015/gps2/2015/09/08")
#print(os.path.exists("/mnt/source/USA 2015/gps/IMG_1909.JPG"))
for path, dirs, files in os.walk(source):
for filename in files:
pathfile = os.path.join(path, filename)
file = accessories.File(pathfile)
print(file.HashTime)
if not file.AllreadyInDB():
CreationDate = file.creationDate
FileDestination = os.path.join(destination, CreationDate[0],
CreationDate[1], CreationDate[2])
if accessories.CreateDestionation(FileDestination):
success, error, time = file.CopyTo(FileDestination)
if not success:
print(error)
else:
print((file.Size / 1024 / 1024) / time)
file.WriteFiletoDB()
else:
print("Could not create destination %s" % (FileDestination))
else:
print("Allready in database %s" % str(file.Filename))
# with exiftool.ExifTool() as et:
# #metadata = et.get_metadata(source)
# metadata = et.get_tag("QuickTime:ContentDistributor", source)
# print("%s" % (metadata))
Camera = accessories.GetCameraModel(source)
destination = DestinatioPath
#print("copy %s to %s" % (source, destination))
#ExifDate = accessories.GetExifTagData(file,'EXIF DateTimeOriginal')
FileDate = accessories.GetFileDate(source)
#print("Date %s %s %s" % (FileDate[0], FileDate[1], FileDate[2]))
destinationDir = os.path.join(destination, FileDate[0],
FileDate[1], FileDate[2],
str(Camera))
#print(destinationDir)
accessories.CreateDestionation(destinationDir)
copy_time = time.time()
print("Copy %s to %s" % (source, destinationDir))
try:
shutil.copy2(source, destinationDir)
# If source and destination are same
except shutil.SameFileError:
print("Source and destination represents the same file.")
# If there is any permission issue
except PermissionError:
print("Permission denied.")
# For other errors
except:
print("Error occurred while copying file.")
CopyElapsedTime = time.time() - copy_time
FileSizeMB = FileSize / 1024 / 1024
def processfile(file):
global total_size, FileCount, log
Reason = "unknown"
mydb = pymysql.connect(host="localhost",
user="******",
passwd="password",
database="photosynk")
cursor = mydb.cursor()
if file[0] != ".": #ignore files starting with a '.'
start_time = time.time()
(mode, ino, dev, nlink, uid, gid, size, atime, mtime,
ctime) = os.stat(file)
starthashtime = time.time()
FileHash = hash.md5sum_full(file)
hashtime = time.time() - starthashtime
sql_select_query = """select * from Files where Hash = %s"""
startquery1time = time.time()
cursor.execute(sql_select_query, (FileHash.hexdigest(), ))
record = cursor.fetchall()
FilesFoundCount = cursor.rowcount
query1time = time.time() - startquery1time
FileSize = os.stat(file).st_size
total_size += FileSize
FileCount += 1
startexiftime = time.time()
CameraModel = accessories.GetExifTagData(file, 'Image Model')
exiftime = time.time() - startexiftime
PercentageProgress = (FileCount * 100) / FilesCount
#if FreeSpace > FileSize:
# #print("Enough space on destination storage, file can be copied")
#else:
# #print("Not enough space on destination storage, file should NOT be copied")
if FilesFoundCount > 0:
#print("%s File %s allready in database should not be copied" % (PercentageProgress, file))
Reason = "File allready in database"
fullpath, filename = os.path.split("/tmp/d/a.dat")
if os.path.exists(
file
) and FilesFoundCount == 0 and filename not in accessories.IgnoredFiles(
):
mydict = {
"filename": file,
"Hash": FileHash.hexdigest(),
"Camera": str(CameraModel),
"Created": str(time.ctime(mtime))
}
startquery2time = time.time()
sql = "INSERT INTO Files (Camera, Hash, FileName) VALUES (%s, %s, %s)"
val = (str(CameraModel), FileHash.hexdigest(), file)
cursor.execute(sql, val)
mydb.commit()
query2time = time.time() - startquery2time
source = file
Camera = accessories.GetCameraModel(source)
destination = DestinatioPath
FileDate = accessories.GetFileDate(source)
destinationDir = os.path.join(destination, FileDate[0],
FileDate[1], FileDate[2],
str(Camera))
accessories.CreateDestionation(destinationDir)
copy_time = time.time()
try:
shutil.copy2(source, destinationDir)
except shutil.SameFileError:
print("Source and destination represents the same file.")
log.write("Source and destination represents the same file.")
except PermissionError:
print("Permission denied.")
log.write("Permission denied.")
except:
print("Error occurred while copying file.")
log.write("Error occurred while copying file.")
CopyElapsedTime = time.time() - copy_time
FileSizeMB = FileSize / 1024 / 1024
CopySpeed = FileSizeMB / CopyElapsedTime
ElapsedTime = time.time() - start_time
print(
"%s, %s, %s, %s, %s, %s, %sMB, %s, %s, %s, %s, %s %s" %
(round(PercentageProgress), FileCount, FilesCount, file,
CameraModel, FileHash.hexdigest(),
round(FileSize / 1024 / 1024, 2), round(ElapsedTime, 2),
round(exiftime, 2), round(hashtime, 2), round(query1time, 2),
round(query2time, 2), round(CopyElapsedTime, 2)))
log.write(
"%s, %s, %s, %s, %s, %s, %sMB, %s, %s, %s, %s, %s %s" %
(round(PercentageProgress), FileCount, FilesCount, file,
CameraModel, FileHash.hexdigest(),
round(FileSize / 1024 / 1024, 2), round(ElapsedTime, 2),
round(exiftime, 2), round(hashtime, 2), round(query1time, 2),
round(query2time, 2), round(CopyElapsedTime, 2)))
else:
mydict = {
"filename": file,
"Hash": FileHash.hexdigest(),
"Camera": str(CameraModel),
"Created": str(time.ctime(mtime)),
"Error": Reason
}
sql = "INSERT INTO Errors (Camera, Hash, FileName, Error) VALUES (%s, %s, %s, %s)"
val = (str(CameraModel), FileHash.hexdigest(), file, Reason)
cursor.execute(sql, val)
mydb.commit()