def processfile(pathfile): global total_size global FileCount global target global FilesCount global log log = open("output.log", "a") if accessories.IsIgnoredFile(pathfile): return file = accessories.File(pathfile) FileCount += 1 #print(file.HashTime) if not file.AllreadyInDB(): CreationDate = file.creationDate if file.Make == "Other" and file.Model == "Other": Camera = "Other" elif file.Make == "Other" and file.Model != "Other": Camera = file.Model elif file.Make != "Other" and file.Model == "Other": Camera = file.Make elif str(file.Make) in str(file.Model): Camera = file.Model else: Camera = str(file.Make) + " " + str(file.Model) FileDestination = os.path.join(target, CreationDate[0], CreationDate[1], CreationDate[2], Camera) if accessories.CreateDestionation(FileDestination): success, error, time = file.CopyTo(FileDestination) if not success: print("%s : %s" % (error, file.Filename)) log.write("%s : %s" % (error, file.Filename)) file.WriteErrorToDB("%s : %s" % (error, file.Filename)) else: print("%s/%s, %s, %s, %s, %s, %s, %s" % (FileCount, FilesCount, file.Filename, file.Make, file.Model, round(file.Size / 1024 / 1024, 2), round(file.HashTime, 2), round((file.Size / 1024 / 1024) / time, 2))) log.write("%s/%s, %s, %s, %s, %s, %s, %s\n" % (FileCount, FilesCount, file.Filename, file.Make, file.Model, round(file.Size / 1024 / 1024, 2), round(file.HashTime, 2), round((file.Size / 1024 / 1024) / time, 2))) file.WriteFiletoDB() else: print("Could not create destination %s" % (FileDestination)) log.write("Could not create destination %s\n" % (FileDestination)) file.WriteErrorToDB("Could not create destination %s\n" % (FileDestination)) else: print("Allready in database %s" % str(file.Filename)) log.write("Allready in database %s\n" % str(file.Filename)) file.WriteErrorToDB("Allready in database %s" % str(file.Filename))
def processfile(pathfile): log = open("output.log", "a") ProcessStart = datetime.now() with job_lock: global ProcessedFiles LocalProcessedFiles = ProcessedFiles LocalProcessedFiles += 1 ProcessedFiles = LocalProcessedFiles file = accessories.File(pathfile) with job_lock: FileInDB = file.AllreadyInDB(mydb) if not FileInDB: CreationDate = file.creationDate if file.Make == "Other" and file.Model == "Other": Camera = "Other" elif file.Make == "Other" and file.Model != "Other": Camera = file.Model elif file.Make != "Other" and file.Model == "Other": Camera = file.Make elif str(file.Make) in str(file.Model): Camera = file.Model else: Camera = str(file.Make) + " " + str(file.Model) FileDestination = os.path.join(target, CreationDate[0], CreationDate[1], CreationDate[2], Camera) if accessories.CreateDestionation(FileDestination): newFileName = Path(file.Filename) #print(FileDestination) while Path.exists(newFileName): #print("before : %s" % newFileName) fullpath, filename = os.path.split(newFileName) currentfile = Path(filename) nameNoSuffix = currentfile.with_suffix('') namewithSuffix = str( nameNoSuffix) + "-dup" + currentfile.suffix newFileName = os.path.join(FileDestination, namewithSuffix) newFileName = Path(newFileName) #print("after : %s" % newFileName) success, error, time = file.CopyTo(newFileName) if not success: print("%s/%s %s : %s : %s" % (LocalProcessedFiles, FilesCount, threading.current_thread().name, error, file.Filename)) log.write( "%s/%s %s : %s : %s\n" % (LocalProcessedFiles, FilesCount, threading.current_thread().name, error, file.Filename)) with job_lock: file.WriteErrorToDB(("%s : %s" % (error, file.Filename)), mydb) else: print( "%s/%s %s : %s, %s, %s, %s, %s, %s" % (LocalProcessedFiles, FilesCount, threading.current_thread().name, file.Filename, file.Make, file.Model, round(file.Size / 1024 / 1024, 2), round(file.HashTime, 2), round((file.Size / 1024 / 1024) / time, 2))) log.write( "%s/%s, %s, %s, %s, %s, %s, %s\n" % (LocalProcessedFiles, FilesCount, file.Filename, file.Make, file.Model, round(file.Size / 1024 / 1024, 2), round(file.HashTime, 2), round((file.Size / 1024 / 1024) / time, 2))) with job_lock: file.WriteFiletoDB(mydb) else: print("Could not create destination %s" % (FileDestination)) log.write("Could not create destination %s\n" % (FileDestination)) with job_lock: file.WriteErrorToDB( ("Could not create destination %s\n" % (FileDestination)), mydb) else: with job_lock: file.WriteErrorToDB( ("Allready in database %s" % str(file.Filename)), mydb) ProcessingTime = (datetime.now() - ProcessStart).total_seconds() print("%s/%s %s : Allready in database %s, %s, %s" % (LocalProcessedFiles, FilesCount, threading.current_thread().name, str(file.Filename), round(file.HashTime, 2), round(ProcessingTime, 2))) log.write("Allready in database %s\n" % str(file.Filename))
#file ="/mnt/target/2006/12/08/CYBERSHOT/DSC08820.JPG" #fileobj = accessories.File(file) print("-----------------------------------") #print(accessories.GetFileDate(file)) #print(accessories.GetCameraModel(file)) #print(accessories.GetGPS(file)) #accessories.CreateDestionation("/mnt/source/USA 2015/gps3/1/2/3") #accessories.CreateDestionation("/mnt/source/USA 2015/gps2/2015/09/08") #print(os.path.exists("/mnt/source/USA 2015/gps/IMG_1909.JPG")) for path, dirs, files in os.walk(source): for filename in files: pathfile = os.path.join(path, filename) file = accessories.File(pathfile) print(file.HashTime) if not file.AllreadyInDB(): CreationDate = file.creationDate FileDestination = os.path.join(destination, CreationDate[0], CreationDate[1], CreationDate[2]) if accessories.CreateDestionation(FileDestination): success, error, time = file.CopyTo(FileDestination) if not success: print(error) else: print((file.Size / 1024 / 1024) / time) file.WriteFiletoDB() else: print("Could not create destination %s" % (FileDestination)) else: print("Allready in database %s" % str(file.Filename))
# with exiftool.ExifTool() as et: # #metadata = et.get_metadata(source) # metadata = et.get_tag("QuickTime:ContentDistributor", source) # print("%s" % (metadata)) Camera = accessories.GetCameraModel(source) destination = DestinatioPath #print("copy %s to %s" % (source, destination)) #ExifDate = accessories.GetExifTagData(file,'EXIF DateTimeOriginal') FileDate = accessories.GetFileDate(source) #print("Date %s %s %s" % (FileDate[0], FileDate[1], FileDate[2])) destinationDir = os.path.join(destination, FileDate[0], FileDate[1], FileDate[2], str(Camera)) #print(destinationDir) accessories.CreateDestionation(destinationDir) copy_time = time.time() print("Copy %s to %s" % (source, destinationDir)) try: shutil.copy2(source, destinationDir) # If source and destination are same except shutil.SameFileError: print("Source and destination represents the same file.") # If there is any permission issue except PermissionError: print("Permission denied.") # For other errors except: print("Error occurred while copying file.") CopyElapsedTime = time.time() - copy_time FileSizeMB = FileSize / 1024 / 1024
def processfile(file): global total_size, FileCount, log Reason = "unknown" mydb = pymysql.connect(host="localhost", user="******", passwd="password", database="photosynk") cursor = mydb.cursor() if file[0] != ".": #ignore files starting with a '.' start_time = time.time() (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(file) starthashtime = time.time() FileHash = hash.md5sum_full(file) hashtime = time.time() - starthashtime sql_select_query = """select * from Files where Hash = %s""" startquery1time = time.time() cursor.execute(sql_select_query, (FileHash.hexdigest(), )) record = cursor.fetchall() FilesFoundCount = cursor.rowcount query1time = time.time() - startquery1time FileSize = os.stat(file).st_size total_size += FileSize FileCount += 1 startexiftime = time.time() CameraModel = accessories.GetExifTagData(file, 'Image Model') exiftime = time.time() - startexiftime PercentageProgress = (FileCount * 100) / FilesCount #if FreeSpace > FileSize: # #print("Enough space on destination storage, file can be copied") #else: # #print("Not enough space on destination storage, file should NOT be copied") if FilesFoundCount > 0: #print("%s File %s allready in database should not be copied" % (PercentageProgress, file)) Reason = "File allready in database" fullpath, filename = os.path.split("/tmp/d/a.dat") if os.path.exists( file ) and FilesFoundCount == 0 and filename not in accessories.IgnoredFiles( ): mydict = { "filename": file, "Hash": FileHash.hexdigest(), "Camera": str(CameraModel), "Created": str(time.ctime(mtime)) } startquery2time = time.time() sql = "INSERT INTO Files (Camera, Hash, FileName) VALUES (%s, %s, %s)" val = (str(CameraModel), FileHash.hexdigest(), file) cursor.execute(sql, val) mydb.commit() query2time = time.time() - startquery2time source = file Camera = accessories.GetCameraModel(source) destination = DestinatioPath FileDate = accessories.GetFileDate(source) destinationDir = os.path.join(destination, FileDate[0], FileDate[1], FileDate[2], str(Camera)) accessories.CreateDestionation(destinationDir) copy_time = time.time() try: shutil.copy2(source, destinationDir) except shutil.SameFileError: print("Source and destination represents the same file.") log.write("Source and destination represents the same file.") except PermissionError: print("Permission denied.") log.write("Permission denied.") except: print("Error occurred while copying file.") log.write("Error occurred while copying file.") CopyElapsedTime = time.time() - copy_time FileSizeMB = FileSize / 1024 / 1024 CopySpeed = FileSizeMB / CopyElapsedTime ElapsedTime = time.time() - start_time print( "%s, %s, %s, %s, %s, %s, %sMB, %s, %s, %s, %s, %s %s" % (round(PercentageProgress), FileCount, FilesCount, file, CameraModel, FileHash.hexdigest(), round(FileSize / 1024 / 1024, 2), round(ElapsedTime, 2), round(exiftime, 2), round(hashtime, 2), round(query1time, 2), round(query2time, 2), round(CopyElapsedTime, 2))) log.write( "%s, %s, %s, %s, %s, %s, %sMB, %s, %s, %s, %s, %s %s" % (round(PercentageProgress), FileCount, FilesCount, file, CameraModel, FileHash.hexdigest(), round(FileSize / 1024 / 1024, 2), round(ElapsedTime, 2), round(exiftime, 2), round(hashtime, 2), round(query1time, 2), round(query2time, 2), round(CopyElapsedTime, 2))) else: mydict = { "filename": file, "Hash": FileHash.hexdigest(), "Camera": str(CameraModel), "Created": str(time.ctime(mtime)), "Error": Reason } sql = "INSERT INTO Errors (Camera, Hash, FileName, Error) VALUES (%s, %s, %s, %s)" val = (str(CameraModel), FileHash.hexdigest(), file, Reason) cursor.execute(sql, val) mydb.commit()