def findASDFiles(datadir, capture):
# find corresponding ASD dir
datestr = datetime.strftime(capture, "%Y-%m-%d")
pathASD = os.path.join(datadir, datestr, "ASD")
if not os.path.exists(pathASD):
return []
# find all capture time dirs
captureDirs = utility.findFiles(pathASD, mode=2)
captureDirs[:] = [
dir for dir in captureDirs
if utility.verifyDateTime(os.path.basename(dir), "%H.%M.%S")
]
if len(captureDirs) <= 0:
return []
# find an ASD capture time within small threshold of HDR capture time
pathCapture = None
for dir in captureDirs:
timestr = datestr + " " + os.path.basename(dir)
time = datetime.strptime(timestr, "%Y-%m-%d %H.%M.%S")
delta = (capture - time).total_seconds()
if abs(delta) <= common.CaptureEpsilon:
pathCapture = os.path.join(pathASD, os.path.basename(dir))
break
if not os.path.exists(pathCapture):
return []
# gather all .txt versions of ASD files taken at capture timestamp
return utility.findFiles(pathCapture, mode=1, ext=["txt"])
def ASDRenameFiles(args):
print("Renaming files in:\n" + args.directory)
# ensure directory exists
if (not os.path.exists(args.directory)):
return
# grab all capture timestamp directories
timeDirs = utility.findFiles(args.directory, mode=2)
if (len(timeDirs) <= 0):
print("No directories found at all.")
return
timeDirs[:] = [dir for dir in timeDirs if utility.verifyDateTime(os.path.basename(dir), "%H.%M.%S")]
if (len(timeDirs) <= 0):
print("No capture time directories found.")
return
# for each timestamp directory
for dir in timeDirs:
print(dir)
# grab all asd files
asdFiles = utility.findFiles(dir, mode=1, ext=["asd"])
if (len(asdFiles) <= 0):
print("No asd files found in this directory.")
return
# THIS ASSUMES EACH FILE IN THIS DIRECTORY IS FROM A DIFFERENT SAMPLING PATTERN LOCATION,
# AND THE ORDERING IS THE SAME AS THE CAPTURE PATTERN DEFINED ABOVE!
# THIS IS A MAJOR ASSUMPTION! IT IS BELIEVED TO BE CORRECT IN THE DATA I HAVE AT THE TIME OF WRITING THIS CODE,
# BUT PLEASE CONFIRM THIS BEFORE RUNNING THIS CODE ON YOUR OWN DATA!
# First, make sure the list is NATURAL sorted (for humans, like in Windows Explorer)
# We have to do this because someone may not have padded the filename digits.
# If the files are not natural sorted, THEY WILL BE RENAMED TO THE WRONG CAPTURE PATTERN LOCATIONS!
asdFiles.sort(key=utility.naturalSortKey)
# for the number of files in the sampling pattern
for idx in range(0, len(SamplingPattern)):
if (idx >= len(asdFiles)):
print("ASD file #" + str(idx+1) + " is beyond the capture pattern location count (" + str(len(SamplingPattern)) + ")")
break
# rename to format: ##_TTT.TT_PP.PPPP_.asd
oldName = os.path.basename(asdFiles[idx])
newName = '{0:02d}_{1:06.02f}_{2:07.04f}_.asd'.format(idx, SamplingPattern[idx][0], SamplingPattern[idx][1])
print("Rename: " + oldName + " to " + newName)
if (not args.readonly):
os.rename(asdFiles[idx], os.path.join(dir, newName))
if (os.path.exists(os.path.join(dir, oldName) + ".rad")):
old = oldName + ".rad"
new = newName + ".rad"
print("Rename: " + old + " to " + new)
if (not args.readonly):
os.rename(os.path.join(dir, old), os.path.join(dir, new))
if (os.path.exists(os.path.join(dir, oldName) + ".rad.txt")):
old = oldName + ".rad.txt"
new = newName + ".rad.txt"
print("Rename: " + old + " to " + new)
if (not args.readonly):
os.rename(os.path.join(dir, old), os.path.join(dir, new))
def ASDRenameDirs(args):
print("Renaming directories in:\n" + args.directory)
# ensure directory exists
if (not os.path.exists(args.directory)):
return
# grab all capture timestamp directories
timeDirs = utility.findFiles(args.directory, mode=2)
if (len(timeDirs) <= 0):
print("No directories found at all.")
return
timeDirs[:] = [dir for dir in timeDirs if utility.verifyDateTime(os.path.basename(dir), "%Y-%m-%d___%H.%M.%S")]
if (len(timeDirs) <= 0):
print("No capture timestamp directories found.")
return
# for each timestamp directory
for dir in timeDirs:
dirBaseOld = os.path.basename(dir)
dirBaseNew = str(datetime.strptime(dirBaseOld, "%Y-%m-%d___%H.%M.%S").time()) # we want 24h format
dirBaseNew = dirBaseNew.replace(':', '.') # folder names can't have colons
print("Rename: " + dirBaseOld + " to " + dirBaseNew)
dirNew = os.path.join(os.path.dirname(dir), dirBaseNew)
if (not args.readonly):
os.rename(dir, dirNew)
def HDRPostProcessPhotos(args):
print("Postprocess raw photos in:\n" + args.directory)
# ensure directory exists
if (not os.path.exists(args.directory)):
return
# grab all raw photos
photos = utility.findFiles(args.directory, mode=1, recursive=True, ext=["cr2"])
if (len(photos) <= 0):
print("No photos found in this directory.")
return
prevdir = os.path.dirname(photos[0])
print(prevdir)
for p in photos:
pName, pExt = os.path.basename(p).split(".")
pNameNew = pName + ".tiff"
pNew = os.path.join(os.path.dirname(p), pNameNew)
if (os.path.dirname(p) != prevdir):
prevdir = os.path.dirname(p)
print(prevdir)
print("Postprocess " + p)
if (not args.readonly):
with rawpy.imread(p) as raw:
rgb = raw.postprocess(no_auto_bright=True, user_wb=raw.camera_whitebalance, gamma=(1, 1))
imageio.imsave(pNew, rgb)
def HDRRotatePhotos(args):
print("Rotating photos in:\n" + args.directory)
# ensure directory exists
if (not os.path.exists(args.directory)):
return
args.hdrextension = args.hdrextension.lower()
# grab all photos
photos = utility.findFiles(args.directory, mode=1, recursive=True, ext=[args.hdrextension])
if (len(photos) <= 0):
print("No photos found in this directory.")
return
for p in photos:
print("Rotate (" + str(args.hdrrotate) + "°): " + p)
if (not args.readonly):
img = Image.open(p)
img2 = img.rotate(args.hdrrotate)
dpi = img.info.get('dpi')
if (args.hdrextension == 'jpg'):
exif = img.info['exif']
img2.save(p, dpi=dpi, subsampling=-1, quality=100, exif=exif)
elif (args.hdrextension == 'tiff'):
img2.save(p, dpi=dpi)
img2.close()
img.close()
def HDRRenameFilesCounter(args):
print("Renaming photos starting from counter " + str(args.counter) + " in:\n" + args.directory)
# ensure directory exists
if (not os.path.exists(args.directory)):
return
# grab all photos
photos = utility.findFiles(args.directory, mode=1, ext=["jpg", "cr2"])
if (len(photos) <= 0):
print("No photos found in this directory.")
return
renameMap = {}
counter = args.counter
# for all photos
for p in photos:
pName, pExt = os.path.basename(p).split(".")
newName = "IMG_"
if pName not in renameMap.keys():
renameMap[pName] = str(counter).zfill(4)
newName += renameMap[pName]
counter += 1
else:
newName += renameMap[pName]
newName += "." + pExt
print("Rename: " + pName + "." + pExt + " to " + newName)
if (not args.readonly):
os.rename(p, os.path.join(os.path.dirname(p), newName))
def OffsetCaptureTimes(args):
print("Offsetting all capture time directories in:\n" + args.directory)
# ensure directory exists
if (not os.path.exists(args.directory)):
return
# grab all capture timestamp directories
timeDirs = utility.findFiles(args.directory, mode=2)
if (len(timeDirs) <= 0):
print("No directories found at all.")
return
timeDirs[:] = [dir for dir in timeDirs if utility.verifyDateTime(os.path.basename(dir), "%H.%M.%S")]
if (len(timeDirs) <= 0):
print("No capture time directories found.")
return
# prevent collisions between directories with the same name
# sort directories ascending when timeoffset is negative
if args.timeoffset < 0:
timeDirs.sort()
# sort directories descending when timeoffset is positive
elif args.timeoffset > 0:
timeDirs.sort(reverse=True)
# for each timestamp directory
for dir in timeDirs:
oldname = os.path.basename(dir)
oldtime = datetime.strptime(oldname, "%H.%M.%S")
newtime = oldtime + timedelta(hours=args.timeoffset)
newname = str(newtime.time()).replace(':', '.') # folder names can't have colons
print("Rename: " + oldname + " to " + newname)
if (not args.readonly):
os.rename(dir, os.path.join(os.path.dirname(dir), newname))
def main():
# handle command line args
parser = argparse.ArgumentParser(description='Script to find and delete files of a specified extension. Use -r (--readonly) flag to preview changes first.', formatter_class=argparse.RawTextHelpFormatter)
parser.add_help = True
parser.add_argument('directory', help='directory to search')
parser.add_argument('extension', help='file extension to look for')
parser.add_argument('-r', '--readonly', dest='readonly', action='store_true', help='read only mode (no writes)', default=False)
parser.add_argument('-R', '--recursive', dest='recursive', action='store_true', help='recurse subdirs (def = true)', default=True)
args = parser.parse_args()
# valid dir required
if not os.path.exists(args.directory):
print("Error: directory not found: '" + args.directory + "'")
sys.exit(2)
# valid ext required
args.extension = args.extension.strip()
if len(args.extension) <= 0:
print("Error: invalid extension")
sys.exit(2)
# do it
files = utility.findFiles(args.directory, 1, args.recursive, [args.extension])
for f in files:
print("Deleting: " + f)
if not args.readonly:
os.unlink(f)
def main():
# handle command line args
parser = argparse.ArgumentParser(
description=
'Script to rename GitHub Classroom repos to use student roster identifiers.',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_help = True
parser.add_argument('directory', help='a directory of student repos')
parser.add_argument('roster', help='a classroom roster file from GitHub')
parser.add_argument('-r',
'--readonly',
dest='readonly',
action='store_true',
help='read only mode (no writes)',
default=False)
args = parser.parse_args()
# dir not found
if not os.path.exists(args.directory):
print("Error: no directory found: '" + args.directory + "'")
sys.exit(2)
# roster not found
if not os.path.exists(args.roster):
print("Error: no file found: '" + args.roster + "'")
sys.exit(2)
# read the classroom roster file from GitHub Classroom
username2identifier = {}
with open(args.roster) as roster:
reader = csv.DictReader(roster, delimiter=',')
for row in reader:
identifier = row['identifier']
username = row['github_username']
username2identifier[username] = identifier
# do the renaming
print("Renaming repos in: " + args.directory)
dirs = utility.findFiles(args.directory, 2)
for d in dirs:
dirpath = os.path.dirname(d)
username = os.path.basename(d)
if username in username2identifier:
print("Renaming: " + username + " -> " +
username2identifier[username])
if not args.readonly:
dnew = os.path.join(dirpath, username2identifier[username])
os.rename(d, dnew)
def findHDRFile(datadir, capture, exposure, extension):
datestr = datetime.strftime(capture, "%Y-%m-%d")
timestr = datetime.strftime(capture, "%H.%M.%S")
expidx = common.ExposureIdxMap[exposure]
# find image path of capture timestamp
path = os.path.join(datadir, datestr, "HDR", timestr)
if not os.path.exists(path):
return ''
# gather all exposure photos taken at capture timestamp
photos = utility.findFiles(path, mode=1, ext=[extension])
if len(photos) <= 0:
return ''
# is there a photo for the currently selected exposure?
if expidx >= len(photos):
return ''
return photos[expidx]
def HDROrganizePhotos(args):
print("Organizing HDR photos into timestamp directories in:\n" + args.directory)
# ensure directory exists
if (not os.path.exists(args.directory)):
return
# grab all photos
photos = utility.findFiles(args.directory, mode=1, ext=["jpg", "cr2"])
if (len(photos) <= 0):
print("No photos found in this directory.")
return
# we want to separate photos into directories for each capture
captures = []
captureIntervals = [utility_data.imageEXIFDateTime(photos[0])] # start with first photo timestamp
threshold = 4 # look for next timestamp after this amount of time (next capture interval)
if (args.interval): # user can specify capture interval
threshold = args.interval
captureFolder = os.path.join(args.directory, str(captureIntervals[-1].time()).replace(':', '.'))
print(captureFolder)
if (not args.readonly):
os.mkdir(captureFolder)
# for each loose photo
for p in photos:
last = captureIntervals[-1]
next = utility_data.imageEXIFDateTime(p)
# we've encountered next capture interval
if ((next - last).total_seconds() / 60.0 >= threshold):
captureIntervals.append(next)
captureFolder = os.path.join(args.directory, str(next.time()).replace(':', '.'))
print(captureFolder)
if (not args.readonly):
os.mkdir(captureFolder)
# put photo in folder
destPath = os.path.join(captureFolder, os.path.basename(p))
print("Move " + os.path.basename(p) + " to " + destPath)
if (not args.readonly):
shutil.move(p, destPath)
def HDRRenameFiles(args):
print("Renaming files in:\n" + args.directory)
# ensure directory exists
if (not os.path.exists(args.directory)):
return
# grab all photos
photos = utility.findFiles(args.directory, mode=1, ext=["jpg", "cr2"])
if (len(photos) <= 0):
print("No photos found in this directory.")
return
for p in photos:
oldName = os.path.basename(p)
idx = oldName.find("IMG")
if (idx < 0):
print("Cound not find substring 'IMG' in photo.")
continue
newName = oldName[idx:len(oldName)]
if (newName != oldName):
print("Rename: " + oldName + " to " + newName)
if (not args.readonly):
os.rename(p, os.path.join(os.path.dirname(p), newName))
def ASDOrganizeFiles(args):
print("Organizing ASD files into timestamp directories in:\n" + args.directory)
# ensure directory exists
if (not os.path.exists(args.directory)):
return
# grab all asd files
asdFiles = utility.findFiles(args.directory, mode=1, ext=["asd"])
if (len(asdFiles) <= 0):
print("No asd files found in this directory.")
return
# THIS ASSUMES EACH FILE IN THIS DIRECTORY IS FROM A DIFFERENT SAMPLING PATTERN LOCATION,
# AND THE ORDERING IS THE SAME AS THE CAPTURE PATTERN DEFINED ABOVE!
# THIS IS A MAJOR ASSUMPTION! IT IS BELIEVED TO BE CORRECT IN THE DATA I HAVE AT THE TIME OF WRITING THIS CODE,
# BUT PLEASE CONFIRM THIS BEFORE RUNNING THIS CODE ON YOUR OWN DATA!
# we want to organize all asd files into folders - one folder for each capture interval
captures = []
captureIntervals = [datetime.fromtimestamp(os.path.getmtime(asdFiles[0]))] # start with first timestamp
threshold = 4 # look for next timestamp after this amount of time (next capture interval)
if (args.interval): # user can specify capture interval
threshold = args.interval
captureFolder = os.path.join(args.directory, str(captureIntervals[-1].time()).replace(':', '.'))
print(captureFolder)
if (not args.readonly):
os.mkdir(captureFolder)
# this stuff is for error checking - to make sure we are on track
filesPerCapture = 0
dirsToMake = []
filesToMove = []
# for each asd file
for asdfile in asdFiles:
last = captureIntervals[-1]
next = datetime.fromtimestamp(os.path.getmtime(asdfile))
# we've encountered next capture interval
if ((next - last).total_seconds() / 60.0 >= threshold):
if (filesPerCapture < len(SamplingPattern)): # warning, last folder was not complete
print("ASD file count (" + str(filesPerCapture) + ") for last capture folder was less than capture pattern location count (" + str(len(SamplingPattern)) + ")")
captureIntervals.append(next)
captureFolder = os.path.join(args.directory, str(next.time()).replace(':', '.'))
print(captureFolder)
dirsToMake.append(captureFolder)
filesPerCapture = 0 # reset file count as we are considering a new capture folder
# check to see if we've tried to move more files than there are in a capture pattern
filesPerCapture += 1
if (filesPerCapture > len(SamplingPattern)):
print("ASD file #" + str(filesPerCapture) + " is beyond the capture pattern location count (" + str(len(SamplingPattern)) + ")")
return
# put file into folder
destPath = os.path.join(captureFolder, os.path.basename(asdfile))
print("Move " + os.path.basename(asdfile) + " to " + destPath)
filesToMove.append([asdfile, destPath])
if (os.path.exists(asdfile + ".rad")):
old = asdfile + ".rad"
new = destPath + ".rad"
print("Move " + os.path.basename(old) + " to " + new)
filesToMove.append([old, new])
if (os.path.exists(asdfile + ".rad.txt")):
old = asdfile + ".rad.txt"
new = destPath + ".rad.txt"
print("Move " + os.path.basename(old) + " to " + new)
filesToMove.append([old, new])
# if we've gotten this far, then we haven't errored out, and it's safe to actually do this work
if (not args.readonly):
for dir in dirsToMake:
os.mkdir(dir)
for f in filesToMove:
shutil.move(f[0], f[1])
def CorrelateCaptures(args):
print("Finding HDR/ASD capture times w/in " + str(args.correlatecaptures) + "s in:\n" + args.directory)
# ensure directory exists
if (not os.path.exists(args.directory)):
return
# grab all capture DATE directories
dateDirs = utility.findFiles(args.directory, mode=2)
if (len(dateDirs) <= 0):
print("No directories found at all.")
return
dateDirs[:] = [dir for dir in dateDirs if utility.verifyDateTime(os.path.basename(dir), "%Y-%m-%d")]
if (len(dateDirs) <= 0):
print("No capture date directories found.")
return
# iterate through all capture dates and find all HDR and ASD captures
hdrCaptures = []
asdCaptures = []
correlatedCaptures = []
for datedir in dateDirs:
#print(datedir)
# grab all capture HDR captures
hdrDir = os.path.join(datedir, "HDR")
hdrCaptures = utility.findFiles(hdrDir, mode=2)
if (len(hdrCaptures) <= 0):
print("No HDR directories found at all.")
return
hdrCaptures = [(datetime.strptime(os.path.basename(datedir) + " " + os.path.basename(dir), '%Y-%m-%d %H.%M.%S'), dir) for dir in hdrCaptures if utility.verifyDateTime(os.path.basename(dir), "%H.%M.%S")]
if (len(hdrCaptures) <= 0):
print("No HDR captures found.")
return
#for capture in hdrCaptures:
# print(capture[0], capture[1])
# grab all capture ASD captures
asdDir = os.path.join(datedir, "ASD")
asdCaptures = utility.findFiles(asdDir, mode=2)
if (len(asdCaptures) <= 0):
print("No ASD directories found at all.")
return
asdCaptures = [(datetime.strptime(os.path.basename(datedir) + " " + os.path.basename(dir), '%Y-%m-%d %H.%M.%S'), dir) for dir in asdCaptures if utility.verifyDateTime(os.path.basename(dir), "%H.%M.%S")]
if (len(asdCaptures) <= 0):
print("No ASD captures found.")
return
#for capture in asdCaptures:
# print(capture[0], capture[1])
# correlate HDR to ASD captures
for i in range(0, len(hdrCaptures)):
for j in range(0, len(asdCaptures)):
diff = abs((hdrCaptures[i][0] - asdCaptures[j][0]).total_seconds())
#print(hdrCaptures[i][0], asdCaptures[j][0], diff)
if (diff <= args.correlatecaptures):
correlatedCaptures.append((hdrCaptures[i][0], hdrCaptures[i][1], asdCaptures[j][0], asdCaptures[j][1]))
print("Found " + str(len(correlatedCaptures)) + " captures w/in " + str(args.correlatecaptures) + "s")
prev = None
for capture in correlatedCaptures:
if prev != capture[0].date():
print(capture[1])
print(capture[3])
prev = capture[0].date()
print("HDR:", capture[0], "ASD:", capture[2])
print("Found " + str(len(correlatedCaptures)) + " captures w/in " + str(args.correlatecaptures) + "s")
def ListSubDirectories(args):
print("Listing subdirectories of:\n" + args.directory)
dirs = utility.findFiles(args.directory, mode=2)
for dir in dirs:
print(dir)
