def update_photo_commit_dates(starting_commit):
"""
The old redpandafinder update logic only worked on the basis of commits
in the last week or so. When files are re-sorted, added, or removed for
periods of time, it becomes meaningful to search the entire git repo,
find when a photo URI first appeared, and then track it using its first
commit-date into redpandafinder.
"""
uri_to_commit_date = {}
repo = git.Repo(".")
# List of sha1-name commits from the repo, oldest to newest
commit_list = list(reversed(list(map(lambda x: x.hexsha, repo.iter_commits()))))
if starting_commit != None:
try:
index = commit_list.index(starting_commit)
except IndexError as e:
raise CommitError("%s not a valid commit in this repo." % starting_commit)
commit_list = commit_list[index:] # All after, and including the given commit
for index, commitish in enumerate(commit_list):
# End of the commit list? Call it a day
if commitish == commit_list[len(commit_list) - 1]:
break
# Get the diff
start = commitish
end = commit_list[index + 1]
diff_raw = repo.git.diff(start, end,
ignore_blank_lines=True,
ignore_space_at_eol=True)
patch = PatchSet(diff_raw)
for change in patch:
filename = change.path
if filename.find(".txt") == -1:
# Don't care about non-data files
continue
elif change.added <= 0:
# No lines were added, so we don't care
continue
else:
for hunk in change:
for line in hunk:
if line.is_added:
if re.match("photo.\d+:", line.value) == None:
# Not a photo line
continue
if line.value.find(": ") == -1:
# No correct delimiter, which we see in old commits
continue
if len(line.value.strip().split(": ")) != 2:
# Probably bad linebreaks
continue
[key, value] = line.value.strip().split(": ")
if (value in uri_to_commit_date):
# Photo we've already seen
continue
if (value.find("http") != 0):
# Not a URI, so not a photo reference
continue
dt = repo.commit(end).committed_datetime
date = str(dt.year) + "/" + str(dt.month) + "/" + str(dt.day)
if value not in uri_to_commit_date:
# Only insert a comit date once
uri_to_commit_date[value] = date
# print(str(uri_to_commit_date))
# Now walk the repo, find all files with photo lines that have no commit dates,
# and add commitdate to each photo that needs one
for file_path in [PANDA_PATH, ZOO_PATH, MEDIA_PATH]:
section = None
for section_name in ["media", "zoos", "pandas"]:
if section_name in file_path.split("/"):
section = section_name.split("s")[0] # HACK
# Enter the pandas subdirectories
for root, dirs, files in os.walk(file_path):
for filename in files:
path = root + os.sep + filename
# print(path)
photo_list = PhotoFile(section, path)
photo_count = photo_list.photo_count()
photo_index = 1
while (photo_index <= photo_count):
photo_option = "photo." + str(photo_index)
photo_uri = photo_list.get_field(photo_option)
date_option = photo_option + ".commitdate"
if photo_uri not in uri_to_commit_date:
photo_index = photo_index + 1
continue
date_value = uri_to_commit_date[photo_uri]
photo_list.set_field(date_option, date_value)
# print(photo_uri + " ==> " + date_value)
photo_index = photo_index + 1
photo_list.update_file()
def update_entity_commit_dates(starting_commit):
"""
When moving pandas, the old redpandafinder updater logic considered "new"
animals as anything that was a new file in a location. So when an animal
moved zoos, it became _new_ again. Rectify this by tracking when the
commitdate for each new animal is. Track commit dates for other files too,
just for the hell of it.
"""
filename_to_commit_date = {}
type_id_to_commit_date = {}
repo = git.Repo(".")
# List of sha1-name commits from the repo, oldest to newest
commit_list = list(reversed(list(map(lambda x: x.hexsha, repo.iter_commits()))))
if starting_commit != None:
try:
index = commit_list.index(starting_commit)
except IndexError as e:
raise CommitError("%s not a valid commit in this repo." % starting_commit)
commit_list = commit_list[index:] # All after, and including the given commit
for index, commitish in enumerate(commit_list):
# End of the commit list? Call it a day
if commitish == commit_list[len(commit_list) - 1]:
break
# Get the diff
start = commitish
end = commit_list[index + 1]
diff_raw = repo.git.diff(start, end,
ignore_blank_lines=True,
ignore_space_at_eol=True)
patch = PatchSet(diff_raw)
for change in patch:
filename = change.path
if filename.find(".txt") == -1:
# Don't care about non-data files
continue
elif change.is_added_file == True:
compare = "./" + filename
dt = repo.commit(end).committed_datetime
date = str(dt.year) + "/" + str(dt.month) + "/" + str(dt.day)
just_file = filename.split("/").pop()
just_type = None
just_id = None
if compare.find(PANDA_PATH) == 0:
just_type = "panda"
just_id = just_file.split("_")[0]
elif compare.find(ZOO_PATH) == 0:
just_type = "zoo"
just_id = just_file.split("_")[0]
elif compare.find(MEDIA_PATH) == 0:
just_type = "media"
just_id = filename # Need full path for media files
else:
continue # Not a file we're tracking commitdates for
filename_to_commit_date[just_file] = date
type_id_to_commit_date[just_type + "_" + just_id] = date
else:
continue
# print(str(filename_to_commit_date))
# print(str(type_id_to_commit_date))
# Now walk the repo, find all panda files without commit dates,
# and add commitdate to each photo that needs one
for file_path in [MEDIA_PATH, PANDA_PATH, ZOO_PATH]:
section = None
for section_name in ["media", "zoos", "pandas"]:
if section_name in file_path.split("/"):
section = section_name.split("s")[0] # HACK
# Enter the pandas subdirectories
for root, dirs, files in os.walk(file_path):
for filename in files:
path = root + os.sep + filename
photo_list = PhotoFile(section, path)
if photo_list.get_field("commitdate") == None:
if filename not in filename_to_commit_date:
# file's name was changed at some point
just_file = filename.split("/").pop()
just_type = None
just_id = None
if path.find(PANDA_PATH) == 0:
just_type = "panda"
just_id = just_file.split("_")[0]
elif path.find(ZOO_PATH) == 0:
just_type = "zoo"
just_id = just_file.split("_")[0]
elif path.find(MEDIA_PATH) == 0:
just_type = "media"
just_id = path # Need full path for media files
else:
continue # Not a file we're tracking commitdates for
just_key = just_type + "_" + just_id
if just_key not in type_id_to_commit_date:
print("warning: %s commitdate undetermined" % filename)
continue
else:
date = type_id_to_commit_date[just_key]
photo_list.set_field("commitdate", date)
else:
date = filename_to_commit_date[filename]
photo_list.set_field("commitdate", date)
photo_list.update_file()