def importFile(filename, verbose):
# load JSON from file
try:
with open(filename) as f:
j = json.load(f)
except:
sys.exit("could not load JSON file")
try:
searches = j['searches']
except:
sys.exit("No 'searches' field in file")
AO3datalist = []
for s in searches:
if verbose:
print "search: ", s
x = AO3search.AO3data()
x.searchParams = s
AO3datalist.append(x)
return AO3datalist
def FetchMetaTags(tagName):
url = getAO3TagStructureURL(tagName)
tagData = AO3search.AO3data()
tagData.metaURL = url
tagData.getMetaTags()
if DEBUG:
print "Meta tags:"
print tagData.metaTags
return (tagData.metaTags)
def FetchMetaTags(tagName):
print "fetching meta tags"
# time.sleep(PAUSE_INTERVAL)
url = getAO3TagStructureURL(tagName)
tagData = AO3search.AO3data()
tagData.metaURL = url
tagData.getMetaTags()
if DEBUG:
print "Meta tags:"
print tagData.metaTags
return (tagData.metaTags)
def FetchTopTagInfo(primaryTag, includeTags, excludeTags):
url = getAO3SearchURL(primaryTag, includeTags, excludeTags)
# if DEBUG:
# print "including " + str(len(includeTags)) + " tags: "
# print includeTags
# print "excluding " + str(len(excludeTags)) + " tags: "
# print excludeTags
# print url
primaryTagData = AO3search.AO3data()
primaryTagData.searchURL = url
primaryTagData.getTopInfo()
primaryTagData.getNumWorks(True)
return primaryTagData
def FetchTopTagInfo(primaryTag, includeTags, excludeTags):
print "fetching top tag info: " + primaryTag
print "excluding: "
print excludeTags
# time.sleep(PAUSE_INTERVAL)
url = getAO3SearchURL(primaryTag, includeTags, excludeTags)
# if DEBUG:
# print "including " + str(len(includeTags)) + " tags: "
# print includeTags
# print "excluding " + str(len(excludeTags)) + " tags: "
# print excludeTags
# print url
primaryTagData = AO3search.AO3data()
primaryTagData.searchURL = url
primaryTagData.getTopInfo()
primaryTagData.getNumWorks(True)
return primaryTagData
for c in characters:
print "**********************************"
print "Fetching AO3 data for", c.name, "(", c.movie.title, ")"
if verbose:
print "Age (actor):", c.age
print "Gender:", c.gender
print "Fraction of dialogue:", c.fractionOfDialogue
# first make sure that searching for the movie title on AO3
# returns some fanworks
numResults = 0
if moviesWithFewAO3Results.has_key(c.movie.title):
numResults = moviesWithFewAO3Results[c.movie.title]
else:
movieData = AO3search.AO3data()
bareTitle = c.movie.title.translate(None, string.punctuation)
# in case of a movie like Alien^3 with encoding errors...
bareTitle = unicode(bareTitle, errors='ignore')
# movieData.searchURL = FandomSearchURL(bareTitle, verbose)
movieData.searchURL = FandomSearchURL(bareTitle, False)
numResults = movieData.getNumWorks(False)
if verbose:
print "Num works returned for movie:", numResults
if numResults < MIN_AO3_MOVIE_RESULTS:
moviesWithFewAO3Results[c.movie.title] = numResults
c.numFandomWorksLiteral = -1
else:
includeTags = [line.rstrip('\n') for line in open(includefile)]
excludeTags = [line.rstrip('\n') for line in open(excludefile)]
if DEBUG:
print includeTags
print excludeTags
http = setupUrllib()
for i in range(recursionDepth):
url = getSearchURL(tag, includeTags, excludeTags)
print i
if DEBUG:
print url
# create an AO3 search object for the tag
tagData = AO3search.AO3data()
tagData.searchURL = url
# get the top 10 characters
tagData.getTopInfo()
chars = tagData.categories["character"]["top"]
if DEBUG:
print chars
# merge the dictionaries into topFandoms and topShips
# topFandoms = mergeDictionaries(topFandoms, fandoms)
# topShips = mergeDictionaries(topShips, ships)
topCharacters = mergeDictionaries(topCharacters, chars)
# exclude the top ships (but not the fandoms, because there may be
# other ships that match the search criteria in the same fandom)