def query_bitly(longUrl, user):
l = urllib.quote(longUrl, '')
if (longUrl[:7].lower() != 'http://'
and urllib.unquote(longUrl)[:7].lower() != 'http://'
and longUrl[:8].lower() != 'https://'
and urllib.unquote(longUrl)[:8].lower() != 'https://'):
l = urllib.quote('http://' + longUrl, '')
result = urlfetch.fetch(JMP_URL + l)
logging.debug('posted to bit.ly: %s' % l)
if result.status_code != 200:
return 'Sorry! Query failed.'
j = json.JSONDecoder()
data = j.decode(result.content)
if data.get('status_code') == 403:
logging.warning('RATE LIMIT EXCEEDED')
return 'Sorry! Experiencing rate limits from bit.ly'
if data.get('status_code') != 200:
logging.error(result.content)
return 'Sorry! bit.ly did not accept the query. Make sure that your message only contains a URL.'
url = Url(longUrl=data.get('data').get('long_url'),
shortUrl=data.get('data').get('url'),
creator=user)
url.put()
return data.get('data').get('url')
def query_bitly(longUrl, user):
l = urllib.quote(longUrl,'')
if (longUrl[:7].lower() != 'http://' and urllib.unquote(longUrl)[:7].lower() != 'http://' and
longUrl[:8].lower() != 'https://' and urllib.unquote(longUrl)[:8].lower() != 'https://'):
l = urllib.quote('http://'+longUrl,'')
result = urlfetch.fetch(JMP_URL+l)
logging.debug('posted to bit.ly: %s' % l)
if result.status_code != 200:
return 'Sorry! Query failed.'
j = json.JSONDecoder()
data = j.decode(result.content)
if data.get('status_code') == 403:
logging.warning('RATE LIMIT EXCEEDED')
return 'Sorry! Experiencing rate limits from bit.ly'
if data.get('status_code') != 200:
logging.error(result.content)
return 'Sorry! bit.ly did not accept the query. Make sure that your message only contains a URL.'
url = Url(longUrl=data.get('data').get('long_url'), shortUrl=data.get('data').get('url'), creator=user)
url.put()
return data.get('data').get('url')
def importIntoVersion(version):
logging.info("comparing to version number %i" % version.number)
topic = Topic.get_by_id("art-history", version)
if not topic:
parent = Topic.get_by_id("humanities---other", version)
if not parent:
raise Exception(
"Could not find the Humanities & Other topic to put art history into"
)
topic = Topic.insert(
title="Art History",
parent=parent,
id="art-history",
standalone_title="Art History",
description=
"Spontaneous conversations about works of art where the speakers are not afraid to disagree with each other or art history orthodoxy. Videos are made by Dr. Beth Harris and Dr. Steven Zucker along with other contributors."
)
urls = topic.get_urls(include_descendants=True)
href_to_key_dict = dict((url.url, url.key()) for url in urls)
hrefs = [url.url for url in urls]
content = getSmartHistoryContent()
if content is None:
raise Exception(
"Aborting import, could not read from smarthistory")
subtopics = topic.get_child_topics()
subtopic_dict = dict((t.title, t) for t in subtopics)
subtopic_child_keys = {}
new_subtopic_keys = []
child_keys = []
i = 0
for link in content:
href = link["href"]
title = link["title"]
parent_title = link["parent"]
if parent_title not in subtopic_dict:
subtopic = Topic.insert(title=parent_title,
parent=topic,
standalone_title="Art History: %s" %
parent_title,
description="")
subtopic_dict[parent_title] = subtopic
else:
subtopic = subtopic_dict[parent_title]
if subtopic.key() not in new_subtopic_keys:
new_subtopic_keys.append(subtopic.key())
if parent_title not in subtopic_child_keys:
subtopic_child_keys[parent_title] = []
if href not in hrefs:
logging.info("adding %i %s %s to %s" %
(i, href, title, parent_title))
models.VersionContentChange.add_new_content(
models.Url, version, {
"title": title,
"url": href
}, ["title", "url"])
url = Url(url=href, title=title, id=id)
url.put()
subtopic_child_keys[parent_title].append(url.key())
else:
subtopic_child_keys[parent_title].append(
href_to_key_dict[href])
i += 1
logging.info("updating child_keys")
change = False
for parent_title, child_keys in subtopic_child_keys.iteritems():
subtopic = subtopic_dict[parent_title]
if subtopic.child_keys != subtopic_child_keys[parent_title]:
change = True
subtopic.update(child_keys=subtopic_child_keys[parent_title])
if topic.child_keys != new_subtopic_keys:
change = True
topic.update(child_keys=new_subtopic_keys)
if change:
logging.info("finished updating version number %i" %
version.number)
else:
logging.info("nothing changed")
return change
def importIntoVersion(version):
logging.info("comparing to version number %i" % version.number)
topic = Topic.get_by_id("art-history", version)
if not topic:
parent = Topic.get_by_id("humanities---other", version)
if not parent:
raise Exception("Could not find the Humanities & Other topic to put art history into")
topic = Topic.insert(title="Art History",
parent=parent,
id="art-history",
standalone_title="Art History",
description="Spontaneous conversations about works of art where the speakers are not afraid to disagree with each other or art history orthodoxy. Videos are made by Dr. Beth Harris and Dr. Steven Zucker along with other contributors.")
urls = topic.get_urls(include_descendants=True)
href_to_key_dict = dict((url.url, url.key()) for url in urls)
hrefs = [url.url for url in urls]
content = getSmartHistoryContent()
if content is None:
raise Exception("Aborting import, could not read from smarthistory")
subtopics = topic.get_child_topics()
subtopic_dict = dict((t.title, t) for t in subtopics)
subtopic_child_keys = {}
new_subtopic_keys = []
child_keys = []
i = 0
for link in content:
href = link["href"]
title = link["title"]
parent_title = link["parent"]
if parent_title not in subtopic_dict:
subtopic = Topic.insert(title=parent_title,
parent=topic,
standalone_title="Art History: %s" % parent_title,
description="")
subtopic_dict[parent_title] = subtopic
else:
subtopic = subtopic_dict[parent_title]
if subtopic.key() not in new_subtopic_keys:
new_subtopic_keys.append(subtopic.key())
if parent_title not in subtopic_child_keys:
subtopic_child_keys[parent_title] = []
if href not in hrefs:
logging.info("adding %i %s %s to %s" % (i, href, title, parent_title))
models.VersionContentChange.add_new_content(
models.Url,
version,
{"title": title,
"url": href
},
["title", "url"])
url = Url(url=href,
title=title,
id=id)
url.put()
subtopic_child_keys[parent_title].append(url.key())
else:
subtopic_child_keys[parent_title].append(href_to_key_dict[href])
i += 1
logging.info("updating child_keys")
change = False
for parent_title, child_keys in subtopic_child_keys.iteritems():
subtopic = subtopic_dict[parent_title]
if subtopic.child_keys != subtopic_child_keys[parent_title]:
change = True
subtopic.update(child_keys=subtopic_child_keys[parent_title])
if topic.child_keys != new_subtopic_keys:
change = True
topic.update(child_keys=new_subtopic_keys)
if change:
logging.info("finished updating version number %i" % version.number)
else:
logging.info("nothing changed")
return change
