def getEntityById(self, entityId):
"""Gets entity from LRI
entityId: ID of the entity to get
returns: JSON of the entity
"""
entities = []
q = '{"urn:lri:property_type:id":"%s"}' % entityId
url = self.toUrlForm(q)
print("StandardQuery.getEntityById: Running query: %s" % url)
results = requests.get(url)
statusCode = results.status_code
j = utils.getJson(results)
response = j["response"]
for entity in response:
entityId = entity["props"]["urn:lri:property_type:id"]
entities.append(entity)
print("StandardQuery.getEntityById: Added entity: %s" % entityId)
print("StandardQuery.getEntityById: Got %d entity(s)" % len(entities))
if len(entities) > 1:
print("StandardQuery.getEntityById: ERROR: Too many entities: %d from id: %d" % (len(entities), entityId))
entity = None
try:
entity = entities[0]
except IndexError, e:
raise IndexError("ERROR getting: %s" % entityId)
def runQuery(self):
"""Runs the query. Returns QueryResult"""
msg = "QueryRunner.runQuery: url = %s" % self.url
web.debug(msg)
r = requests.get(self.url)
queryResult = QueryResult(r.status_code, utils.getJson(r))
return queryResult
def get(self):
tic = utils.microtime(True)
for n in range(1, 11):
json_url = 'http://www.noticiashacker.com/nuevo.json?pagina=%d'
noticias = utils.getJson(json_url % n)
for noticia in noticias['posts']:
user = utils.put_user(noticia['user'])
post = utils.put_post(noticia,user)
toc = utils.microtime(True)
print "Time elapsed %f seconds" % (toc-tic)
def doPost(pathInfo, opts, data):
"""Does HTTP POST"""
web.debug("doPost")
formatted = ""
results = QueryResult()
code = 201
try:
# Create & run inserts (create, update)
toFormat = "xml"
if not "format" in opts:
userData = web.input()
toFormat = userData.get("format", "xml")
web.debug("to format = %s" % toFormat)
pattern = "(/[a-z_]+)(/[a-z]+)"
m = re.match(pattern, pathInfo)
ccssType, action = m.groups()
if action == "/create":
i = insert.InsertFactory().CreateInsert(ccssType, action, opts, data)
for url in i.getUrls():
r = QueryRunner(url)
results = r.runQuery()
if not isinstance(results, QueryResult):
raise WrongTypeError("doPost: QueryRunner.runQuery(): expected QueryResult, got %s" % type(results))
code = results.getHttpStatusCode()
# Convert to requested format
f = format.FormatterFactory().CreateFormatter(toFormat)
formatted = f.format(results.getData())
elif action == "/update":
web.debug("===== %s =====" % action)
updates = insert.parseXml(data)
web.debug("doPost: updates = %r" % updates)
responses = insert.runUpdates(updates, opts)
count = 0
for r in responses:
f = format.FormatterFactory().CreateFormatter(toFormat)
formatted += f.format(utils.getJson(r))
# What to do about multiple <?xml...?> lines?
# tmp = f.format(r.json)
# if count > 0:
# if toFormat in ["xml", "johnxml", "oldxml"]:
# lines = tmp.split("\n")
# lines.remove(lines[0])
# tmp = "\n".join(lines)
# formatted += tmp
# count += 1
except NotImplementedError, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
formatted = "doPost: Caught NotImplementedError: " + str(e) + "\n"
formatted += "-" * 60 + "\n"
formatted += traceback.format_exc()
formatted += "-" * 60 + "\n"
code = 501
# Size of result set returned w/cursor
limit = 1
q = query.QueryFactory().CreateQuery(pathInfo, opts, httpConfig, limit)
if q is None:
code = 404
return code, formatted
# Run query once w/small limit
# Extract cursor from results
# Run subsequent queries in loop
# until cursor comes back False
print ("doGet: url = %s" % q.getUrl())
response = requests.get(q.getUrl())
j = utils.getJson(response)
if "cursor" in j:
cursor = j["cursor"]
print ("\ndoGet: cursor = %s\n" % cursor)
statusCode = response.status_code
status = json.dumps(j["status"])
items = []
if cursor is False:
items = j["response"]
else:
count = 0
while cursor is not False: # != "false":
print ("doGet: Count = %d" % count)
print ("doGet: Getting next %d results" % limit)
url = 'http://%s:%s/entity/search?q={"limit":%d,"cursor":%s}&opts=%s' % (
def getStandards(self):
"""Gets standards from LRI
returns: JSON of the standards
"""
print("StandardQuery.getStandards")
print("StandardQuery.getStandards: self.opts = %r" % self.opts)
print("StandardQuery.getStandards: Running query: %s" % self.toUrlForm())
results = requests.get(self.toUrlForm())
statusCode = results.status_code
j = utils.getJson(results)
print("StandardQuery.getStandards: j=",j)
response = j["response"]
# Get standards
standards = []
for item in response:
print
itemId = item["props"]["urn:lri:property_type:id"]
print("StandardQuery.getStandards: Processing: %s" % itemId)
# If item has a CCID it is a CCSS type
if "urn:ccss:property_type:ccid" not in item["props"]:
# Custom standards will not have ccid
print("StandardQuery.getStandards: Processing Custom item: %s" % itemId)
# Custom standards will be type competency
customType = "urn:lri:entity_type:competency"
if customType in item["props"]["urn:lri:property_type:types"]:
standards.append(item)
print("StandardQuery.getStandards: Added Custom standard: %s" % itemId)
else:
print("StandardQuery.getStandards: ERROR: type: %s not in: %r" % (customType,
item["props"]["urn:lri:property_type:types"]))
else:
ccid = item["props"]["urn:ccss:property_type:ccid"]
print("StandardQuery.getStandards: Processing CCSS item: %s" % ccid)
if ccid.find("Math") != -1:
# Prune CCSS Math structure
print("StandardQuery.getStandards: Processing CCSS Math item: %s" % ccid)
if "urn:ccss:entity_type:standard" in item["props"]["urn:lri:property_type:types"]:
standards.append(item)
print("StandardQuery.getStandards: Added CCSS Math standard: %s" % itemId)
elif ccid.find("ELA-Literacy") != -1:
# Prune CCSS ELA structure
print("StandardQuery.getStandards: Processing CCSS ELA item: %s" % ccid)
cbProp = "urn:lri:property_type:contained_by"
containedBy = item["props"][cbProp]
if type(containedBy) == str or type(containedBy) == unicode:
containedBy = [containedBy]
# Handle contained_by domain or grade_level
for cb in containedBy:
print("StandardQuery.getStandards: contained_by = %s" % cb)
if cb.find(":domain:") != -1:
# contained_by domain means item is grade_level
# grade_level contains standard
gradeId = item["props"]["urn:lri:property_type:id"]
print("StandardQuery.getStandards: Checking: (%s, %s)" % (self.grade_level, gradeId))
if gradeId == self.grade_level:
print("StandardQuery.getStandards: Processing grade_level: %s" % gradeId)
for standardId in item["props"]["urn:lri:property_type:contains"]:
print("StandardQuery.getStandards: Processing CCSS ELA standard: %s" % standardId)
standard = self.getEntityById(standardId)
standards.append(standard)
newId = standard["props"]["urn:lri:property_type:id"]
if newId != standardId:
print("StandardQuery.getStandards: ERROR: %s != %s" % (newId, standardId))
print("StandardQuery.getStandards: Added CCSS ELA standard: %s, (domain = %s, grade = %s)" % (newId, cb, gradeId))
else:
print("StandardQuery.getStandards: Skipping grade_level: %s" % gradeId)
elif cb.find(":grade") != -1:
# contained_by grade_level means item is standard
standard = self.getEntityById(itemId)
standards.append(standard)
newId = standard["props"]["urn:lri:property_type:id"]
if newId != itemId:
print("StandardQuery.getStandards: ERROR: %s != %s" % (newId, itemId))
print("StandardQuery.getStandards: Added CCSS ELA standard: %s" % newId)
print("StandardQuery.getStandards: Processed %d standards" % len(standards))
# Get components
if self.getChildren:
print("StandardQuery.getStandards: Getting standard_components")
for std in standards:
if not "urn:lri:property_type:contains" in std["props"]:
continue
stdId = std["props"]["urn:lri:property_type:id"]
print("StandardQuery.getStandards: Getting components of: %s" % stdId)
components = []
print("StandardQuery.getStandards: contains: ")
print(std["props"]["urn:lri:property_type:contains"])
print("StandardQuery.getStandards: /contains")
contains = std["props"]["urn:lri:property_type:contains"]
if type(contains) == str or type(contains) == unicode:
contains = [contains]
for componentId in contains:
print("StandardQuery.getStandards: Getting standard_component: %s" % componentId)
component = self.getEntityById(componentId)
components.append(component)
newId = component["props"]["urn:lri:property_type:id"]
if newId != componentId:
print("StandardQuery.getStandards: ERROR: %s != %s" % (newId, standardId))
std["props"]["urn:lri:property_type:contains"] = components
print("StandardQuery.getStandards: Added: %s components to standard: %s" % (len(std["props"]["urn:lri:property_type:contains"]), stdId))
# Get anchors
if self.getAnchors:
print("StandardQuery.getStandards: Getting anchor_standards")
for std in standards:
if not "urn:ccss:property_type:is_anchored_to" in std["props"]:
continue
stdId = std["props"]["urn:lri:property_type:id"]
print("StandardQuery.getStandards: Getting anchor for: %s" % stdId)
anchorId = std["props"]["urn:ccss:property_type:is_anchored_to"]
anchor = self.getEntityById(anchorId)
newId = anchor["props"]["urn:lri:property_type:id"]
if newId != anchorId:
print("StandardQuery.getStandards: ERROR: %s != %s" % (newId, anchorId))
std["props"]["urn:ccss:property_type:is_anchored_to"] = anchor
print("StandardQuery.getStandards: Set is_anchored_to: %s" % anchor)
# Flatten
r = {}
r["response"] = []
r["response"].append({})
r["response"][0]["standards"] = standards
r["status"] = "success"
r["status_code"] = statusCode
return r
