def refresh_courses():
s = courses.login()
course_list = courses.list_courses(s)
with db("db.sqlite") as cursor:
db_courses = [(c["id"], _slugify(c["key_code"]), c["name"]) for c in course_list.values()]
cursor.executemany("INSERT INTO course VALUES (?, ?, ?)", db_courses)
def get_download_ids():
s = courses.login()
with db("db.sqlite") as cursor:
documents = list(
cursor.execute(
"SELECT course_id, document_id FROM document WHERE download_id IS NULL"
))
for doc in documents:
course_id = int(doc[0])
document_id = int(doc[1])
u = "http://" + courses.DOMAIN + "/documents/%i/publication/%i" % (
course_id, document_id)
url = courses.get_doc_url(s, u)
if url.strip():
params = (_download_id_from_url(url), course_id, document_id)
logger.info(params)
with db("db.sqlite") as cursor:
cursor.execute(
"UPDATE document SET download_id=? WHERE course_id=? AND document_id=?",
params)
else:
logger.warning("Document failed: %s, %s" %
(course_id, document_id))
def refresh_courses():
s = courses.login()
course_list = courses.list_courses(s)
with db("db.sqlite") as cursor:
db_courses = [(c['id'], _slugify(c['key_code']), c['name'])
for c in course_list.values()]
cursor.executemany("INSERT INTO course VALUES (?, ?, ?)", db_courses)
def refresh_documents():
s = courses.login()
with db("db.sqlite") as cursor:
for course in list(cursor.execute("SELECT id FROM course")):
course_id = int(course[0])
documents = courses.list_course_files(s, course_id)
db_documents = [(d["name"], course_id, _document_id_from_url(d["pageurl"])) for d in documents]
cursor.executemany("INSERT INTO document (name, course_id, document_id) VALUES (?, ?, ?)", db_documents)
def refresh_documents():
s = courses.login()
with db("db.sqlite") as cursor:
for course in list(cursor.execute("SELECT id FROM course")):
course_id = int(course[0])
documents = courses.list_course_files(s, course_id)
db_documents = [(d['name'], course_id,
_document_id_from_url(d['pageurl']))
for d in documents]
cursor.executemany(
"INSERT INTO document (name, course_id, document_id) VALUES (?, ?, ?)",
db_documents)
def get_download_ids():
s = courses.login()
with db("db.sqlite") as cursor:
documents = list(cursor.execute("SELECT course_id, document_id FROM document WHERE download_id IS NULL"))
for doc in documents:
course_id = int(doc[0])
document_id = int(doc[1])
u = "http://" + courses.DOMAIN + "/documents/%i/publication/%i" % (course_id, document_id)
url = courses.get_doc_url(s, u)
if url.strip():
params = (_download_id_from_url(url), course_id, document_id)
logger.info(params)
with db("db.sqlite") as cursor:
cursor.execute("UPDATE document SET download_id=? WHERE course_id=? AND document_id=?", params)
else:
logger.warning("Document failed: %s, %s" % (course_id, document_id))
def main(limit=None):
login()
courses = list(list_courses().values())[:limit]
for course in courses:
get_course.delay(course)
logger.info("%i courses were enqued", len(courses))