def update_db(self, **kwargs):
"""
update self.**kwargs and write present state to db, also set
'last_checked'
"""
for k,v in kwargs.items():
setattr(self, k, v)
cur = db.cursor()
self.last_checked = datetime.now()
fields = [f for f in self.db_fields.keys()
if f != 'link_id' and getattr(self, f) is not None]
values = [getattr(self, f) for f in fields]
if self.link_id:
query = "UPDATE links SET {},urlhash=MD5(url) WHERE link_id = %s".format(
",".join(k+"=%s" for k in fields))
cur.execute(query, values + [self.link_id])
else:
query = "INSERT INTO links ({},urlhash) VALUES ({},MD5(url))".format(
",".join(fields), ",".join(("%s",)*len(fields)))
try:
cur.execute(query, values)
except:
debug(1, "oops, %s: %s", query, ','.join(map(str, values)))
raise
self.link_id = cur.lastrowid
debug(4, cur._last_executed)
db.commit()
def store_page(self, url, name):
"""write page <url> for author <name> to db"""
cur = db.cursor()
query = "INSERT INTO sources (status,sourcetype,url,default_author,name,found_date)"
query += "VALUES (0,'personal',%s,%s,%s, NOW())"
cur.execute(query, (url,name,"{}'s site".format(name)))
db.commit()
def assign_category(self, cat_id, strength):
"""inserts or updates a docs2cats entry in the db"""
if not self.doc_id:
raise Exception("cannot assign category: document has no id")
cur = db.cursor()
query = ("INSERT INTO docs2cats (cat_id, doc_id, strength) VALUES (%s,%s,%s)"
" ON DUPLICATE KEY UPDATE strength=%s")
cur.execute(query, (cat_id, self.doc_id, strength, strength))
debug(4, cur._last_executed)
db.commit()
def update_db(self, **kwargs):
"""write **kwargs to db, also update 'last_checked'"""
if self.source_id:
cur = db.cursor()
kwargs['last_checked'] = time.strftime('%Y-%m-%d %H:%M:%S')
query = "UPDATE sources SET {},urlhash=MD5(url) WHERE source_id = %s".format(
",".join(k+"=%s" for k in kwargs.keys()))
cur.execute(query, tuple(kwargs.values()) + (self.source_id,))
debug(3, cur._last_executed)
db.commit()
def save_to_db(self):
"""write object to db"""
cur = db.cursor()
fields = [f for f in self.db_fields.keys()
if f != 'link_id' and getattr(self, f) is not None]
values = [getattr(self, f) for f in fields]
query = "INSERT INTO sources ({}, urlhash) VALUES ({}, MD5(url))".format(
",".join(fields), ",".join(("%s",)*len(fields)))
cur.execute(query, values)
debug(3, cur._last_executed)
db.commit()
self.source_id = cur.lastrowid
def testdb():
"""set up test database"""
db.close()
db.connection(db='test_opp')
cur = db.cursor()
for t in ('sources', 'links', 'docs'):
cur.execute('DELETE FROM {}'.format(t))
db.commit()
Source(
url='http://umsu.de/papers/',
sourcetype='personal',
status=0,
last_checked=datetime.now()).save_to_db()
Source(
url='http://consc.net/papers.html',
sourcetype='personal',
status=1).save_to_db()
def update_db(self, **kwargs):
"""update self.**kwargs and write present state to db"""
for k, v in kwargs.items():
setattr(self, k, v)
cur = db.cursor()
fields = [f for f in self.db_fields.keys()
if f != 'doc_id' and getattr(self, f) is not None]
values = [getattr(self, f) for f in fields]
if self.doc_id:
query = "UPDATE docs SET {},urlhash=MD5(url) WHERE doc_id = %s".format(
",".join(k+"=%s" for k in fields))
cur.execute(query, values + [self.doc_id])
else:
query = "INSERT INTO docs ({},urlhash) VALUES ({},MD5(url))".format(
",".join(fields), ",".join(("%s",)*len(fields)))
cur.execute(query, values)
self.doc_id = cur.lastrowid
debug(4, cur._last_executed)
db.commit()
def run(self):
cur = db.cursor()
findings = []
for url in self.select_journals():
logger.debug("looking for author names on %s", url)
for name in self.get_authornames(url):
query = "INSERT INTO author_names (name, last_searched) VALUES (%s, '1970-01-01')"
try:
cur.execute(query, (name,))
db.commit()
except MySQLdb.IntegrityError:
logger.debug("{} already in db".format(name))
findings = [f for f in findings if f[0] != name]
else:
logger.debug("+++ new author name {}".format(name))
name_id = cur.lastrowid
findings.append((name, name_id, url))
if findings:
self.sendmail(findings)
def remove_from_db(doc):
cur = db.cursor()
query = "DELETE FROM docs WHERE doc_id = %s"
cur.execute(query, (doc.doc_id,))
debug(4, cur._last_executed)
db.commit()
def update_author(self, name):
"""update last_searched field for author <name>"""
cur = db.cursor()
query = "UPDATE author_names SET last_searched=NOW() WHERE name=%s"
cur.execute(query, (name,))
db.commit()
