def main(): db_conn = sqa.connect_db() Session = sessionmaker(bind=db_conn) session = Session() res = session.query(sqa.UrlRecords).filter(sqa.UrlRecords.num_redir>='5').all() for i in res: print i.exp_url
def main(): logging.debug("main called") db_conn = sqa.connect_db() Session = sessionmaker(bind=db_conn) session = Session() count1 = session.query(sqa.UrlRecords).count() session.close() in_queue = Queue() # pt = ParseTwitter(sys.argv[1], sys.argv[2], in_queue) # pt.parse() url_dump = FileIoUpd(sys.argv[1], sys.argv[2], in_queue) url_dump.run() count2 = session.query(sqa.UrlRecords).count() print "Records Added: ", count2-count1
def redirect_urls(inputq, total_q, indb_q, todb_q): """ Updated: input remains the same, however the output gets written to the database """ """ Takes the list of Url's as input and writes the input record and the number of redirects to a file""" db_conn = sqa.connect_db() Session = sessionmaker(bind=db_conn) session = Session() while True: with lock4: try: item = inputq.get_nowait() except Exception, e: print "Empty exception", e session.close() # return sys.exit(0) with lock1: total_q.put(total_q.get() + 1) try: rec = json.loads(item) except: print "json error" tc_url = rec['twit_url'] ex_url = rec['expanded_url'] print "item", ex_url try: ex_url = ex_url.lower() # Expect the encoding errors here, unicode string except Exception, e: logging.debug("terminating %s" %exp_url) inputq.task_done() continue