def start(kwargs): session, engine = initSession() with MySQLdb.connect(kwargs['db_host'], kwargs['db_user'], kwargs['db_pass'], '1188ys_resultdb') as cursor: #cursor = db.cursor() limit = 1000 cursor.execute('''SELECT count(*) from meishi_recipe''') rowCount = cursor.fetchone()[0] runtimes = math.ceil(rowCount/limit) for x in xrange(0, int(runtimes)): sql = "SELECT taskid, result from meishi_recipe limit %d, %d" % (int(x*limit ), int(limit)) cursor.execute(sql) allRows = cursor.fetchall() i = 0 for row in allRows: try: itemObj = json.loads(row[1]) recipe = searchAndSave(session, itemObj) saveRecipeInfo(session, itemObj, recipe) saveCategoryRelation(session, itemObj, recipe) i += 1 if i % 200 == 0: session.commit() except Exception, e: traceback.print_exc(file=sys.stdout) myLogger.error('Error: %s - taskid : %s',e, row[0]) #sys.exit() #logging.error('task id is %s, eval error ', row[0]) session.commit()
def start(kwargs): session, engine = initSession() with MySQLdb.connect(kwargs['db_host'], kwargs['db_user'], kwargs['db_pass'], '1188ys_resultdb') as cursor: #cursor = db.cursor() limit = 1000 cursor.execute('''SELECT count(*) from meishi_recipe''') rowCount = cursor.fetchone()[0] runtimes = math.ceil(rowCount / limit) for x in xrange(0, int(runtimes)): sql = "SELECT taskid, result from meishi_recipe limit %d, %d" % ( int(x * limit), int(limit)) cursor.execute(sql) allRows = cursor.fetchall() i = 0 for row in allRows: try: itemObj = json.loads(row[1]) recipe = searchAndSave(session, itemObj) saveRecipeInfo(session, itemObj, recipe) saveCategoryRelation(session, itemObj, recipe) i += 1 if i % 200 == 0: session.commit() except Exception, e: traceback.print_exc(file=sys.stdout) myLogger.error('Error: %s - taskid : %s', e, row[0]) #sys.exit() #logging.error('task id is %s, eval error ', row[0]) session.commit()
def start(kwargs): session, engine = initSession() with MySQLdb.connect(kwargs['db_host'], kwargs['db_user'], kwargs['db_pass'], '1188ys_resultdb') as cursor: #cursor = db.cursor() cursor.execute('''SELECT taskid, result from meishi_category''') allRows = cursor.fetchall() i = 0 for row in allRows: try: itemObjs = ast.literal_eval(row[1]) searchAndSaveCategory(session, itemObjs) i += 1 if i % 200 == 0: session.commit() except Exception, e: myLogger.error('Error: %s - taskid : %s', e, row[0]) #logging.error('task id is %s, eval error ', row[0]) session.commit()
def start(kwargs): session, engine = initSession() with MySQLdb.connect(kwargs['db_host'], kwargs['db_user'], kwargs['db_pass'], '1188ys_resultdb') as cursor: #cursor = db.cursor() cursor.execute('''SELECT taskid, result from meishi_material''') allRows = cursor.fetchall() i = 0 for row in allRows: try: itemObj = ast.literal_eval(row[1]) searchAndSaveMaterial(session, itemObj) i += 1 if i % 200 == 0: session.commit() except Exception, e: #traceback.print_exc(file=sys.stdout) myLogger.error('Error: %s - taskid : %s',e, row[0]) #sys.exit() #logging.error('task id is %s, eval error ', row[0]) session.commit()