def start(): session, engine = initSession() with MySQLdb.connect('172.16.1.248', 'qiye_dev', 'qiye..dev', '1188ys_resultdb') as cursor: #cursor = db.cursor() limit = 1000 cursor.execute( '''SELECT count(*) from meishi_material_recipe_relation''') rowCount = cursor.fetchone()[0] runtimes = math.ceil(rowCount / limit) for x in xrange(0, int(runtimes)): sql = "SELECT taskid, result from meishi_material_recipe_relation limit %d, %d" % ( int(x * limit), int(limit)) cursor.execute(sql) allRows = cursor.fetchall() i = 0 for row in allRows: try: itemObj = json.loads(row[1]) searchAndSave(session, itemObj) #sys.exit() i += 1 if i % 200 == 0: session.commit() except Exception, e: traceback.print_exc(file=sys.stdout) myLogger.error('Error: %s - taskid : %s', e, row[0]) #sys.exit() #logging.error('task id is %s, eval error ', row[0]) session.commit()
def start(): session, engine = initSession() with MySQLdb.connect('172.16.1.248', 'qiye_dev', 'qiye..dev', '1188ys_resultdb') as cursor: #cursor = db.cursor() limit = 1000 cursor.execute('''SELECT count(*) from meishi_recipe''') rowCount = cursor.fetchone()[0] runtimes = math.ceil(rowCount/limit) for x in xrange(0, int(runtimes)): sql = "SELECT taskid, result from meishi_recipe limit %d, %d" % (int(x*limit ), int(limit)) cursor.execute(sql) allRows = cursor.fetchall() i = 0 for row in allRows: try: itemObj = json.loads(row[1]) recipe = searchAndSave(session, itemObj) saveRecipeInfo(session, itemObj, recipe) saveCategoryRelation(session, itemObj, recipe) i += 1 if i % 200 == 0: session.commit() except Exception, e: traceback.print_exc(file=sys.stdout) myLogger.error('Error: %s - taskid : %s',e, row[0]) #sys.exit() #logging.error('task id is %s, eval error ', row[0]) session.commit()
def start(): session, engine = initSession() with MySQLdb.connect('172.16.1.248', 'qiye_dev', 'qiye..dev', '1188ys_resultdb') as cursor: #cursor = db.cursor() cursor.execute('''SELECT taskid, result from meishi_collection''') allRows = cursor.fetchall() i = 0 for row in allRows: try: itemObj = ast.literal_eval(row[1]) searchAndSave(session, itemObj) i += 1 if i % 200 == 0: session.commit() except Exception, e: #traceback.print_exc(file=sys.stdout) myLogger.error('Error: %s - taskid : %s',e, row[0]) #sys.exit() #logging.error('task id is %s, eval error ', row[0]) session.commit()
def start(): session, engine = initSession() with MySQLdb.connect('172.16.1.248', 'qiye_dev', 'qiye..dev', '1188ys_resultdb') as cursor: #cursor = db.cursor() cursor.execute('''SELECT taskid, result from meishi_collection''') allRows = cursor.fetchall() i = 0 for row in allRows: try: itemObj = ast.literal_eval(row[1]) searchAndSave(session, itemObj) i += 1 if i % 200 == 0: session.commit() except Exception, e: #traceback.print_exc(file=sys.stdout) myLogger.error('Error: %s - taskid : %s', e, row[0]) #sys.exit() #logging.error('task id is %s, eval error ', row[0]) session.commit()