def start(kwargs):
session, engine = initSession()
with MySQLdb.connect(kwargs['db_host'], kwargs['db_user'], kwargs['db_pass'], '1188ys_resultdb') as cursor:
#cursor = db.cursor()
limit = 1000
cursor.execute('''SELECT count(*) from meishi_recipe''')
rowCount = cursor.fetchone()[0]
runtimes = math.ceil(rowCount/limit)
for x in xrange(0, int(runtimes)):
sql = "SELECT taskid, result from meishi_recipe limit %d, %d" % (int(x*limit ), int(limit))
cursor.execute(sql)
allRows = cursor.fetchall()
i = 0
for row in allRows:
try:
itemObj = json.loads(row[1])
recipe = searchAndSave(session, itemObj)
saveRecipeInfo(session, itemObj, recipe)
saveCategoryRelation(session, itemObj, recipe)
i += 1
if i % 200 == 0:
session.commit()
except Exception, e:
traceback.print_exc(file=sys.stdout)
myLogger.error('Error: %s - taskid : %s',e, row[0])
#sys.exit()
#logging.error('task id is %s, eval error ', row[0])
session.commit()
def start(kwargs):
session, engine = initSession()
with MySQLdb.connect(kwargs['db_host'], kwargs['db_user'],
kwargs['db_pass'], '1188ys_resultdb') as cursor:
#cursor = db.cursor()
limit = 1000
cursor.execute('''SELECT count(*) from meishi_recipe''')
rowCount = cursor.fetchone()[0]
runtimes = math.ceil(rowCount / limit)
for x in xrange(0, int(runtimes)):
sql = "SELECT taskid, result from meishi_recipe limit %d, %d" % (
int(x * limit), int(limit))
cursor.execute(sql)
allRows = cursor.fetchall()
i = 0
for row in allRows:
try:
itemObj = json.loads(row[1])
recipe = searchAndSave(session, itemObj)
saveRecipeInfo(session, itemObj, recipe)
saveCategoryRelation(session, itemObj, recipe)
i += 1
if i % 200 == 0:
session.commit()
except Exception, e:
traceback.print_exc(file=sys.stdout)
myLogger.error('Error: %s - taskid : %s', e, row[0])
#sys.exit()
#logging.error('task id is %s, eval error ', row[0])
session.commit()
def start(kwargs):
session, engine = initSession()
with MySQLdb.connect(kwargs['db_host'], kwargs['db_user'],
kwargs['db_pass'], '1188ys_resultdb') as cursor:
#cursor = db.cursor()
cursor.execute('''SELECT taskid, result from meishi_category''')
allRows = cursor.fetchall()
i = 0
for row in allRows:
try:
itemObjs = ast.literal_eval(row[1])
searchAndSaveCategory(session, itemObjs)
i += 1
if i % 200 == 0:
session.commit()
except Exception, e:
myLogger.error('Error: %s - taskid : %s', e, row[0])
#logging.error('task id is %s, eval error ', row[0])
session.commit()
def start(kwargs):
session, engine = initSession()
with MySQLdb.connect(kwargs['db_host'], kwargs['db_user'], kwargs['db_pass'], '1188ys_resultdb') as cursor:
#cursor = db.cursor()
cursor.execute('''SELECT taskid, result from meishi_material''')
allRows = cursor.fetchall()
i = 0
for row in allRows:
try:
itemObj = ast.literal_eval(row[1])
searchAndSaveMaterial(session, itemObj)
i += 1
if i % 200 == 0:
session.commit()
except Exception, e:
#traceback.print_exc(file=sys.stdout)
myLogger.error('Error: %s - taskid : %s',e, row[0])
#sys.exit()
#logging.error('task id is %s, eval error ', row[0])
session.commit()
