def fade_add_del_service(addRunningId,delRunningId,msg=""):
time.sleep(5)
session=database.getSession()
for id in addRunningId:
database.update_task(session,id,Task.STATUS_FINISH,Task.RESULT_SUCCESS,msg)
for id in delRunningId:
database.update_task(session,id,Task.STATUS_FINISH,Task.RESULT_SUCCESS,msg)
session.close()
def edit_item(no):
edit = request.POST.task.strip()
status = request.POST.status.strip()
if status == 'Open':
status = 1
else:
status = 0
database.update_task(edit, status, no)
return redirect("/")
def fade_add_del_service(addRunningId, delRunningId, msg=""):
'''
已经弃用
'''
time.sleep(5)
session = database.getSession()
for id in addRunningId:
database.update_task(session, id, Task.STATUS_FINISH,
Task.RESULT_SUCCESS, msg)
for id in delRunningId:
database.update_task(session, id, Task.STATUS_FINISH,
Task.RESULT_SUCCESS, msg)
session.close()
def update(id):
if request.method == 'POST':
new_task = request.form.get('content')
if not new_task:
return render_template('apology.html', msg='No task provided!'), 404
database.update_task(id, new_task)
return redirect('/')
else:
task = database.get_task(task_id=id)
return render_template('update.html', task=task)
def update_task(task_id):
task = database.get_task(CONNECTION, task_id)
print(task)
if len(task) == 0:
abort(404)
if not request.json:
abort(400)
if 'title' in request.json and not isinstance(request.json['title'], str):
abort(400)
if 'description' in request.json and \
not isinstance(request.json['description'], str):
abort(400)
if 'done' in request.json and type(request.json['done']) is not bool:
abort(400)
task[0]['title'] = request.json.get('title', task[0]['title'])
task[0]['description'] = request.json.get('description',
task[0]['description'])
task[0]['status'] = request.json.get('status', task[0]['status'])
database.update_task(CONNECTION, task[0])
return jsonify({'task': make_public_task(task[0])})
def update_task(task_id):
if not request.json:
abort(400)
if 'done' in request.json and type(request.json['done']) is not bool:
abort(400)
task = {
"i": str(task_id),
"done": request.json["done"]
}
try:
response = db.update_task(task, task_id)
except:
abort(404)
return jsonify(make_public(response))
def start_crawl(userid, target, nums, taskid):
is_success, outputs = get_username(userid)
if not is_success:
print("get userinfo failed. exit!")
return False, "crawl failed"
twitter_auth_file = os.path.join(RES_DIR, str(userid), outputs[3])
google_auth_file = os.path.join(RES_DIR, str(userid), outputs[4])
print("authorize Twitter...")
api = twitter.twitter_OAuth_login(twitter_auth_file)
print("start grabbing tweets...")
# Twitter can only return maximum 3000+ tweets
twitter.download_tweets(api, target, int(nums))
print("start downloading images...")
urls = twitter.extract_images_url(file_name=target + '_tweets.json')
image_count = len(urls)
twitter.download_images(target, urls)
images_src = os.path.join('./download_images', target)
images_dst = os.path.join(images_src, 'add_labels')
os.mkdir(images_dst)
image_client = GVision.get_image_client(google_auth_file)
GVision.draw_labels_on_images(image_client, images_src, images_dst)
selfFFMPEG.convert_images_to_video(images_dst, images_src)
image_location = images_src
video_location = images_src
# update task info
is_success, outputs = update_task({'id': taskid,
'image': image_location,
'video': video_location,
'image_count': image_count})
if not is_success:
print(outputs)
return False, "crawl failed"
return True, "crawl success"
def add_host(asyncId, hosts, login):
#保留10秒后清除
session = database.getSession()
idMap = {}
progress = 0
progressStep = int((100 - 10) / len(hosts))
#创建任务
for hostName in hosts:
taskId = database.build_task(session, "async", "add_host", hostName,
"machine", "connect")
idMap[hostName] = taskId
async_push(asyncId, "progressMsg", "build task")
progress = 10
async_set(asyncId, "progress", progress)
#交给ansible运行
async_push(
asyncId, "progressMsg",
"ansible connect and install yum python json.As the yum update automatically,it could be long..."
)
progress = 20
async_set(asyncId, "progress", progress)
newlogin = get_default_login(session, login)
# (user,port,passwd,sudopasswd)
for host in hosts:
taskId = idMap[host]
database.update_task(session, taskId, Task.STATUS_RUNNING,
Task.RESULT_UNFINISH, "")
#即使不实际执行ansible
#也可以获取相关的信息
#要求信息是准确可用的
import ansible_lib
ret = ansible_lib.connect_host(hosts, newlogin[0], newlogin[1],
newlogin[2], newlogin[3])
#假连接
#ret=fade_connect_host(hosts,newlogin[0],newlogin[1],newlogin[2],newlogin[3])
app_log.info("connect with" + str(hosts) + " with :" + str(newlogin))
#处理结果
success = []
failed = []
prepareTaskids = []
for hostName in ret:
taskId = idMap[hostName]
(result, msg, info) = ret[hostName]
if result:
#成功连接，添加到host表
success.append(hostName)
host = Host(hostName, Host.STATUS_CONNECTED)
host.ip = info['ip']
host.cpu = info['cpu']
host.mem = info['mem']
host.disk = info['disk']
host.rack = info['rack']
session.add(host)
session.commit()
if config.install_manager:
#提交一个prepare任务,假如运行成功将修改状态到ready
prepareTaskid = database.build_task(session, "ansible",
"prepare", hostName,
"prepare", "prepare")
prepareTaskids.append(prepareTaskid)
callback_lib.add_callback(session, prepareTaskid,
"changeHostToReady")
#假如登录信息跟默认的不一致，保存这些登录信息到数据库
save_login(session, hostName, login)
#更新任务状态
database.update_task(session, taskId, Task.STATUS_FINISH,
Task.RESULT_SUCCESS, "")
progress += progressStep
async_push(asyncId, "progressMsg", "finish connect " + hostName)
async_set(asyncId, "progress", progress)
else:
failed.append(hostName)
database.update_task(session, taskId, Task.STATUS_FINISH,
Task.RESULT_FAILED, msg)
#全部完成
async_push(asyncId, "progressMsg",
"success:" + ",".join(success) + " failed:" + ",".join(failed))
async_set(asyncId, "progress", "100")
#发送消息到worker
retMsg = ""
msg = ','.join([str(id) for id in prepareTaskids])
if not mm.send(msg):
retMsg = "send message to worker error"
#10秒后清除内存任务
time.sleep(10)
async_remove(asyncId)
session.close()
def button(bot, update):
query = update.callback_query
reply = ast.literal_eval(query.data)
print 'BUTTON:', reply
if reply.get('action') == 'update':
update_task(query.from_user, reply)
if 'time' in reply:
bot.editMessageText(text="Ок, %s" %
get_time(reply['time']).encode('utf8'),
chat_id=query.message.chat_id,
message_id=query.message.message_id)
def create_location_button(num):
return InlineKeyboardButton(
get_location(num),
callback_data="{'action':'update', 'id':%s, 'location':%s}"
% (reply['id'], num))
keyboard = [[create_location_button(i) for i in range(1, 5)]]
reply_markup = InlineKeyboardMarkup(keyboard)
bot.sendMessage(chat_id=query.message.chat_id,
text="Это где?",
reply_markup=reply_markup)
if 'location' in reply:
bot.editMessageText(text="Ок, %s" %
get_location(reply['location']).encode('utf8'),
chat_id=query.message.chat_id,
message_id=query.message.message_id)
if reply.get('action') == 'show':
number_of_tasks, task_id, task = get_task_from_db(
query.from_user, reply.get('time', 0), reply.get('location', 0),
reply.get('number', 0))
reply['number'] = reply.get('number', 0) + 1
if number_of_tasks == 0:
bot.sendMessage(chat_id=query.message.chat_id, text='Таких нет!')
return
if reply['number'] >= number_of_tasks:
reply['number'] = 0
button_name = 'Давай сначала!'
else:
button_name = 'Следущую!'
keyboard = [[
InlineKeyboardButton(
'Делаю!',
callback_data="{'action':'update', 'id':%s, 'status':1}" %
task_id),
InlineKeyboardButton(button_name, callback_data=str(reply)),
InlineKeyboardButton('Удалить!',
callback_data="{'action':'delete', 'id':%s}" %
task_id)
]]
reply_markup = InlineKeyboardMarkup(keyboard)
bot.sendMessage(chat_id=query.message.chat_id,
text=task,
reply_markup=reply_markup)
if reply.get('action') == 'delete':
delete_task(reply.get('id'))
def add_host(asyncId,hosts,login):
#保留10秒后清除
session=database.getSession()
idMap = {}
progress = 0 ;
progressStep = int( (100-10)/len(hosts) )
#创建任务
for hostName in hosts:
taskId = database.build_task(session,"async","add_host",hostName,"machine","connect")
idMap[hostName] = taskId
async_push(asyncId,"progressMsg","build task")
progress = 10
async_set(asyncId,"progress",progress)
#交给ansible运行
async_push(asyncId,"progressMsg","ansible connect and install yum python json.As the yum update automatically,it could be long...")
progress = 20
async_set(asyncId,"progress",progress)
newlogin = get_default_login(session, login)
# (user,port,passwd,sudopasswd)
for host in hosts:
taskId = idMap[host]
database.update_task(session,taskId,Task.STATUS_RUNNING,Task.RESULT_UNFINISH,"")
if config.fade_windows:
ret=fade_connect_host(hosts,newlogin[0],newlogin[1],newlogin[2],newlogin[3])
else:
import ansible_lib
ret=ansible_lib.connect_host(hosts,newlogin[0],newlogin[1],newlogin[2],newlogin[3])
app_log.info("connect with"+str(hosts)+" with :"+str(newlogin) )
#处理结果
success=[]
failed=[]
prepareTaskids=[]
for hostName in ret:
taskId = idMap[hostName]
(result,msg,info) = ret[hostName]
if result:
#成功连接，添加到host表
success.append(hostName)
host = Host(hostName,Host.STATUS_CONNECTED)
host.ip=info['ip']
host.cpu=info['cpu']
host.mem=info['mem']
host.disk=info['disk']
host.rack=info['rack']
session.add(host)
session.commit()
if config.install_manager:
#提交一个prepare任务,假如运行成功将修改状态到ready
prepareTaskid = database.build_task(session,"ansible","prepare",hostName,"prepare","prepare")
prepareTaskids.append(prepareTaskid)
callback_lib.add_callback(session,prepareTaskid,"changeHostToReady")
#假如登录信息跟默认的不一致，保存这些登录信息到数据库
save_login(session, hostName, login)
#更新任务状态
database.update_task(session,taskId,Task.STATUS_FINISH,Task.RESULT_SUCCESS,"")
progress += progressStep
async_push(asyncId,"progressMsg","finish connect "+hostName)
async_set(asyncId,"progress",progress)
else:
failed.append(hostName)
database.update_task(session,taskId,Task.STATUS_FINISH,Task.RESULT_FAILED,msg)
#全部完成
async_push(asyncId,"progressMsg","success:"+",".join(success)+" failed:"+",".join(failed))
async_set(asyncId,"progress","100")
#发送消息到worker
retMsg = ""
msg = ','.join([str(id) for id in prepareTaskids])
if not mm.send(msg):
retMsg = "send message to worker error"
#10秒后清除内存任务
time.sleep(10)
async_remove(asyncId)
session.close()