Example #1
0
File: async.py Project: caimaoy/uhp
def fade_add_del_service(addRunningId,delRunningId,msg=""):
    time.sleep(5)
    session=database.getSession()
    for id in addRunningId:
        database.update_task(session,id,Task.STATUS_FINISH,Task.RESULT_SUCCESS,msg)
    for id in delRunningId:
        database.update_task(session,id,Task.STATUS_FINISH,Task.RESULT_SUCCESS,msg)
    session.close()
Example #2
0
def edit_item(no):
    edit = request.POST.task.strip()
    status = request.POST.status.strip()
    if status == 'Open':
        status = 1
    else:
        status = 0
    database.update_task(edit, status, no)
    return redirect("/")
Example #3
0
def fade_add_del_service(addRunningId, delRunningId, msg=""):
    '''
    已经弃用
    '''
    time.sleep(5)
    session = database.getSession()
    for id in addRunningId:
        database.update_task(session, id, Task.STATUS_FINISH,
                             Task.RESULT_SUCCESS, msg)
    for id in delRunningId:
        database.update_task(session, id, Task.STATUS_FINISH,
                             Task.RESULT_SUCCESS, msg)
    session.close()
Example #4
0
def update(id):
    if request.method == 'POST':
        new_task = request.form.get('content')

        if not new_task:
            return render_template('apology.html', msg='No task provided!'), 404

        database.update_task(id, new_task)
        return redirect('/')

    else:
        task = database.get_task(task_id=id)
        return render_template('update.html', task=task)
Example #5
0
def update_task(task_id):
    task = database.get_task(CONNECTION, task_id)
    print(task)
    if len(task) == 0:
        abort(404)
    if not request.json:
        abort(400)
    if 'title' in request.json and not isinstance(request.json['title'], str):
        abort(400)
    if 'description' in request.json and \
            not isinstance(request.json['description'], str):
        abort(400)
    if 'done' in request.json and type(request.json['done']) is not bool:
        abort(400)
    task[0]['title'] = request.json.get('title', task[0]['title'])
    task[0]['description'] = request.json.get('description',
                                              task[0]['description'])
    task[0]['status'] = request.json.get('status', task[0]['status'])
    database.update_task(CONNECTION, task[0])
    return jsonify({'task': make_public_task(task[0])})
Example #6
0
def update_task(task_id):
	if not request.json:
		abort(400)
	if 'done' in request.json and type(request.json['done']) is not bool:
		abort(400)

	task = {
		"i": str(task_id),
		"done": request.json["done"]
	}
	try:
		response = db.update_task(task, task_id)
	except:
		abort(404)

	return jsonify(make_public(response))
Example #7
0
def start_crawl(userid, target, nums, taskid):
    is_success, outputs = get_username(userid)
    if not is_success:
        print("get userinfo failed. exit!")
        return False, "crawl failed"
    twitter_auth_file = os.path.join(RES_DIR, str(userid), outputs[3])
    google_auth_file = os.path.join(RES_DIR, str(userid), outputs[4])

    print("authorize Twitter...")
    api = twitter.twitter_OAuth_login(twitter_auth_file)
    print("start grabbing tweets...")
    # Twitter can only return maximum 3000+ tweets
    twitter.download_tweets(api, target, int(nums))
    print("start downloading images...")
    urls = twitter.extract_images_url(file_name=target + '_tweets.json')
    image_count = len(urls)
    twitter.download_images(target, urls)

    images_src = os.path.join('./download_images', target)
    images_dst = os.path.join(images_src, 'add_labels')
    os.mkdir(images_dst)
    image_client = GVision.get_image_client(google_auth_file)
    GVision.draw_labels_on_images(image_client, images_src, images_dst)

    selfFFMPEG.convert_images_to_video(images_dst, images_src)
    image_location = images_src
    video_location = images_src

    # update task info
    is_success, outputs = update_task({'id': taskid,
                                       'image': image_location,
                                       'video': video_location,
                                       'image_count': image_count})
    if not is_success:
        print(outputs)
        return False, "crawl failed"

    return True, "crawl success"
Example #8
0
def add_host(asyncId, hosts, login):
    #保留10秒后清除
    session = database.getSession()
    idMap = {}
    progress = 0
    progressStep = int((100 - 10) / len(hosts))

    #创建任务
    for hostName in hosts:
        taskId = database.build_task(session, "async", "add_host", hostName,
                                     "machine", "connect")
        idMap[hostName] = taskId

    async_push(asyncId, "progressMsg", "build task")
    progress = 10
    async_set(asyncId, "progress", progress)

    #交给ansible运行
    async_push(
        asyncId, "progressMsg",
        "ansible connect and install yum python json.As the yum update automatically,it could be long..."
    )
    progress = 20
    async_set(asyncId, "progress", progress)
    newlogin = get_default_login(session, login)
    #     (user,port,passwd,sudopasswd)
    for host in hosts:
        taskId = idMap[host]
        database.update_task(session, taskId, Task.STATUS_RUNNING,
                             Task.RESULT_UNFINISH, "")

    #即使不实际执行ansible
    #也可以获取相关的信息
    #要求信息是准确可用的
    import ansible_lib
    ret = ansible_lib.connect_host(hosts, newlogin[0], newlogin[1],
                                   newlogin[2], newlogin[3])
    #假连接
    #ret=fade_connect_host(hosts,newlogin[0],newlogin[1],newlogin[2],newlogin[3])
    app_log.info("connect with" + str(hosts) + " with :" + str(newlogin))

    #处理结果
    success = []
    failed = []
    prepareTaskids = []
    for hostName in ret:
        taskId = idMap[hostName]
        (result, msg, info) = ret[hostName]
        if result:
            #成功连接,添加到host表

            success.append(hostName)
            host = Host(hostName, Host.STATUS_CONNECTED)
            host.ip = info['ip']
            host.cpu = info['cpu']
            host.mem = info['mem']
            host.disk = info['disk']
            host.rack = info['rack']
            session.add(host)
            session.commit()
            if config.install_manager:
                #提交一个prepare任务,假如运行成功将修改状态到ready
                prepareTaskid = database.build_task(session, "ansible",
                                                    "prepare", hostName,
                                                    "prepare", "prepare")
                prepareTaskids.append(prepareTaskid)
                callback_lib.add_callback(session, prepareTaskid,
                                          "changeHostToReady")
            #假如登录信息跟默认的不一致,保存这些登录信息到数据库
            save_login(session, hostName, login)

            #更新任务状态
            database.update_task(session, taskId, Task.STATUS_FINISH,
                                 Task.RESULT_SUCCESS, "")
            progress += progressStep
            async_push(asyncId, "progressMsg", "finish connect " + hostName)
            async_set(asyncId, "progress", progress)
        else:
            failed.append(hostName)
            database.update_task(session, taskId, Task.STATUS_FINISH,
                                 Task.RESULT_FAILED, msg)

    #全部完成
    async_push(asyncId, "progressMsg",
               "success:" + ",".join(success) + " failed:" + ",".join(failed))
    async_set(asyncId, "progress", "100")
    #发送消息到worker
    retMsg = ""
    msg = ','.join([str(id) for id in prepareTaskids])
    if not mm.send(msg):
        retMsg = "send message to worker error"
    #10秒后清除内存任务
    time.sleep(10)
    async_remove(asyncId)
    session.close()
Example #9
0
def button(bot, update):
    query = update.callback_query

    reply = ast.literal_eval(query.data)
    print 'BUTTON:', reply
    if reply.get('action') == 'update':
        update_task(query.from_user, reply)
        if 'time' in reply:
            bot.editMessageText(text="Ок, %s" %
                                get_time(reply['time']).encode('utf8'),
                                chat_id=query.message.chat_id,
                                message_id=query.message.message_id)

            def create_location_button(num):
                return InlineKeyboardButton(
                    get_location(num),
                    callback_data="{'action':'update', 'id':%s, 'location':%s}"
                    % (reply['id'], num))

            keyboard = [[create_location_button(i) for i in range(1, 5)]]
            reply_markup = InlineKeyboardMarkup(keyboard)

            bot.sendMessage(chat_id=query.message.chat_id,
                            text="Это где?",
                            reply_markup=reply_markup)

        if 'location' in reply:
            bot.editMessageText(text="Ок, %s" %
                                get_location(reply['location']).encode('utf8'),
                                chat_id=query.message.chat_id,
                                message_id=query.message.message_id)

    if reply.get('action') == 'show':

        number_of_tasks, task_id, task = get_task_from_db(
            query.from_user, reply.get('time', 0), reply.get('location', 0),
            reply.get('number', 0))
        reply['number'] = reply.get('number', 0) + 1
        if number_of_tasks == 0:
            bot.sendMessage(chat_id=query.message.chat_id, text='Таких нет!')
            return
        if reply['number'] >= number_of_tasks:
            reply['number'] = 0
            button_name = 'Давай сначала!'
        else:
            button_name = 'Следущую!'
        keyboard = [[
            InlineKeyboardButton(
                'Делаю!',
                callback_data="{'action':'update', 'id':%s, 'status':1}" %
                task_id),
            InlineKeyboardButton(button_name, callback_data=str(reply)),
            InlineKeyboardButton('Удалить!',
                                 callback_data="{'action':'delete', 'id':%s}" %
                                 task_id)
        ]]
        reply_markup = InlineKeyboardMarkup(keyboard)
        bot.sendMessage(chat_id=query.message.chat_id,
                        text=task,
                        reply_markup=reply_markup)

    if reply.get('action') == 'delete':

        delete_task(reply.get('id'))
Example #10
0
File: async.py Project: caimaoy/uhp
def add_host(asyncId,hosts,login):  
    #保留10秒后清除
    session=database.getSession()
    idMap = {}
    progress = 0 ;
    progressStep = int( (100-10)/len(hosts) )
    
    #创建任务
    for hostName in hosts:
        taskId = database.build_task(session,"async","add_host",hostName,"machine","connect")
        idMap[hostName] = taskId
    
    async_push(asyncId,"progressMsg","build task")
    progress = 10
    async_set(asyncId,"progress",progress) 

    #交给ansible运行
    async_push(asyncId,"progressMsg","ansible connect and install yum python json.As the yum update automatically,it could be long...")
    progress = 20
    async_set(asyncId,"progress",progress) 
    newlogin = get_default_login(session, login)
#     (user,port,passwd,sudopasswd) 
    for host in hosts:
        taskId = idMap[host] 
        database.update_task(session,taskId,Task.STATUS_RUNNING,Task.RESULT_UNFINISH,"") 
    
    if config.fade_windows:
        ret=fade_connect_host(hosts,newlogin[0],newlogin[1],newlogin[2],newlogin[3])
    else:
        import ansible_lib
        ret=ansible_lib.connect_host(hosts,newlogin[0],newlogin[1],newlogin[2],newlogin[3])
    app_log.info("connect with"+str(hosts)+" with :"+str(newlogin) )
        
    #处理结果
    success=[]
    failed=[]
    prepareTaskids=[]
    for hostName in ret:
        taskId = idMap[hostName]
        (result,msg,info) = ret[hostName]
        if result:
            #成功连接,添加到host表
            
            success.append(hostName)
            host = Host(hostName,Host.STATUS_CONNECTED)
            host.ip=info['ip']
            host.cpu=info['cpu']
            host.mem=info['mem']
            host.disk=info['disk']
            host.rack=info['rack']
            session.add(host)
            session.commit()
            if config.install_manager:
                #提交一个prepare任务,假如运行成功将修改状态到ready
                prepareTaskid = database.build_task(session,"ansible","prepare",hostName,"prepare","prepare")
                prepareTaskids.append(prepareTaskid)
                callback_lib.add_callback(session,prepareTaskid,"changeHostToReady")
            #假如登录信息跟默认的不一致,保存这些登录信息到数据库
            save_login(session, hostName, login)
            
            #更新任务状态
            database.update_task(session,taskId,Task.STATUS_FINISH,Task.RESULT_SUCCESS,"")
            progress += progressStep
            async_push(asyncId,"progressMsg","finish connect "+hostName)
            async_set(asyncId,"progress",progress)
        else:
            failed.append(hostName)
            database.update_task(session,taskId,Task.STATUS_FINISH,Task.RESULT_FAILED,msg)

    #全部完成
    async_push(asyncId,"progressMsg","success:"+",".join(success)+" failed:"+",".join(failed))
    async_set(asyncId,"progress","100")   
    #发送消息到worker
    retMsg = ""
    msg = ','.join([str(id) for id in prepareTaskids])
    if not mm.send(msg):
        retMsg = "send message to worker error"
    #10秒后清除内存任务
    time.sleep(10)
    async_remove(asyncId)
    session.close()