예제 #1
0
 def get(self, file):
     logger.info('get [{0}].[{1}] to [{2}]'.format(self.remote_path, file,
                                                   self.local_path))
     if self.local_path:
         local_file = '{0}/{1}'.format(self.local_path, file)
     self.client.get(file, local_file)
     return self
예제 #2
0
    def eval_k(self, algorithm, topks=None, measures=None):
        """评估"""
        if measures is None:
            measures = ["rmse", "mae"]
        logger.info('algorithm:' + algorithm.__class__.__name__)

        eval_results = []
        cur = 0
        for train_set, test_set in self.cross_validation():
            cur += 1
            algorithm.train(train_set)

            if self.save_sim:
                temp_path = '{}/sim/{}/'.format(self.output_path,
                                                algorithm.__class__.__name__)
                if not os.path.exists(temp_path):
                    os.makedirs(temp_path)

                np.savetxt('{}/fold{}.csv'.format(temp_path, cur),
                           algorithm.sim,
                           fmt='%0.3f')

            eval_result_fold = []
            for k in topks:
                logger.info('k={}'.format(k))
                algorithm.topk = k
                eval_result_fold.append(algorithm.estimate(test_set, measures))
            eval_results.append(eval_result_fold)

            if self.just_test_one:
                break
        self.show_result_k(algorithm, measures, eval_results, topks)
예제 #3
0
def teardown():
    response = api.query({'ksql': 'show queries;'})
    for query in response.json()[0]['queries']:
        api.query({'ksql': 'terminate {};'.format(query['id'])})
    for entry in stmts_teardown:
        logger.info(entry['stmt'])
        api.query({'ksql': entry['stmt']}, force_exit=False, show_output=False)
예제 #4
0
    def estimate(self, raw_test_dataset, measures):
        with tl.Timer() as t:
            error = self._estimate(raw_test_dataset, measures)

        logger.info("{} algorithm predict process cost {:.3f} sec".format(
            self.__class__.__name__, t.interval))
        return error
예제 #5
0
def update_feed(feedid, rset, itemlist: list):
    # cleanup list of new feeds
    _itemlist = itemlist.copy()  # list of new item's links
    olditems = Item.query.filter(
        Item.feedid == feedid).all()  # list of old items in feed
    for iter in olditems:
        if str(iter.link) in _itemlist: _itemlist.remove(iter.link)

    # 새 feed가 없고, 이전에도 feed가 없었던 경우
    if (len(_itemlist) == 0) and (len(itemlist) == len(olditems)):
        return False

    # delete all old feed items
    # todo: feed별 삭제여부를 지정할 수 있도록 할 것
    logger.debug("remove all old items in feed: {}".format(feedid))
    for olditem in olditems:
        db_session.delete(olditem)

    count = 1
    for iter in itemlist:
        logger.info("processing...: {count} / {length}".format_map({
            'count':
            count,
            'length':
            len(itemlist)
        }))
        count = count + 1

        feeditem = update_feeditem(feedid, iter, rset)
        if feeditem:
            db_session.add(feeditem)
    db_session.commit()
    return True
예제 #6
0
 def ssh(self):
     self.client.connect(hostname=self.host_config['hostname'],
                         port=self.host_config['port'],
                         username=self.host_config['username'],
                         password=self.host_config['password'])
     logger.info('login to {0} -> {1}'.format(self.host_config, 'success'))
     return self
예제 #7
0
 def prerun(self, t, verbose=True):
     if verbose:
         msg = "=============\nAlgorithm Time: {}\nCash: {}\nPositions: {}\n".format(
             t, self.get_cash(), self.get_positions()
         )
         logger.info(msg)
     self.t = t
예제 #8
0
def create_and_add_reply(replyable, response_url, hero_id):
    """Method that creates a reply in reddit format and adds the reply to comment/submission.
    The reply consists of a link to the response audio file, the response itself, a warning about the sound
    and an ending added from the config file (post footer).
    
    Image is currently ignored due to new reddit redesign not rendering flairs properly.

    :param replyable: The comment/submission on reddit
    :param response_url: The url to the response audio file
    :param hero_id: The hero_id to which the response belongs to.
    :return: The text for the comment reply.
    """
    original_text = replyable.body if isinstance(replyable,
                                                 Comment) else replyable.title
    original_text = original_text.strip()

    if '>' in original_text:
        original_text = get_quoted_text(original_text).strip()
    if '::' in original_text:
        original_text = original_text.split('::', 1)[1].strip()

    hero_name = db_api.get_hero_name(hero_id)

    reply = "[{}]({}) (sound warning: {}){}".format(original_text,
                                                    response_url, hero_name,
                                                    config.COMMENT_ENDING)
    replyable.reply(reply)

    logger.info("Replied to: " + replyable.fullname)
예제 #9
0
def update_reply(replyable, response_info):
    """Method to edit and update existing response comment by the bot with a new hero as requested.

    :param replyable: The comment/submission on reddit
    :param response_info: ResponseInfo containing hero_id and link for response
    :return: None
    """
    bot_comment = replyable.parent()
    root_replyable = bot_comment.parent()

    # TODO maybe get original text from bot's command, rather than the original post, as it might be edited by the time this command is called
    original_text = root_replyable.body if isinstance(
        root_replyable, Comment) else root_replyable.title
    original_text = original_text.strip()

    if '>' in original_text:
        original_text = get_quoted_text(original_text).strip()

    # Getting name with Proper formatting
    hero_name = db_api.get_hero_name(response_info.hero_id)

    reply = "[{}]({}) (sound warning: {}){}".format(original_text,
                                                    response_info.link,
                                                    hero_name,
                                                    config.COMMENT_ENDING)
    bot_comment.edit(reply)

    logger.info("Updated Reply: " + replyable.fullname)
예제 #10
0
	def check_duration(self, step, duration):
		try:
			if (duration > step.deadline):
				logger.info('Sync step %s missed deadline, took %.2f seconds'%(step.name,duration))
		except AttributeError:
			# S doesn't have a deadline
			pass
예제 #11
0
def login_user(userid, password):
    logger.info("User Login")
    user_data = User.query.filter_by(userid=userid).first()

    if user_data is not None:
        if not user_data.check_password(password):
            logger.error("Authentication error: Wrong userid or password")
            return {
                'message': 'Authentication error: Wrong userid or password',
                "authenticated": False
            }, 401
        # Set JWT_ACCESS_TOKEN_EXPIRES to change token duration.
        access_token = create_access_token(identity=user_data)
        refresh_token = create_refresh_token(identity=user_data)
        logger.info("Access token created")
        logger.debug(f'access_token: {access_token}')
        resp = {
            'login': True,
            'msg': 'New login',
            'access_token': access_token,
            #'refresh_token': refresh_token
        }
        _set_cookies(access_token, refresh_token)
        return resp, 200
    else:
        logger.error("User Does Not Exist")
        return {'message': 'User Does Not Exist', "authenticated": False}, 401
예제 #12
0
    def register_execution(self, seconds_from_now: str, start_at: str):

        start_time = compute_start_time(seconds_from_now, start_at)
        start_time_ms = start_time * 1000

        for ID in self.active_workers_ids():
            url = f"{self._workers[ID]['baseURL']}/worker/execution?start_time={start_time_ms}"
            try:
                rs = requests.post(url)
                logger.info(f"POST {url} -> {rs.status_code}")
            except RequestException as e:
                logger.error(e)
                self.stop()
                raise e

        self._sequence_scheduler.register_task(0, init_mixer)

        audio_file = self.sequence_file('ogg')
        self._sequence_scheduler.register_task(
            1, lambda: load_audio_file(audio_file))

        register_countdown(
            lambda delay_seconds, fun: self._sequence_scheduler.register_task(
                start_time_ms - PLAYBACK_LATENCY +
                (delay_seconds * 1000), fun))

        self._sequence_scheduler.register_task(
            start_time_ms - PLAYBACK_LATENCY, play_music)
        super().register_tasks(start_time_ms)

        self._active_sequence['start_time'] = start_time
예제 #13
0
def batch_consumers(fi_id, generator, q, batch_limit=PER_THREAD_BATCH_LIMIT):
    logger.info("Starting API batching for FI {}".format(fi_id))

    # iterate over each consumer
    num_batches = 0

    for batch in generator:
        api_batch = []
        num_batches += 1
        # TODO: Modify generator SQL to name fields instead of iterating over them all
        for row in batch:
            api_batch.append(map_row_to_marketingcloud_api(row, fi_id))

        # send to MC when we have gathered enough for batch limit

        while True:
            try:
                q.put({
                    "batch_num": num_batches,
                    "fi_id": fi_id,
                    "data": api_batch
                }, False)
                break
            except Full:
                sleep(1)
            except Exception:
                logger.exception('Failed to put data on the worker queue')
            
        logger.debug("Queuing batch {} for FI {} from thread {}".format(num_batches, fi_id,
                                                                       threading.current_thread().getName()))

    logger.info(f"Completed delivery of marketing cloud messages for FI {fi_id}, {num_batches} were delivered")
예제 #14
0
def set_position():
    logger.info('Update absolute camera position')
    try:
        data = request.get_json()
        x = data.get('x')
        y = data.get('y')
        z = data.get('z')
        logger.info('newpos: ', {"x": x, "y": y, "z": z})
        resp = requests.get(
            f'http://{DEVICE_IP}/isp/appispmu.cgi?btOK=submit&i_mt_dirx={x}&i_mt_diry={y}&i_mt_dirz={z}',
            auth=HTTPDigestAuth(DEVICE_ID, DEVICE_PW))
        # logger.info(resp.text)
        if resp.status_code == 200:
            return {
                'message': 'Successfully updated camera position.',
                'result': {
                    "x": x,
                    "y": y,
                    "z": z
                }
            }, 200
        else:
            raise CGIException(resp)
    except Exception as e:
        logger.error(e)
        logger.debug(traceback.format_exc())
        raise e
예제 #15
0
    def run(self):
        threads = []

        for name, producer_func in self.producers.items():
            self.event_manager.add_event(name)
            thread = threading.Thread(name=name,
                                      target=self.__producer,
                                      args=(producer_func, ))
            threads.append(thread)
            thread.start()

        num = 0
        while num < self.num_consumers:
            thread = threading.Thread(name="consumer_{}".format(num + 1),
                                      target=self.__consumer,
                                      args=(self.consumer_func, ))
            threads.append(thread)
            thread.start()
            num += 1

        for t in threads:
            t.join()

        self.queue.join()

        logger.info(
            f"completed work, {len(self.thread_exception_list)} exceptions were encountered. "
            f"{self.thread_exception_list if len(self.thread_exception_list) > 0 else ''}"
        )
예제 #16
0
    def _create_model(self, input_shape, num_classes):
        model = Sequential()
        model.add(Conv2D(64, kernel_size=(3, 3),
                         activation='relu',
                         input_shape=input_shape, name="conv1_1"))
        model = self.add_conv(model, 1, block_num=1, filter_num=64, kernel_size=(3, 3))
        model.add(MaxPooling2D(pool_size=(2, 2), padding="same"))

        model = self.add_conv(model, 2, block_num=2, filter_num=128, kernel_size=(3, 3))
        model.add(MaxPooling2D(pool_size=(2, 2), padding="same"))

        model = self.add_conv(model, 3, block_num=3, filter_num=256, kernel_size=(3, 3))
        model.add(MaxPooling2D(pool_size=(2, 2), padding="same"))

        model = self.add_conv(model, 3, block_num=4, filter_num=512, kernel_size=(3, 3))
        model.add(MaxPooling2D(pool_size=(2, 2), padding="same"))

        # model = self.add_conv(model, 3, block_num=5, filter_num=512, kernel_size=(3, 3))
        # model.add(MaxPooling2D(pool_size=(2, 2), padding="same"))

        model.add(Flatten())
        # model.add(Dense(4096, activation='relu'))
        # model.add(Dropout(0.5))
        model.add(Dense(4096, activation='relu'))
        model.add(Dropout(0.5))
        model.add(Dense(num_classes, activation='softmax'))
        logger.info("mode.summary:{}".format(model.summary()))
        return model
예제 #17
0
def processMongoData(collection=None, ids=None, date=None):
    '''
    多线程处理mongo数据信息
    :param collection: 表
    :param ids: 对应的ids
    :return: none
    '''
    # key0: 商场id , key2(内层字典的key): neg(负面内容) ,pos(正面的内容),type(商场/店铺)
    commentMap = getMongoData(collection=collection, ids=ids,
                              date=date)  ## 获取mongodb中的评论数据
    if len(commentMap) == 0:
        logger.info('没有获取到数据,param:{0},{1}'.format(collection, ids))
    else:
        for key, value in commentMap.items():
            for commentType in ['pos', 'neg']:
                ## 处理好评数据
                if value is None or value.get(commentType) is None:
                    continue
                result = getAnalyseData(value[commentType])
                if result is None:
                    continue
                ## 将缓存中的前50 跟当前的结果相比,如果当前词出现在缓存中,则直接将该词的出现次数相加。
                ## 如果缓存中不存在,则直接缓存当前值。对所有词根据词频进行排序,取前50 缓存,前50保存到数据库
                result = MergeHistroyFromES(key, value["type"],
                                            const.NEGATIVE_COMMENT, result)
                if result is not None:
                    saveDataToDb(key, value["type"], commentType, result)
                result = None

    commentMap = None  ## 将所有的对象都置空,防止出现内存泄漏的情况
예제 #18
0
    def _evaluate(self, step=None, record=False, idx=None):
        """Run one rollout if in eval mode
        Run num_record_samples rollouts if in train mode
        """
        vids = []
        for i in range(self._config.num_record_samples):
            rollout, info, frames = self._runner.run_episode(is_train=False,
                                                             record=record)

            if record:
                ep_rew = info["rew"]
                ep_success = "s" if info["episode_success"] else "f"
                fname = "{}_step_{:011d}_{}_r_{}_{}.mp4".format(
                    self._config.env,
                    step,
                    idx if idx is not None else i,
                    ep_rew,
                    ep_success,
                )
                self._save_video(fname, frames)
                vids.append(frames)

            if idx is not None:
                break

        logger.info("rollout: %s",
                    {k: v
                     for k, v in info.items() if not "qpos" in k})
        return rollout, info, np.array(vids)
예제 #19
0
 def wrapper(*args, **kwargs):
     start_time = time.time()
     get_str = func(*args, **kwargs)
     end_time = time.time()
     logger.info(
         str(func) + " cost_time: {}".format(end_time - start_time))
     return get_str
예제 #20
0
def get_twitter_data(client, node: str, field: dict):
    for k, v in dict(field).items():
        if not v:
            field.pop(k)

    encoded_field = parse.urlencode(field)
    url = base + node + '?' + encoded_field

    logger.info(f'request to "{url}"')
    logger.debug(
        f'request info : <base : "{base}"> <node: {node}> <encoded_field: {field}>'
    )

    response, data = client.request(url)  # type : dict, bytes
    logger.debug(f'response status : {response["status"]}')

    try:
        if response['status'] == '200':
            ret = json.loads(data.decode('utf-8'))
            # print(ret)
            return ret
    except Exception as e:
        print("ERR:", e)
        logger.error('cannot responsed from : "%s" ' %
                     (response['status'], url))
        logger.error('error message :', e)
        return None
예제 #21
0
 def instantiate(event):
     # Try to parse the event into json.
     try:
         event = json.loads((event.decode()))
     except ValueError:
         return ActionError(
             EventError.instantiate("{}: is not valid json".format(event)))
     # Ensure the event has a type and payload.
     if any(key not in event for key in ("eventType", "payload")):
         return ActionError(
             EventError.instantiate(
                 "Event does not contain an event type or payload"))
     logger.info("New message: {}".format(event["eventType"]))
     # Instantiate the event.
     return {
         EventType.REGISTER_TEST_STRATEGY.value:
         lambda: ActionRegisterTestStrategy(event["payload"]),
         EventType.LIMIT_ORDER.value:
         lambda: ActionLimitOrder(event["payload"]),
         EventType.TICK.value:
         lambda: ActionTick(),
         EventType.END_OF_CHART_DATA.value:
         lambda: ActionEndOfChartData(event["payload"]),
         EventType.NEW_CHART_DATA.value:
         lambda: ActionNewChartData(event["payload"]),
         EventType.ERROR.value:
         lambda: ActionError(event["payload"]),
         EventType.OK.value:
         lambda: ActionOk(),
     }[event["eventType"]]().action_type
예제 #22
0
def update_device(**kwargs):
    logger.info('Update existing device')
    now = datetime.utcnow()
    try:
        query = db.session.query(Device).filter_by(
            serial=kwargs.get('serial')).one()
        if kwargs.get('newserial'):
            query.serial = kwargs.get('newserial')
        if kwargs.get('model'):
            query.model = kwargs.get('model')
        if kwargs.get('company'):
            query.company = db.session.query(Company).filter_by(
                name=kwargs.get('company')).one()
        if kwargs.get('ip'):
            query.ip = kwargs.get('ip')
        if kwargs.get('owner'):
            query.owner = db.session.query(User).filter_by(
                userid=kwargs.get('owner')).one()
        query.edited = now
        query.edited_by = current_user
        db.session.commit()
        return {'message': f'Updated device<{query.serial}> from db.'}, 200
    except Exception as e:
        logger.error(e)
        logger.debug(traceback.format_exc())
        raise e
예제 #23
0
파일: worker.py 프로젝트: bopopescu/lave
 def task_entrance(self, task):
     try:
         with gevent.Timeout(self.workload.timeout):
             self.__func(task)
     except gevent.Timeout:
         self.workload.complete_workload(task, '52', 'NULL')
         logger.info('>>>>>>>>>>>>>> task timeout!' + str(task))
예제 #24
0
 def _train_model(self):
     logger.info('Training -----------')
     for step in range(5000):
         cost = self.model.train_on_batch(self.train_data_x,
                                          self.train_data_y)
         if step % 50 == 0:
             logger.info("After %d trainings, the cost: %f" % (step, cost))
예제 #25
0
def create_device(**kwargs):
    logger.info('Register new device')
    now = datetime.utcnow()
    try:
        query = Device.query.filter_by(serial=kwargs.get('serial')).first()
        if query is not None:
            logger.error("Device already exists")
            return {'message': 'Device already exists'}, 409

        company = db.session.query(Company).filter_by(
            name=kwargs.get('company')).one()
        owner = db.session.query(User).filter_by(
            userid=kwargs.get('owner')).one()
        device = Device(model=kwargs.get('model'),
                        serial=kwargs.get('serial'),
                        company=company,
                        owner=owner,
                        ip=kwargs.get('ip'),
                        created=now,
                        created_by=kwargs.get('created_by', current_user),
                        last_edited=now,
                        edited_by=kwargs.get('edited_by', current_user),
                        is_deleted=False)
        db.session.add(device)
        db.session.commit()
        return {
            'message': f'Posted device<{kwargs.get("serial")}> to db.'
        }, 201
    except Exception as e:
        logger.error(e)
        logger.debug(traceback.format_exc())
        raise e
예제 #26
0
def openPage(url, query=dict(), headers=dict()) -> responsedString:
    global opener

    if len(query) == 0:
        query = None
    else:
        query = urlencode(query)

    req = request.Request(url, query, headers, origin_req_host=None)

    res = None
    count = 0
    while count < 3:
        import socket
        try:
            res = opener.open(req)
        except error.URLError:
            count = count + 1
            logger.info("-* failed connect to : {}. retrying {} / 3".format(
                (url, count)))
            continue
        except socket.error:
            count = count + 1
            continue
        except:
            raise ConnectionError
        else:
            break
    if not res: raise ConnectionError
    ret = responsedString(res)

    logger.debug("page opened. returned {} length. Info : {}".format(
        (len(ret)), res.info()))
    return ret
예제 #27
0
def on_capture_success(task_id):
    logger.info("Callback triggered: Adding image record of capture.")
    try:
        res = celery_app.AsyncResult(task_id)
        data = res.get()
        res.forget()
        logger.info(f'data: {data}')
        image = Image(cell=data.get('cell'),
                      path=data.get('path'),
                      device=data.get('device'),
                      created=data.get('created'),
                      created_by=data.get('created_by'),
                      label=data.get('label'),
                      offset_x=data.get('offset_x'),
                      offset_y=data.get('offset_y'),
                      offset_z=data.get('offset_z'),
                      pos_x=data.get('pos_x'),
                      pos_y=data.get('pos_y'),
                      pos_z=data.get('pos_z'))
        db.session.add(image)
        db.session.commit()
        return jsonify('Successfully registered image record'), 200
    except Exception as e:
        logger.error(e)
        return jsonify('Failed to register image record'), 200
예제 #28
0
def load_audio_file(filename: str):
    global sound
    sound = pygame.mixer.Sound(filename)
    sound.set_volume(1.0)

    logger.info(
        f"▶ audio file loaded [{os.stat(filename).st_size}]: '{filename}'")
예제 #29
0
def create_user(**kwargs):
    logger.info("User registration")
    now = datetime.utcnow()
    try:
        query = User.query.filter_by(userid=kwargs.get('userid')).first()
        if query is not None:
            logger.error("Userid already exists")
            return {'message': 'userid already exists'}, 409

        comp_name = kwargs.get('company')
        if comp_name is None:
            return {'message': 'Company name is empty.'}, 400

        kwargs.update({
            'company': Company.query.filter_by(name=comp_name).one(),
            'created': now,
            'last_edited': now,
            'is_deleted': False
        })

        user = User(**kwargs)
        user.hash_password()
        db.session.add(user)
        db.session.commit()

        logger.info('User registration successful')
        return {'message': 'User registration successful'}, 201
    except NoResultFound as e:
        return {'message': 'Company doesn\'t exist.'}, 400
    except Exception as e:
        logger.error(traceback.format_exc())
        raise e
예제 #30
0
def get_post_page(shortcode):
    node_path = 'p/'
    field = {
        '__a': 1,
    }
    logger.info(f'get {shortcode} post')
    return get_instagram_data(node_path, shortcode, field)
예제 #31
0
    def _save_model1(self, params, save_path):
        config = tf.ConfigProto()
        config.gpu_options.allow_growth = True
        keras.backend.set_session(tf.Session(config=config))

        params = param_preprocess.param_prepare(params, 'save')
        keras.backend.clear_session()

        anchors = np.array(params['model']['anchors']).reshape(-1, 2)

        num_anchors = 9
        num_classes = len(params['model']['class_names'])

        input_image = keras.layers.Input(shape=(None, None, 3), name='input_image')
        input_image_meta = keras.layers.Input(shape=(2,), name='input_image_meta')

        yolo_model = yolo_body(input_image, num_anchors // 3, num_classes)
        logger.info('load weights from {}'.format(params['evaluate']['weights_path']))
        yolo_model.load_weights(params['evaluate']['weights_path'])

        boxes, box_scores = keras.layers.Lambda(my_yolo_eval, output_shape=None, name='yolo_eval1',
                                                arguments={'anchors': anchors, 'num_classes': num_classes})(
            [yolo_model.output[0],
             yolo_model.output[1],
             yolo_model.output[2],
             input_image_meta])

        boxes = keras.layers.Lambda(lambda x: identity(x, name="boxes"))(boxes)
        box_scores = keras.layers.Lambda(lambda x: identity(x, name="box_scores"))(box_scores)

        model = keras.models.Model([input_image, input_image_meta], [box_scores, boxes])

        h52pb(model, save_path)
        logger.info('save pb model in {}'.format(save_path))
예제 #32
0
	def load_sync_steps(self):
		dep_path = Config().observer_dependency_graph
		logger.info('Loading model dependency graph from %s' % dep_path)
		try:
			# This contains dependencies between records, not sync steps
			self.model_dependency_graph = json.loads(open(dep_path).read())
		except Exception,e:
			raise e
예제 #33
0
 def run(self):
     logger.debug("Running %s" % self)
     while(not self.stop_event.isSet()):
         StateThread.pause_event.wait()
         if callable(self.fxn):
             self.fxn()
     self.stop_event.clear()
     logger.info("Exiting %s" % self)
예제 #34
0
	def check_schedule(self, step):
		time_since_last_run = time.time() - self.last_run_times.get(step.__name__, 0)
		try:
			if (time_since_last_run < step.requested_interval):
				raise StepNotReady
		except AttributeError:
			logger.info('Step %s does not have requested_interval set'%step.__name__)
			raise StepNotReady
예제 #35
0
파일: worker.py 프로젝트: dangpu/momoko
 def __init__(self, workers, thread_name, func):
     '''
     '''
     self.__workers = workers
     self.__busy = False
     self.thread_name = thread_name
     self.__func = func
     threading.Thread.__init__(self, None, None, self.thread_name, (), {})
     logger.info("%s init complete" % self.thread_name)
예제 #36
0
파일: worker.py 프로젝트: dangpu/momoko
    def run(self):
        '''
        '''
        self.__busy = True
        
        while( self.__busy ):
            self.__func()

        self.__busy = False
        logger.info("%s stop" %self.thread_name)
예제 #37
0
파일: slave.py 프로젝트: dangpu/momoko
 def register_in_master(self):
     '''
         向master注册,并获得master分配的id
     '''
     data = {"name": self.info.name,
             "server": self.info.server,
             "path": self.info.path,
             "server_ip": self.info.server_ip,
             "type": self.info.type,
             'recv_real_time_request': self.info.recv_real_time_request}
     path = "/register_slave?" + urllib.urlencode(data)
     id = self.__client.get(path).strip()
     id = id.strip('\0')
     if len(id) == 0 or not id.isdigit():
         return False
     self.info.id = int(id)
     logger.info("slave register id is : %d" % self.info.id)
     return True
예제 #38
0
	def load_sync_step_modules(self, step_dir=None):
		if step_dir is None:
			if hasattr(Config(), "observer_steps_dir"):
				step_dir = Config().observer_steps_dir
			else:
				step_dir = "/opt/planetstack/observer/steps"

		for fn in os.listdir(step_dir):
			pathname = os.path.join(step_dir,fn)
			if os.path.isfile(pathname) and fn.endswith(".py") and (fn!="__init__.py"):
				module = imp.load_source(fn[:-3],pathname)
				for classname in dir(module):
					c = getattr(module, classname, None)

					# make sure 'c' is a descendent of SyncStep and has a
					# provides field (this eliminates the abstract base classes
					# since they don't have a provides)

					if inspect.isclass(c) and issubclass(c, SyncStep) and hasattr(c,"provides") and (c not in self.sync_steps):
						self.sync_steps.append(c)
		logger.info('loaded sync steps: %s' % ",".join([x.__name__ for x in self.sync_steps]))
예제 #39
0
	def wake_up(self):
		logger.info('Wake up routine called. Event cond %r'%self.event_cond)
		self.event_cond.acquire()
		self.event_cond.notify()
		self.event_cond.release()
예제 #40
0
#/usr/bin/env python

__author__="henry"
__date__ ="$Mar 25, 2011 3:35:38 PM$"

from devices.factory import factory
from util.logger import logger
import config


def main():
    pass

if __name__ == "__main__":
    logger.info("Starting Elixys Server")
예제 #41
0
파일: populate.py 프로젝트: bkeyvani/fsnd
            SELECT players.id, players.name, v_numWins.wins,
                   v_numMatches.matchesPlayed AS matches
            FROM players
            LEFT JOIN v_numWins ON
            (players.id = v_numWins.id)
            JOIN v_numMatches ON (players.id = v_numMatches.id)
            ORDER BY wins DESC;
        """)

    conn.commit()
    conn.close()


if __name__ == '__main__':
    # start logging
    logger.info('Started')

    # create the tournament DB
    create_db()
    logger.info('Created DB')

    # create tables and views
    create_tables()
    logger.info('Created tables')
    create_indices()
    logger.info('Created indices')
    create_views()
    logger.info('Created views')

    # Register players
    PLAYERS = ['Player 1', 'Player 2', 'Player 3', 'Player 4', 'Player 5', 'Player 6', 'Player 7', 'Player 8', 'Player 9', 'Player 10', 'Player 11', 'Player 12', 'Player 13', 'Player 14', 'Player 15', 'Player 16', 'Player 17', 'Player 18', 'Player 19', 'Player 20', 'Player 21', 'Player 22', 'Player 23', 'Player 24', 'Player 25', 'Player 26', 'Player 27', 'Player 28', 'Player 29', 'Player 30', 'Player 31', 'Player 32', 'Player 33', 'Player 34', 'Player 35', 'Player 36', 'Player 37', 'Player 38', 'Player 39', 'Player 40', 'Player 41', 'Player 42', 'Player 43', 'Player 44', 'Player 45', 'Player 46', 'Player 47', 'Player 48', 'Player 49', 'Player 50', 'Player 51', 'Player 52', 'Player 53', 'Player 54', 'Player 55', 'Player 56', 'Player 57', 'Player 58', 'Player 59', 'Player 60', 'Player 61', 'Player 62', 'Player 63', 'Player 64',]
예제 #42
0
	def run(self):
		if not self.driver.enabled:
			return
		if (self.driver_kind=="openstack") and (not self.driver.has_openstack):
			return

		while True:
			try:
				logger.info('Waiting for event')
				tBeforeWait = time.time()
				self.wait_for_event(timeout=30)
				logger.info('Observer woke up')

				# Set of whole steps that failed
				failed_steps = []

				# Set of individual objects within steps that failed
				failed_step_objects = set()

				for S in self.ordered_steps:
					step = self.step_lookup[S]
					start_time=time.time()
					
					sync_step = step(driver=self.driver)
					sync_step.__name__ = step.__name__
					sync_step.dependencies = []
					try:
						mlist = sync_step.provides
						
						for m in mlist:
							sync_step.dependencies.extend(self.model_dependency_graph[m.__name__])
					except KeyError:
						pass
					sync_step.debug_mode = debug_mode

					should_run = False
					try:
						# Various checks that decide whether
						# this step runs or not
						self.check_class_dependency(sync_step, failed_steps) # dont run Slices if Sites failed
						self.check_schedule(sync_step) # dont run sync_network_routes if time since last run < 1 hour
						should_run = True
					except StepNotReady:
						logging.info('Step not ready: %s'%sync_step.__name__)
						failed_steps.append(sync_step)
					except:
						failed_steps.append(sync_step)

					if (should_run):
						try:
							duration=time.time() - start_time

							logger.info('Executing step %s' % sync_step.__name__)

							# ********* This is the actual sync step
							#import pdb
							#pdb.set_trace()
							failed_objects = sync_step(failed=list(failed_step_objects))


							self.check_duration(sync_step, duration)
							if failed_objects:
								failed_step_objects.update(failed_objects)
							self.update_run_time(sync_step)
						except:
							failed_steps.append(S)
				self.save_run_times()
			except:
				logger.log_exc("Exception in observer run loop")
				traceback.print_exc()
예제 #43
0
    except AttributeError,e:
        raise e

def update_dep(d, o):
    try:
        print 'Trying to update %s'%d
        save_fields = []
        if (d.updated < o.updated):
            save_fields = ['updated']

        if (save_fields):
            d.save(update_fields=save_fields)
    except AttributeError,e:
        raise e
    except Exception,e:
            logger.info('Could not save %r. Exception: %r'%(d,e))

def delete_if_inactive(d, o):
    try:
        d.delete()
        print "Deleted %s (%s)"%(d,d.__class__.__name__)
    except:
        pass
    return


@atomic
def execute_model_policy(instance, deleted):
    # Automatic dirtying
    if (instance in bad_instances):
        return