Ejemplo n.º 1
0
def init_runlist():
    '''生成任务列表'''
    # 获取所有配置项目
    items = config('agent')
    items.remove('global')
    # 运行脚本目录
    script_dir = 'scripts/'
    # 运行状态
    run_list = []
    # 初始化执行列表
    for item in items:
        item_conf = config('agent', item)
        if not item_conf['timeout']:
            item_conf['timeout'] = agent_conf['timeout']
        # 加入如开始时间
        item_conf['last_time'] = None
        # 脚本目录
        item_conf['script_ptah'] = script_dir + item_conf['name']
        script_path = item_conf['script_ptah']
        # 检查脚本是否存在
        if not os.path.exists(script_path):
            raise  Exception('script %s not find' % script_path)
            sys.exit(1)
        run_list.append(item_conf)
    
    # 初始任务化时间状态
    for task in run_list:
        if not task['last_time']:
            task['last_time'] = time.time()
        cmd = task['script_ptah'] 
        task['timeout'] = int(task['timeout']) 
        task['interval']= int(task['interval'])
    return run_list
Ejemplo n.º 2
0
class Feishu(object):
    FEISHU_APP_ID = config('FEISHU_APP_ID')
    FEISHU_APP_SECRET = config('FEISHU_APP_SECRET')
    FEISHU_CHARGE_CHAT_ID = config('FEISHU_CHARGE_CHAT_ID', default='oc_a4bc2f10dd9ec84f08f2bbcaa82e08cd')
    FEISHU_MAC_CHAT_ID = config('FEISHU_MAC_CHAT_ID', default='oc_3a7065d01efdb36d949088341aada466')
    FEISHU_SESSION_CHAT_ID = config('FEISHU_SESSION_CHAT_ID', default='oc_19b2404bb0917fc066cce1b3a58c3558')

    @classmethod
    def send_groud_msg(cls, receiver_id: str, text: str):
        data = {
            'app_id': cls.FEISHU_APP_ID,
            'app_secret': cls.FEISHU_APP_SECRET,
        }
        response = requests.post('https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal/', json=data)
        assert response.ok
        body = json.loads(response.text)
        if body['code'] != 0:
            raise Exception('飞书获取access_token失败')
        access_token = response.json()['tenant_access_token']
        #
        headers = {
            'Authorization': f'Bearer {access_token}'
        }
        data = {
            'chat_id': receiver_id,
            'msg_type': 'text',
            'content': {
                'text': text,
            }
        }
        response = requests.post('https://open.feishu.cn/open-apis/message/v4/send/', json=data, headers=headers)
        assert response.ok
        body = json.loads(response.text)
        if body['code'] != 0:
            raise Exception('信息发送到飞书败')
Ejemplo n.º 3
0
def save_figs(train_returns, test_returns, train_loss_critic, train_loss_actor, prefix=""):
    tr_cycle = config().metrics.train_cycle_length
    ts_cycle = config().metrics.test_cycle_length
    filepath = os.path.abspath(os.path.join(config().sim.output.path, f"{prefix}metrics"))
    plt.clf()
    plt.cla()
    plt.figure(0)
    plt.plot(range(0, len(train_returns) * tr_cycle, tr_cycle), train_returns, label="Train")
    plt.plot(range(0, len(test_returns) * ts_cycle, ts_cycle), test_returns, label="Test")
    plt.legend()
    plt.xlabel("Episodes")
    plt.ylabel("Expected return")
    plt.savefig(filepath + "_returns.eps", type="eps", dpi=1000)

    plt.clf()
    plt.cla()
    plt.figure(1)
    if config().sim.agent.type == "dqn":
        plt.plot(range(0, len(train_loss_critic) * tr_cycle, tr_cycle), train_loss_critic)
    else:
        plt.plot(range(0, len(train_loss_critic) * tr_cycle, tr_cycle), train_loss_critic, label="Critic")
        plt.plot(range(0, len(train_loss_actor) * tr_cycle, tr_cycle), train_loss_actor, label="Actor")
        plt.legend()
    plt.xlabel("Episodes")
    plt.ylabel("Training Losses")
    plt.savefig(filepath + "_losses.eps", type="eps", dpi=1000)
Ejemplo n.º 4
0
def download_file(version, format, type_, extension):
    v = version.replace('.', '_')
    archive_path = '{}/{}/{}/{}.{}'.format(config()['data']['data_folder'],
                                           config()['data']['build_name'],
                                           format, type_, extension)
    return FileResponse(archive_path,
                        filename='echr_{}_{}_{}.{}'.format(
                            v, format, type_, extension))
Ejemplo n.º 5
0
def get_categories(userid):
    # get user's categories based on id
    retrieve = """SELECT X."CategoryId", X."CategoryName" FROM "Categories" X, "UserCategories"Y 
                    WHERE X."CategoryId" = Y."CategoryId" AND Y."UserId" = '{}';""".format(
        userid)
    conn = None
    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()
        # check if user exists
        cur.execute(retrieve)
        # store all results
        categories = cur.fetchall()
        # close the cursor
        cur.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
    return categories
Ejemplo n.º 6
0
def recommend_by_recent():
    """ finds recipe based on search """
    checkdb = """SELECT "RecipeId", "RecipeName", "CreationDate"
                 FROM "Recipes"
                 ORDER BY "CreationDate" DESC;"""
    conn = None
    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()
        # check if user exists
        cur.execute(checkdb)
        # store all results
        results = cur.fetchall()
        # close the cursor
        cur.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
    return results
Ejemplo n.º 7
0
def recommend_by_rating():
    """ finds recipe based on search """
    checkdb = """SELECT "RecipeId", "RecipeName", "avg" FROM
                 (SELECT "Recipes"."RecipeId", "Recipes"."RecipeName", ROUND(AVG("CookedRecipes"."Rating") ,2) AS "avg"
                 FROM "Recipes" INNER JOIN "CookedRecipes"
                 ON  "Recipes"."RecipeId" = "CookedRecipes"."RecipeId"
                 GROUP BY "Recipes"."RecipeId") AS "Ratings"
                 ORDER BY "avg" DESC;"""

    conn = None
    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()
        # check if user exists
        cur.execute(checkdb)
        # store all results
        results = cur.fetchall()
        # close the cursor
        cur.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
    return results
Ejemplo n.º 8
0
def cook(scale, rating, userid, recipeid):
    """ gets a user's pantry data """
    checkdb = """INSERT INTO "CookedRecipes"("DateCooked", "Scale", "Rating", "UserId", "RecipeId") 
                    VALUES(%s, %s, %s, %s, %s)"""
    ct = datetime.datetime.utcnow()
    conn = None
    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()
        # check if user exists
        cur.execute(checkdb, (ct, scale, rating, userid, recipeid))
        # store all results
        conn.commit()
        # close the cursor
        cur.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
    return 'success'
Ejemplo n.º 9
0
def unregister_services_conf():
    """
            从.ini获取需要下架的服务
            :return:
    """
    env_type = env_file_conf('ENV_TYPE').upper() if env_file_conf(
        'ENV_TYPE') else "DEV"
    seervices_conf = config('configs/unregister_services.ini')
    services_str = seervices_conf.getOption(env_type, 'services', default='{}')
    service_dict = json.loads(services_str)
    all_services = []
    if not service_dict: return all_services
    for product, service_str in service_dict.items():
        # print("Debug {}".format(service_str))
        services_list = service_str.split(',')
        for service in services_list:
            service_attr = {
                "product": product,
                "service": service,
                "env_type": env_type.lower()
            }
            try:
                all_services.index(service_attr)
            except ValueError:
                all_services.append(service_attr)
    print("Deleted services: {}".format(all_services))
    return all_services
Ejemplo n.º 10
0
def insert_category(recipe_id, category_id):
    """ inserts into the RecipeCategories Table """
    check_sql = """SELECT * FROM "RecipeCategories" 
                    WHERE "RecipeId" = '{}' AND "CategoryId" = '{}';""".format(
        recipe_id, category_id)
    insert_category_sql = """INSERT INTO "RecipeCategories"("RecipeId", "CategoryId") VALUES(%s, %s)"""
    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()
        # check if user exists
        cur.execute(check_sql)
        category = cur.fetchone()

        if category is None:
            # execute the INSERT statement to the RecipeCategories table
            cur.execute(insert_category_sql, (recipe_id, category_id))
            # commit the changes to the database
            conn.commit()
        else:
            return 'failed'

        # close communication with the database
        cur.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
        return 'failed2'
    finally:
        if conn is not None:
            conn.close()
    return 'success'
Ejemplo n.º 11
0
def search_ingredient(keyword):
    """ finds recipe based on search """
    results = None
    checkdb = """SELECT X."IngredientId", X."IngredientName" FROM "Ingredients" X
                        WHERE X."IngredientName" LIKE '%{}%';""".format(
        keyword)
    conn = None
    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()
        # check if user exists
        cur.execute(checkdb)
        # store all results
        results = cur.fetchall()
        # close the cursor
        cur.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
    return results
Ejemplo n.º 12
0
 def __init__(self, mode='train', generate_img=True):
     super(point_modelnet40_Dataset_cls, self).__init__()
     cfg = config.config()
     if mode == 'train':
         self.files = h5_helper.getDataFiles(
             osp.join(cfg.modelnet_cls_dir, 'train_files.txt'))
     elif mode == 'test':
         self.files = h5_helper.getDataFiles(
             osp.join(cfg.modelnet_cls_dir, 'test_file.txt'))
     self.files = [self.files[0]]
     self.data = None
     self.label = None
     if generate_img:
         self.compose = point_preprocess.get_train_test_compose()
     else:
         self.compose = point_preprocess.get_norm_points_compose()
     for f in self.files:
         cur_data, cur_label = h5_helper.loadDataFile(osp.join('../', f))
         if self.data is None:
             self.data = cur_data
             self.label = cur_label
         else:
             self.data = np.concatenate((self.data, cur_data), 0)
             self.label = np.concatenate((self.label, cur_label), 0)
     self.label.astype(np.int64)
Ejemplo n.º 13
0
def show_pantry(uid):
    """ gets a user's pantry data """
    checkdb = """SELECT I."IngredientName", P."CurrentQuantity", P."ExpirationDate", P."OrderId"
                        FROM "UserOrders" U, "OrderIngredients" O, "Ingredients" I, "Pantry" P
                        WHERE U."UserId" = '{}' AND U."OrderId" = O."OrderId" AND
                             O."IngredientId" = I."IngredientId" AND U."OrderId" = P."OrderId";""".format(uid)
    conn = None
    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()
        # check if user exists
        cur.execute(checkdb)
        # store all results
        results = cur.fetchall()
        # close the cursor
        cur.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
    return results
Ejemplo n.º 14
0
def main():

    param_config = config()
    gpt_config = GPT2Config(vocab_size_or_config_json_file=param_config.input_dim, n_positions=param_config.sequence_length, n_ctx=param_config.sequence_length)

    model = GPT2LMHeadModel(gpt_config)
    # Load Data
    # Load Data
    if param_config.input == 'bytes':
        # Load Data for bytes
        _, validation_data = get_wili_data_bytes(param_config)
    else:
        # Load Data
        _, validation_data = get_wili_data(param_config)

    validation_loader = DataLoader(validation_data,
                             batch_size=1,
                             shuffle=False,
                             drop_last=False)

    if param_config.model_checkpoint is not None:
        with open(param_config.model_checkpoint, 'rb') as f:
            state_dict = torch.load(f)
            model.load_state_dict(state_dict)
            print("Model Loaded From: {}".format(param_config.model_checkpoint))

    model = model.to(device)
    predict(model, validation_loader, validation_data, param_config)
Ejemplo n.º 15
0
def get_ingredients_with_ids(recipeid):
    # gets ingredients based on recipeid
    retrieve = """SELECT X."IngredientId", X."IngredientName", Y."Amount" FROM "Ingredients" X, "IngredientsForRecipe" Y
        WHERE X."IngredientId" = Y."IngredientId" AND Y."RecipeId" = '{}';""".format(
        recipeid)

    conn = None
    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()
        # check if user exists

        cur.execute(retrieve)
        # store all results
        ingredients = cur.fetchall()
        # close the cursor
        cur.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
    return ingredients
Ejemplo n.º 16
0
def create_tables():
    commands = ("""
       DROP TABLE IF EXISTS personalise_app;
       """, """
       CREATE TABLE news (
         -- model fields here
        )
        """, """
        # insert values here
        """)
    try:
        # read the connection parameters
        params = config()
        # connect to the PostgreSQL server
        conn = psycopg2.connect(**params)
        cur = conn.cursor()
        # create table one by one
        for command in commands:
            cur.execute(command)
        # close communication with the PostgreSQL database server
        cur.close()
        # commit the changes
        conn.commit()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
Ejemplo n.º 17
0
def get_my_recipes(uid):
    """ gets all a users' recipes """
    result = None
    conn = None

    get_recipes = """SELECT  "RecipeId", "RecipeName" FROM "Recipes"
                       WHERE "UserId" = %s
    """

    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()

        # check if user exists
        cur.execute(get_recipes, (uid, ))

        result = cur.fetchall()

        # close the cursor
        cur.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
    return result
Ejemplo n.º 18
0
def get_rating(recipeid):
    # gets rating based on recipeid
    avgrating = """SELECT ROUND(AVG("Rating") ,2) FROM "CookedRecipes" WHERE "RecipeId" = '{}';""".format(
        recipeid)
    conn = None
    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()
        # check if user exists

        cur.execute(avgrating)
        # store all results
        rating = cur.fetchone()
        # close the cursor
        cur.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
    return rating
Ejemplo n.º 19
0
def show_categories(user_id):
    """ gets a user's pantry data """
    global results
    checkdb = """SELECT "Categories"."CategoryName" FROM "Categories" INNER JOIN "UserCategories" On "Categories"."CategoryId" = "UserCategories"."CategoryId"
        WHERE "UserId" = '{}'""".format(user_id)
    conn = None
    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()
        # check if user exists
        cur.execute(checkdb)
        # store all results
        results = cur.fetchall()
        # close the cursor
        cur.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
    return results
def ini_conf(section, option, default, file_name=None):

    ini_config = config(filename=file_name)
    conf_value = ini_config.getOption(section=section,
                                      option=option,
                                      default=default)

    return conf_value
Ejemplo n.º 21
0
def main():
    args = config.config()

    if not args.train_data_path:
        logger.info("please input train dataset path")
        exit()
    # if not (args.dev_data_path or args.test_data_path):
    #     logger.info("please input dev or test dataset path")
    #     exit()

    all_ = data_preprocess.load_dataset(args.train_data_path, args.dev_data_path, args.test_data_path, \
                     args.src_embedding_path, args.tgt_embedding_path, args.train_batch_size, \
                                                         args.dev_batch_size, args.test_batch_size)
    src_TEXT, tgt_TEXT, src_vocab_size, tgt_vocab_size, src_word_embeddings, tgt_word_embeddings, \
           train_iter, dev_iter, test_iter = all_

    bos_id = dict(tgt_TEXT.vocab.stoi)['<bos>']
    eos_id = dict(tgt_TEXT.vocab.stoi)['<eos>']
    index2tgtword = tgt_TEXT.vocab.itos

    model = NMT_Atten(src_vocab_size, tgt_vocab_size, args.src_embedding_dim, args.tgt_embedding_dim, \
              src_word_embeddings, tgt_word_embeddings, args.hidden_size, args.tgt_max_len, bos_id)

    if torch.cuda.is_available():
        model = model.cuda()

    train_data, dev_data = data_preprocess.train_dev_split(train_iter, 0.9)
    loss_func = torch.nn.CrossEntropyLoss(reduction='none')
    if args.load_model:
        model.load_state_dict(torch.load(args.load_model, map_location='cpu'))
        while True:
            test_sent = input("Input source sentence (q exit) >>>>")
            if test_sent.lower() == 'q':
                break
            #分词
            sent = ' '.join(jieba.cut(test_sent, cut_all=False))
            #print(sent)
            test_sent = src_TEXT.preprocess(sent)
            #print(test_sent)
            test_idx = [[src_TEXT.vocab.stoi[x] for x in test_sent]]
            #print(test_idx)
            inference(model, test_idx, eos_id, index2tgtword)
        return

    best_score = 0.0
    for epoch in range(args.epoch):
        train_loss, eval_loss, eval_score = train_model(model, train_data, dev_data, epoch,\
                                                               args.lr, loss_func, eos_id, index2tgtword)

        logger.info('Epoch:%d, Training Loss:%.4f', epoch, train_loss)
        logger.info('Epoch:%d, Eval Loss:%.4f, Eval BLEU score:%.4f', epoch,
                    eval_loss, eval_score)

        if eval_score > best_score:
            best_score = eval_score
            torch.save(
                model.state_dict(),
                'results/%d_%s_%s.pt' % (epoch, 'Model', str(best_score)))
Ejemplo n.º 22
0
def main(args):
    global conn, cur

    # connect to PostgreSQL database
    db_params = config(filename=CONFIG_INI_FILE)
    conn = psycopg2.connect(**db_params)
    cur = conn.cursor()

    # load detection model
    model = detection.load_inference_resnet50()

    # fetch areas that will be analyzed
    spots = fetch_parking_spots(args.cam_ids)

    # get video data
    vcap = cv2.VideoCapture(args.video_file)
    fps = int(vcap.get(cv2.CAP_PROP_FPS))
    frame_counter = 0
    detection_interval = fps * time2seconds(args.time_interval)

    # start analyzing parking lot
    while vcap.isOpened():
        ret, frame = vcap.read()
        key = cv2.waitKey(fps) & 0xFF

        # end of video or user exit
        if not ret or key == ord("q"):
            print("Video stopped")
            break

        # check if parking spots are occupied every nth frame
        if frame_counter % detection_interval == 0:
            # set occupancy for each spot to false
            reset_occupancy(args.cam_ids)

            # detect which spots are occupied
            bboxes = detection.detect_objects(model,
                                              frame, [3, 4],
                                              threshold=0.5)
            occupied_spots = fetch_occupied_spots(spots, bboxes)

            # update occupancy in table for each spot
            update_occupancy(occupied_spots)

        # check if spot_time > time_threshold
        update_occupied_time(fps)
        update_overtime(args.limit)
        frame_counter += 1

        # display video
        # frame = display(frame, args.cam_ids)
        # cv2.imshow("parking lot", frame)

    # reset and close connections
    vcap.release()
    reset()
    cur.close()
    conn.close()
Ejemplo n.º 23
0
def make_stations_db_table(stop_db=False):
    """Load dataset with Russian weather stations and their characteristics
    to PostgreSQL database.
    """

    start_instance()

    conn = None
    try:
        # read connection parameters
        db_params = config(section="postgresql")

        # connect to the PostgreSQL server
        logging.info("Connecting to the PostgreSQL database...")
        conn = psycopg2.connect(**db_params)
        # conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)

        # create a cursor to perform database operations
        # cur = conn.cursor()
        with conn.cursor() as cur, open("../data/rus_weather_stations.csv",
                                        "r") as f:
            # ['id', 'lat', 'lon', 'elev', 'state', 'name', 'extra1', 'extra2'],
            sql = (
                "DROP TABLE IF EXISTS weather_stations; "
                "CREATE TABLE weather_stations (station_id varchar(11) UNIQUE NOT NULL, "
                "latitude real NOT NULL, longitude real NOT NULL, elevation real NOT NULL, "
                "state varchar(2), station_name varchar(30), extra1 varchar(7), "
                "extra2 real)")
            for q in sql.split("; "):
                logging.debug(f"SQL query to be executed: {q}")
                cur.execute(q)
                logging.debug(f"cur.statusmessage is {cur.statusmessage}")

            next(f)  # Skip the header row
            # for the command below to work, the single null value (empty string)
            # in the last column (extra2) had to be manually replaced with a
            # real value (just copied from the row just below)
            # otherwise, this exception was raised:
            # psycopg2.errors.InvalidTextRepresentation: invalid input syntax for type real: ""
            cur.copy_from(f, "weather_stations", sep="\t")

        # Make the changes to the database persistent
        conn.commit()

        for line in conn.notices:
            logging.debug(line.strip("\n"))

    except (Exception, psycopg2.DatabaseError) as error:
        logging.exception("Exception occurred")

    finally:
        if conn is not None:
            conn.close()
            logging.info("Database connection closed.")

    if stop_db:
        stop_instance()
Ejemplo n.º 24
0
def get_build_info(build_name):
    current_build_info_path = os.path.join(config()['data']['data_folder'],
                                           build_name, 'build_info.yml')
    try:
        with open(current_build_info_path, 'r') as f:
            current_build_info = yaml.full_load(f)
        return current_build_info
    except Exception as e:
        return {}
Ejemplo n.º 25
0
def update_build():
    available, new_build = is_new_build_available()
    if not available:
        return {'message': 'Build up-to-date'}
    if os.path.isdir(TRANSITION_DIR):
        shutil.rmtree(TRANSITION_DIR)
    current_build = os.path.join(config()['data']['data_folder'],
                                 config()['data']['build_name'])
    new_build_path = os.path.join(config()['data']['data_folder'], new_build)
    shutil.copytree(current_build, TRANSITION_DIR)
    shutil.rmtree(current_build)
    shutil.copytree(new_build_path, current_build)
    output = subprocess.run(["cat", "/proc/self/cgroup"],
                            stdout=subprocess.PIPE)
    output = [e for e in output.stdout.decode().splitlines() if 'docker' in e]
    cid = output[-1].split('/')[-1]
    subprocess.run(["docker", "restart", cid])

    return {'message': 'Build updated'}
Ejemplo n.º 26
0
def main(video_file, num, loc):
    # connect to database
    logging.info("Connecting to database")
    db_params = config(filename=CONFIG_INI_FILE)
    conn = psycopg2.connect(**db_params)
    cur = conn.cursor()

    # Create necessary tables
    cur.execute("SAVEPOINT table_creation")
    try:
        logging.info("Instantiating relations")
        cur.execute(open(TABLES_SQL_FILE).read())
    except psycopg2.errors.DuplicateObject:
        logging.info("Relations are already instantiated")
        cur.execute("ROLLBACK TO SAVEPOINT table_creation")
    else:
        cur.execute("RELEASE SAVEPOINT table_creation")

    # Fetch video metadata
    cap = cv2.VideoCapture(video_file)
    res_x = int(cap.get(3))
    res_y = int(cap.get(4))
    fps = int(cap.get(cv2.CAP_PROP_FPS))

    # Populate entries for cameras table
    # If the chosen camera exists, then
    # jump to populating spots
    query = """INSERT INTO cameras VALUES
                 (%s, %s, %s, %s, %s);"""

    cur.execute("SELECT id from cameras;")
    ids = [r[0] for r in cur.fetchall()]
    if num not in ids:
        cur.execute(query, (num, loc, res_x, res_y, fps))
    else:
        logging.warning("Modifying existing camera")

    # Initialize parking lots that will be later analyzed
    query = """INSERT INTO spots VALUES
               (DEFAULT, %s, ST_SetSRID(%s::geometry, %s), false, '00:00:00', false)"""
    logging.info("Select areas of interest")
    ret, frame = cap.read()
    coords = events.select_area(frame)

    # Populate entries for spots table
    for pts in coords:
        polygon = events.sort2cyclic(pts)
        spot = Polygon(polygon)
        cur.execute(query, (num, spot.wkb_hex, 4326))

    # Commit changes and close connection
    logging.info("Committing changes and closing connection to database")
    conn.commit()
    cur.close()
    conn.close()
Ejemplo n.º 27
0
def main():
    args = config.config()

    if not args.train_data_path:
        logger.info("please input train dataset path")
        exit()
    # if not (args.dev_data_path or args.test_data_path):
    #     logger.info("please input dev or test dataset path")
    #     exit()

    all_ = dataset.load_dataset(args.train_data_path, args.dev_data_path, args.test_data_path, \
                     args.txt_embedding_path, args.cpt_embedding_path, args.train_batch_size, \
                                                         args.dev_batch_size, args.test_batch_size)
    txt_TEXT, cpt_TEXT, txt_vocab_size, cpt_vocab_size, txt_word_embeddings, cpt_word_embeddings, \
           train_iter, dev_iter, test_iter, label_size = all_

    model = STCK_Atten(txt_vocab_size, cpt_vocab_size, args.embedding_dim, txt_word_embeddings,\
                        cpt_word_embeddings, args.hidden_size, label_size)

    if torch.cuda.is_available():
        model = model.cuda()

    train_data, test_data = dataset.train_test_split(train_iter, 0.8)
    train_data, dev_data = dataset.train_dev_split(train_data, 0.8)
    loss_func = torch.nn.CrossEntropyLoss()

    if args.load_model:
        model.load_state_dict(torch.load(args.load_model))
        test_loss, acc, p, r, f1 = eval_model(model, test_data, loss_func)
        logger.info(
            'Test Loss:%.4f, Test Acc:%.4f, Test P:%.4f, Test R:%.4f, Test F1:%.4f',
            test_loss, acc, p, r, f1)
        return

    best_score = 0.0
    test_loss, test_acc, test_p, test_r, test_f1 = 0, 0, 0, 0, 0
    for epoch in range(args.epoch):
        train_loss, eval_loss, acc, p, r, f1 = train_model(
            model, train_data, dev_data, epoch, args.lr, loss_func)

        logger.info('Epoch:%d, Training Loss:%.4f', epoch, train_loss)
        logger.info(
            'Epoch:%d, Eval Loss:%.4f, Eval Acc:%.4f, Eval P:%.4f, Eval R:%.4f, Eval F1:%.4f',
            epoch, eval_loss, acc, p, r, f1)

        if f1 > best_score:
            best_score = f1
            torch.save(
                model.state_dict(),
                'results/%d_%s_%s.pt' % (epoch, 'Model', str(best_score)))
            test_loss, test_acc, test_p, test_r, test_f1 = eval_model(
                model, test_data, loss_func)
        logger.info(
            'Test Loss:%.4f, Test Acc:%.4f, Test P:%.4f, Test R:%.4f, Test F1:%.4f',
            test_loss, test_acc, test_p, test_r, test_f1)
Ejemplo n.º 28
0
def get_build_history():
    history_path = os.path.join(config()['data']['data_folder'],
                                '.build_history')
    try:
        with open(history_path, 'r') as f:
            history = f.readlines()
        history = [e.split('::') for e in history]
    except Exception as e:
        print(e)
        history = [['never built', '']]
    return [dict(zip(['date', 'workflow'], e)) for e in history]
Ejemplo n.º 29
0
def stats():
    global __stats_client

    if not __stats_client:
        cfg = config()
        cfg.get('STATS_HOST', 'localhost')
        cfg.get('STATS_PORT', 8125)

        __stats_client = statsd.StatsClient(cfg.STATS_HOST, cfg.STATS_PORT)

    return __stats_client
Ejemplo n.º 30
0
def remove_recipe(recipe_id):
    """ gets all a users' recipes """
    result = False
    conn = None
    get_cooked_recipes = """SELECT "RecipeId" FROM "CookedRecipes"
                              WHERE "RecipeId" = %s
    """

    delete_recipe = """DELETE FROM "Recipes"
                         WHERE "RecipeId" = %s
    """

    delete_ingredients = """DELETE FROM "IngredientsForRecipe"
                                  WHERE "RecipeId" = %s
    """

    delete_categories = """DELETE FROM "RecipeCategories"
                                  WHERE "RecipeId" = %s
    """

    try:
        # read database configuration
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        cur = conn.cursor()

        # check if user exists
        cur.execute(get_cooked_recipes, (recipe_id, ))

        cooked = cur.fetchall()

        # do not allow cooked recipes to be deleted
        if len(cooked) > 0:
            return result

        cur.execute(delete_ingredients, (recipe_id, ))
        cur.execute(delete_categories, (recipe_id, ))
        cur.execute(delete_recipe, (recipe_id, ))

        conn.commit()

        # close the cursor
        cur.close()

        result = True
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()
    return result
Ejemplo n.º 31
0
import sys
import time
import errno
import socket
import select
import multiprocessing
from inspect import currentframe

workdir = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, workdir + "/../")

from utils.config import config
from utils.log import Logger

# 获取log配置文件
logconf = config('nbnet', 'log')
# 调用日志模块
logger = Logger.getLogger()
# debug开关,开启后记录debug日志
debug = logconf['debug']


class DebugLog():
    '''debug日志模块会显示运行的文件和行数'''
    def __init__(self, file_path, debug=True):
        self.file_path = file_path
        self.debug = debug

    def get_linenumber(self):
        '''获取函数运行在一行'''
        cf = currentframe()
Ejemplo n.º 32
0
'''Function Filter
检测客户端发送数据有误异常
'''

import os
import sys
import json

workdir = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, workdir + "/../")

from utils.config import config
from simpleNet.nbNetFramework import bind_socket, nbNet

# ff配置文件
ff_conf = config('nbnet', 'functionfilter')
host_alarm = config('alarm', 'host_alarm') 

# alarm 状态记录字典
alarmStatus = {}

def ff(data):
    '''根据host_alarm获取的值与获取的数据类型进行比较'''
    mon_data = json.loads(data)
    alarm_list = list(host_alarm)
    for key in alarm_list:
        mon_value = mon_data[key]
        alarm_value = host_alarm[key]
        eval_function = str(mon_value) + alarm_value
        ff_result = eval(eval_function)
        if ff_result:
Ejemplo n.º 33
0
#!/usr/bin/env python
# coding=utf-8

import os
import sys

workdir = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, workdir + "/../")

from utils.config import config
from simpleNet.nbNetFramework import bind_socket, nbNet
from simpleNet.nbNetUtils import sendData_mh

# 导入配置文件
trans_conf = config('nbnet', 'trans')
# 监控服务器列表
ff_l = trans_conf['ff_l'].split(';')
# 服务器主机列表
saver_l = trans_conf['saver_l'].split(';')
# ff 和 saver的soket,使用列表,具体参考sendData_mh
# ff socket
ff_sock_l = [None]
# saver socket
saver_sock_l = [None]

# 发送给服务端data是发送的数据
def sendsaver(saver_l, data, sock_l):
    return sendData_mh(saver_l, data, sock_l)

#发送给监控端
def sendff(ff_l, data, sock_l):
Ejemplo n.º 34
0
#!/usr/bin/env python
# coding=utf-8

import os
import sys
import json

workdir = os.path.dirname(os.path.realpath(__file__))
sys.path.insert(0, workdir + "/../")

from utils.config import config
from utils.crypt import encrypt,decrypt
from utils.syscmd import Command, get_iphostname
from simpleNet.nbNetFramework import bind_socket, nbNet

ctrl_conf = config('nbnet', 'controller')

# 监听地址和端口
addr = ctrl_conf['addr']
port = int(ctrl_conf['port'])
def_timeout = ctrl_conf['timeout']

#处理程序
def logic(data):
    dec_data = decrypt(data)
    data = json.loads(dec_data)
    if "cmd" in data:
        send_data = get_iphostname()
        cmd = data['cmd']
        timeout =  data.get("timeout", def_timeout)
        command = Command(cmd)