コード例 #1
0
 def test_set_methods(self):
     """ this tests all of the set methods since they need to work together """
     save_config_file = config.ConfigObject.CONFIG_FILE 
     
     # Test the default with no file is 'unittest'
     config.reset()
     config.ConfigObject.CONFIG_FILE = 'badfile.xxx'
     self.assertEqual(config.get_environment(), 'unittest')
     
     config.ConfigObject.CONFIG_FILE = save_config_file
     
     # test that if we the database name we have an unknown environmentwe
     config.reset()
     config.set_database_name(':memory:')
     self.assertEqual(config.get_environment(), '????')
     
     # test that if we the database name we have an unknown environmentwe
     config.reset()
     config.set_enviornment('test')
     self.assertTrue(config.get_database_name())
     
     # test that we have an instance and database
     config.reset()
     self.assertTrue(config.get_environment()) # All these values must be set... Can't test to what though
     self.assertTrue(config.get_database_name())
     self.assertTrue(config.get_database_instance())
     self.assertTrue(config.get_database())
コード例 #2
0
    def test_set_methods(self):
        """ this tests all of the set methods since they need to work together """
        save_config_file = config.ConfigObject.CONFIG_FILE

        # Test the default with no file is 'unittest'
        config.reset()
        config.ConfigObject.CONFIG_FILE = 'badfile.xxx'
        self.assertEqual(config.get_environment(), 'unittest')

        config.ConfigObject.CONFIG_FILE = save_config_file

        # test that if we the database name we have an unknown environmentwe
        config.reset()
        config.set_database_name('MyTest.db')
        self.assertEqual(config.get_environment(), '????')
        os.remove('MyTest.db')

        # test that if we the database name we have an unknown environmentwe
        config.reset()
        config.set_enviornment('test')
        self.assertTrue(config.get_database_name())

        # test that we have an instance and database
        config.reset()
        self.assertTrue(config.get_environment(
        ))  # All these values must be set... Can't test to what though
        self.assertTrue(config.get_database_name())
        self.assertTrue(config.get_database_instance())
        self.assertTrue(config.get_database())
コード例 #3
0
ファイル: views.py プロジェクト: lshumlich/python-playground
def get_link_row():
    utils = Utils()
    db = config.get_database()
    try:
        print('AppServer.get_link_row running', request.method)
        print('Instance:', config.get_database_name(),
              config.get_environment())
        print('Tables', config.get_database_instance().get_table_names())
        data = utils.json_decode(request)
        link = db.select("LinkTab",
                         TabName=data['TabName'],
                         AttrName=data['AttrName'])
        print('link', link)
        if not link:
            data['ID'] = 0
            data['LinkName'] = ''
            data['BaseTab'] = 0
            data['ShowAttrs'] = ''
        else:
            data['ID'] = link[0].ID
            data['LinkName'] = link[0].LinkName
            data['BaseTab'] = link[0].BaseTab
            data['ShowAttrs'] = link[0].ShowAttrs

        return json.dumps(data)

    except Exception as e:
        print('AppServer.link: ***Error:', e)
        traceback.print_exc(file=sys.stdout)
コード例 #4
0
def get_link_row():
    utils = Utils()
    db = config.get_database()
    try:
        print('AppServer.get_link_row running', request.method)
        print('Instance:', config.get_database_name(), config.get_environment())
        print('Tables', config.get_database_instance().get_table_names())
        data = utils.json_decode(request)
        link = db.select("LinkTab", TabName=data['TabName'], AttrName=data['AttrName'])
        print('link', link)
        if not link:
            data['ID'] = 0
            data['LinkName'] = ''
            data['BaseTab'] = 0
            data['ShowAttrs'] = ''
        else:
            data['ID'] = link[0].ID
            data['LinkName'] = link[0].LinkName
            data['BaseTab'] = link[0].BaseTab
            data['ShowAttrs'] = link[0].ShowAttrs

        return json.dumps(data)

    except Exception as e:
        print('AppServer.link: ***Error:', e)
        traceback.print_exc(file=sys.stdout)
コード例 #5
0
    def test_execute_statement_delete_table(self):
        self.assertEqual(
            config.get_environment(),
            'unittest')  # Destructive Tests must run in unittest environment
        dbi = config.get_database_instance()
        dbu = DatabaseUtilities()
        dbu.delete_all_tables()
        self.assertEqual(len(dbi.get_table_names()), 0,
                         'These should be no tables in the database.')

        statement = """
            CREATE TABLE table1 ('ID' int, 'Name' text);
            CREATE TABLE table2 ('ID' int, 'Name' text);
            CREATE TABLE table3 ('ID' int, 'Name' text);
        """
        dbi.execute_statement(statement)
        self.assertEqual(len(dbi.get_table_names()), 3)
        self.assertIn('table2', dbi.get_table_names())

        dbu.delete_table('table2')
        self.assertEqual(len(dbi.get_table_names()), 2)
        self.assertNotIn('table2', dbi.get_table_names())

        dbu.delete_all_tables()
        self.assertEqual(len(dbi.get_table_names()), 0,
                         'These should be no tables in the database.')
コード例 #6
0
    def setUp(self):
        self.assertEqual('unittest', config.get_environment())  # Destructive Tests must run in unittest environment
        self.dbi = config.get_database_instance()
        self.db = config.get_database()
        self.dbu = DatabaseUtilities()
        self.db_create = DatabaseCreate()

        self.dbu.delete_all_tables()
コード例 #7
0
 def setUp(self):
     self.assertEqual(config.get_environment(),'unittest') # Distructive Tests must run in unittest enviornment
     self.dbi = config.get_database_instance()
     self.db = config.get_database()
     self.dbu = DatabaseUtilities()
     self.db_create = DatabaseCreate()
     
     self.dbu.delete_all_tables()
コード例 #8
0
    def setUp(self):
        self.assertEqual(config.get_environment(),'unittest') # Distructive Tests must run in unittest enviornment
        db_utils = DatabaseUtilities()
        db_utils.delete_all_tables()

        instance = config.get_database_instance()
        instance.execute('create table ' + self.TEST_TABLE_NAME  + ' (myKey int, myText text, myDate date)')
        instance.commit()
        
        instance.execute('insert into ' + self.TEST_TABLE_NAME + " values (1, 'Test Item One', '2016-02-14')")
        instance.execute('insert into ' + self.TEST_TABLE_NAME + " values (2, 'Test Item Two', '2016-02-14')")
        instance.commit()
コード例 #9
0
    def test_run(self):
        #Testing loading an Excel spreadsheet into an sqlite3 database.
#         print ("Creating temporary database %s" % self.TEST_DATABASE)

        self.assertEqual(config.get_environment(),'unittest') # Distructive Tests must run in unittest enviornment

        dbu = DatabaseUtilities()
        dbc = DatabaseCreate()
        loader = Loader()
        dbu.delete_all_tables()
        dbc.create_all()
        dbi = config.get_database_instance()
        
        
        loader.connect()
        loader.open_excel(self.TEST_SPREADSHEET)
        shower = Shower()
        shower.connect()

        #Test that the worksheet has x number of tabs
        self.assertEqual(len(loader.wb.get_sheet_names()), 2)

        #Test that each tab has x number of columns
        self.assertEqual(len(loader.wb['Well'].columns), 12)
        self.assertEqual(len(loader.wb['Royalty Master'].columns), 11)

        #Test that each tab has x number of rows
        self.assertEqual(len(loader.wb['Well'].rows), 9)
        self.assertEqual(len(loader.wb['Royalty Master'].rows), 11)

#         print(dbi.get_table_names())
#         self.assertEqual(len(dbi.get_table_names()), 0)
# 
#         #Test that we have x number of tables
        loader.load_all_sheets()
#         self.assertEqual(len(dbi.get_table_names()), 2)

        #test that each table has x number of columns
        self.assertEqual(len(shower.show_table('Well')), 8)
        self.assertEqual(len(shower.show_table('RoyaltyMaster')), 10)

        #test that each table has x number of row
        self.assertEqual(len(shower.show_columns('Well')), 12)
        self.assertEqual(len(shower.show_columns('RoyaltyMaster')), 11)

        #test column type
        self.assertEqual(shower.column_type('Well', 'ID'), 'integer')
        loader.commit()
コード例 #10
0
    def setUp(self):
        self.assertEqual(
            config.get_environment(),
            'unittest')  # Distructive Tests must run in unittest enviornment
        db_utils = DatabaseUtilities()
        db_utils.delete_all_tables()

        instance = config.get_database_instance()
        instance.execute('create table ' + self.TEST_TABLE_NAME +
                         ' (myKey int, myText text, myDate date)')
        instance.commit()

        instance.execute('insert into ' + self.TEST_TABLE_NAME +
                         " values (1, 'Test Item One', '2016-02-14')")
        instance.execute('insert into ' + self.TEST_TABLE_NAME +
                         " values (2, 'Test Item Two', '2016-02-14')")
        instance.commit()
コード例 #11
0
    def test_run(self):

        self.assertEqual(
            config.get_environment(),
            'unittest')  # Distructive Tests must run in unittest enviornment

        dbu = DatabaseUtilities()
        # dbc = DatabaseCreate()
        loader = Loader()
        dbu.delete_all_tables()
        dbi = config.get_database_instance()
        db = config.get_database()

        loader.open_excel(self.TEST_SPREADSHEET)

        # Test that the worksheet has x number of tabs
        self.assertEqual(len(loader.wb.get_sheet_names()), 2)

        # Test that each tab has x number of columns
        self.assertEqual(len(loader.wb['Well'].columns), 12)
        self.assertEqual(len(loader.wb['Royalty Master'].columns), 11)

        # Test that each tab has x number of rows
        self.assertEqual(len(loader.wb['Well'].rows), 9)
        self.assertEqual(len(loader.wb['Royalty Master'].rows), 11)

        loader.load_all_sheets()

        # Test that we have x number of tables
        self.assertEqual(len(dbi.get_table_names()), 2)

        # check the rows and columns for well
        rows = db.select('Well')
        self.assertEqual(len(rows), 8)
        self.assertEqual(len(dbi.get_column_names()), 12)

        # check the rows and columns for royalty master
        rows = db.select('RoyaltyMaster')
        self.assertEqual(len(rows), 10)
        self.assertEqual(len(dbi.get_column_names()), 11)
コード例 #12
0
    def test_execute_statement_delete_table(self):
        self.assertEqual(config.get_environment(), 'unittest')  # Distructive Tests must run in unittest enviornment
        dbi = config.get_database_instance()
        dbu = DatabaseUtilities()
        dbu.delete_all_tables()
        self.assertEqual(len(dbi.get_table_names()), 0, 'These should be no tables in the database.')
        
        statement = """
            CREATE TABLE table1 ('ID' int, 'Name' text);
            CREATE TABLE table2 ('ID' int, 'Name' text);
            CREATE TABLE table3 ('ID' int, 'Name' text);
        """
        dbi.execute_statement(statement)
        self.assertEqual(len(dbi.get_table_names()), 3)
        self.assertIn('table2', dbi.get_table_names())

        dbu.delete_table('table2')
        self.assertEqual(len(dbi.get_table_names()), 2)
        self.assertNotIn('table2', dbi.get_table_names())
        
        dbu.delete_all_tables()
        self.assertEqual(len(dbi.get_table_names()), 0, 'These should be no tables in the database.')
コード例 #13
0
    def test_run(self):

        self.assertEqual(config.get_environment(), 'unittest')  # Distructive Tests must run in unittest enviornment

        dbu = DatabaseUtilities()
        # dbc = DatabaseCreate()
        loader = Loader()
        dbu.delete_all_tables()
        dbi = config.get_database_instance()
        db = config.get_database()

        loader.open_excel(self.TEST_SPREADSHEET)

        # Test that the worksheet has x number of tabs
        self.assertEqual(len(loader.wb.get_sheet_names()), 2)

        # Test that each tab has x number of columns
        self.assertEqual(len(loader.wb['Well'].columns), 12)
        self.assertEqual(len(loader.wb['Royalty Master'].columns), 11)

        # Test that each tab has x number of rows
        self.assertEqual(len(loader.wb['Well'].rows), 9)
        self.assertEqual(len(loader.wb['Royalty Master'].rows), 11)

        loader.load_all_sheets()

        # Test that we have x number of tables
        self.assertEqual(len(dbi.get_table_names()), 2)

        # check the rows and columns for well
        rows = db.select('Well')
        self.assertEqual(len(rows), 8)
        self.assertEqual(len(dbi.get_column_names()), 12)

        # check the rows and columns for royalty master
        rows = db.select('RoyaltyMaster')
        self.assertEqual(len(rows), 10)
        self.assertEqual(len(dbi.get_column_names()), 11)
コード例 #14
0
 def setUp(self):
     self.assertEqual(config.get_environment(),'unittest') # Distructive Tests must run in unittest enviornment
     self.myapp = app.test_client()
コード例 #15
0
 def setUp(self):
     self.assertEqual(config.get_environment(), "unittest")  # Distructive Tests must run in unittest enviornment
コード例 #16
0
 def setUp(self):
     # Destructive Tests must run in unittest environment
     self.assertEqual("unittest", config.get_environment())
コード例 #17
0
 def setUp(self):
     self.assertEqual(
         config.get_environment(),
         'unittest')  # Distructive Tests must run in unittest enviornment
コード例 #18
0
def send_task_2_worker(task_id):
    """
    定时任务响应函数,负责把任务按账号拆解成job, 并发送给最适合的队列
    :param task_id: 任务id
    :return: 成功返回True, 失败返回False
    """
    try:
        jobs = []
        time_it_beg = datetime.datetime.now()
        db_scoped_session = ScopedSession()
        task = db_scoped_session.query(
            Task.category, Task.configure, Task.limit_counts,
            Task.succeed_counts,
            Task.scheduler).filter(Task.id == task_id).first()
        if not task:
            logger.error(
                'send_task_2_worker can not find the task, id={}. '.format(
                    task_id))
            return False

        category, task_configure, limit_counts, succeed_counts, sch_id = task

        sch_mode = db_scoped_session.query(
            Scheduler.mode).filter(Scheduler.id == sch_id).first()

        # 对于周期性任务,每次产生的job会严格控制, 但对于一次性任务, 用户指定多少个账号,就用多少个账号
        if sch_mode[0] in [1, 2]:
            if limit_counts:
                # 如果当前任务的成功数大于需求数, 或者成功数加上正在运行的job数目大于用于需求数110%, 则不需要继续产生job
                if succeed_counts >= int(limit_counts * 1.2):
                    logger.warning(
                        'send_task_2_worker ignore, task already finished, task id={}, succeed jobs({}) >= limit counts({})*1.2'
                        .format(task_id, succeed_counts, limit_counts))
                    return True

                task_running_jobs = db_scoped_session.query(Job).filter(
                    and_(Job.task == task_id,
                         Job.status == 'running')).count()
                if task_running_jobs + succeed_counts >= int(
                        limit_counts * 1.2):
                    logger.warning(
                        'send_task_2_worker ignore, task will finish, task id={}, succeed jobs({})+running jobs({})  >= limit counts({})*1.2'
                        .format(task_id, succeed_counts, task_running_jobs,
                                limit_counts))
                    return True

                # 一个任务正在运行job积压过多时, 暂时停止产生新的jobs
                if task_running_jobs >= 10000:
                    logger.warning(
                        'task({}) jobs num={} has reached jobs limit 10000'.
                        format(task_id, task_running_jobs))
                    return True

        # 根据task的类别,找到task对应的处理函数
        tcg = db_scoped_session.query(TaskCategory.processor).filter(
            TaskCategory.category == category).first()
        if not tcg:
            return False

        # 每一个类型的任务都对应一个处理器
        task_processor = tcg[0]
        if not task_processor:
            logger.error(
                'Task(id={}) have no processor, ignore processing.'.format(
                    task_id))
            return False

        logger.info(
            '---------send_task_2_worker task id={}. --------'.format(task_id))

        # 找到任务的所有账号
        res = db_scoped_session.query(TaskAccountGroup.account_id).filter(
            TaskAccountGroup.task_id == task_id).all()
        account_ids = [x[0] for x in res]
        accounts = db_scoped_session.query(
            Account.id, Account.status, Account.account, Account.password,
            Account.email, Account.email_pwd, Account.gender,
            Account.phone_number, Account.birthday, Account.national_id,
            Account.name, Account.active_area, Account.active_browser,
            Account.profile_path,
            Account.configure).filter(Account.id.in_(account_ids)).all()

        # agents = db_scoped_session.query(Agent.id, Agent.active_area).filter(Agent.status != -1).order_by(Agent.status).all()

        # 一个任务会有多个账号, 按照账号对任务进行第一次拆分
        real_accounts_num = 0
        for acc in accounts:
            acc_id, status, account, password, email, email_pwd, gender, phone_number, birthday, national_id, name, \
            active_area, active_browser_id, profile_path, account_configure = acc

            if status == 'invalid':
                logger.warning(
                    'account status in invalid. task id={}, account id={}'.
                    format(task_id, acc_id))
                continue

            area = db_scoped_session.query(Area).filter(
                Area.id == active_area).first()
            queue_name = 'default'
            area_id = None
            if area:
                area_id, queue_name = area.id, area.name
            else:
                logger.warning(
                    'There have no optimal agent for task, task id={}, account id={}, account area={}'
                    .format(task_id, acc_id, active_area))

            active_browser = db_scoped_session.query(FingerPrint.value).filter(
                FingerPrint.id == active_browser_id).first()

            if get_system_args()["force_display"] == 0:
                headless = True if get_environment() == 'pro' else False
            else:
                headless = False
            # 构建任务执行必备参数
            inputs = {
                'system': {
                    'headless': headless
                },
                'task': {
                    'task_id': task_id,
                    'configure':
                    json.loads(task_configure) if task_configure else {},
                },
                'account': {
                    'account': account,
                    'password': password,
                    'status': status,
                    'email': email,
                    'email_pwd': email_pwd,
                    'gender': gender,
                    'phone_number': phone_number,
                    'birthday': birthday,
                    'national_id': national_id,
                    'name': name,
                    'active_area': active_area,
                    'active_browser':
                    json.loads(active_browser[0]) if active_browser else {},
                    'profile_path': profile_path,
                    'configure':
                    json.loads(account_configure) if account_configure else {}
                }
            }

            celery_task_name = "tasks.tasks.{}".format(task_processor)
            real_accounts_num += 1

            track = app.send_task(celery_task_name,
                                  args=(inputs, ),
                                  queue=queue_name,
                                  routing_key=queue_name)

            logger.info(
                '-----send sub task to worker, celery task name={}, area id={}, queue={}, '
                'task id={}, account id={}, track id={}'.format(
                    celery_task_name, area_id, queue_name, task_id, acc_id,
                    track.id))

            job = Job()
            job.task = task_id
            job.task = task_id
            job.account = acc_id
            job.area = area_id
            job.status = 'running'
            job.track_id = track.id
            job.start_time = datetime.datetime.now()
            jobs.append(job)

            if sch_mode[0] in [1, 2]:
                # 如果已经在运行的jobs,加上当前产生的jobs数量超过用户需求数量,则break, 停止生产jobs, 下个调度周期重新检测再试
                total_running_jobs = task_running_jobs + real_accounts_num
                if (limit_counts and total_running_jobs >= int(
                        limit_counts * 1.2)) or total_running_jobs >= 10000:
                    logger.warning(
                        'task({}) total running jobs num({}) is already more than limit counts({})*1.2'
                        .format(task_id, total_running_jobs, limit_counts))
                    break

        # 更新任务状态为running
        # task实际可用的账号数目, 会根据每次轮循时account状态的不同而变化
        db_scoped_session.query(Task).filter(and_(Task.id == task_id, Task.status.in_(['new', 'pending'])))\
            .update({Task.status: "running", Task.start_time: datetime.datetime.now(),
                     Task.real_accounts_num: real_accounts_num, Task.last_update: datetime.datetime.now()}, synchronize_session=False)

        if jobs:
            db_scoped_session.add_all(jobs)

        db_scoped_session.commit()

        logger.info(
            '----send_task_2_worker send task {}, produce jobs={}, used {} seconds. '
            .format(task_id, real_accounts_num,
                    (datetime.datetime.now() - time_it_beg).seconds))
    except BaseException as e:
        logger.exception(
            'send_task_2_worker exception task id={}, e={}'.format(task_id, e))
        db_scoped_session.rollback()
    finally:
        ScopedSession.remove()

    return True