def test_set_methods(self):
""" this tests all of the set methods since they need to work together """
save_config_file = config.ConfigObject.CONFIG_FILE
# Test the default with no file is 'unittest'
config.reset()
config.ConfigObject.CONFIG_FILE = 'badfile.xxx'
self.assertEqual(config.get_environment(), 'unittest')
config.ConfigObject.CONFIG_FILE = save_config_file
# test that if we the database name we have an unknown environmentwe
config.reset()
config.set_database_name(':memory:')
self.assertEqual(config.get_environment(), '????')
# test that if we the database name we have an unknown environmentwe
config.reset()
config.set_enviornment('test')
self.assertTrue(config.get_database_name())
# test that we have an instance and database
config.reset()
self.assertTrue(config.get_environment()) # All these values must be set... Can't test to what though
self.assertTrue(config.get_database_name())
self.assertTrue(config.get_database_instance())
self.assertTrue(config.get_database())
def test_set_methods(self):
""" this tests all of the set methods since they need to work together """
save_config_file = config.ConfigObject.CONFIG_FILE
# Test the default with no file is 'unittest'
config.reset()
config.ConfigObject.CONFIG_FILE = 'badfile.xxx'
self.assertEqual(config.get_environment(), 'unittest')
config.ConfigObject.CONFIG_FILE = save_config_file
# test that if we the database name we have an unknown environmentwe
config.reset()
config.set_database_name('MyTest.db')
self.assertEqual(config.get_environment(), '????')
os.remove('MyTest.db')
# test that if we the database name we have an unknown environmentwe
config.reset()
config.set_enviornment('test')
self.assertTrue(config.get_database_name())
# test that we have an instance and database
config.reset()
self.assertTrue(config.get_environment(
)) # All these values must be set... Can't test to what though
self.assertTrue(config.get_database_name())
self.assertTrue(config.get_database_instance())
self.assertTrue(config.get_database())
def get_link_row():
utils = Utils()
db = config.get_database()
try:
print('AppServer.get_link_row running', request.method)
print('Instance:', config.get_database_name(),
config.get_environment())
print('Tables', config.get_database_instance().get_table_names())
data = utils.json_decode(request)
link = db.select("LinkTab",
TabName=data['TabName'],
AttrName=data['AttrName'])
print('link', link)
if not link:
data['ID'] = 0
data['LinkName'] = ''
data['BaseTab'] = 0
data['ShowAttrs'] = ''
else:
data['ID'] = link[0].ID
data['LinkName'] = link[0].LinkName
data['BaseTab'] = link[0].BaseTab
data['ShowAttrs'] = link[0].ShowAttrs
return json.dumps(data)
except Exception as e:
print('AppServer.link: ***Error:', e)
traceback.print_exc(file=sys.stdout)
def get_link_row():
utils = Utils()
db = config.get_database()
try:
print('AppServer.get_link_row running', request.method)
print('Instance:', config.get_database_name(), config.get_environment())
print('Tables', config.get_database_instance().get_table_names())
data = utils.json_decode(request)
link = db.select("LinkTab", TabName=data['TabName'], AttrName=data['AttrName'])
print('link', link)
if not link:
data['ID'] = 0
data['LinkName'] = ''
data['BaseTab'] = 0
data['ShowAttrs'] = ''
else:
data['ID'] = link[0].ID
data['LinkName'] = link[0].LinkName
data['BaseTab'] = link[0].BaseTab
data['ShowAttrs'] = link[0].ShowAttrs
return json.dumps(data)
except Exception as e:
print('AppServer.link: ***Error:', e)
traceback.print_exc(file=sys.stdout)
def test_execute_statement_delete_table(self):
self.assertEqual(
config.get_environment(),
'unittest') # Destructive Tests must run in unittest environment
dbi = config.get_database_instance()
dbu = DatabaseUtilities()
dbu.delete_all_tables()
self.assertEqual(len(dbi.get_table_names()), 0,
'These should be no tables in the database.')
statement = """
CREATE TABLE table1 ('ID' int, 'Name' text);
CREATE TABLE table2 ('ID' int, 'Name' text);
CREATE TABLE table3 ('ID' int, 'Name' text);
"""
dbi.execute_statement(statement)
self.assertEqual(len(dbi.get_table_names()), 3)
self.assertIn('table2', dbi.get_table_names())
dbu.delete_table('table2')
self.assertEqual(len(dbi.get_table_names()), 2)
self.assertNotIn('table2', dbi.get_table_names())
dbu.delete_all_tables()
self.assertEqual(len(dbi.get_table_names()), 0,
'These should be no tables in the database.')
def setUp(self):
self.assertEqual('unittest', config.get_environment()) # Destructive Tests must run in unittest environment
self.dbi = config.get_database_instance()
self.db = config.get_database()
self.dbu = DatabaseUtilities()
self.db_create = DatabaseCreate()
self.dbu.delete_all_tables()
def setUp(self):
self.assertEqual(config.get_environment(),'unittest') # Distructive Tests must run in unittest enviornment
self.dbi = config.get_database_instance()
self.db = config.get_database()
self.dbu = DatabaseUtilities()
self.db_create = DatabaseCreate()
self.dbu.delete_all_tables()
def setUp(self):
self.assertEqual(config.get_environment(),'unittest') # Distructive Tests must run in unittest enviornment
db_utils = DatabaseUtilities()
db_utils.delete_all_tables()
instance = config.get_database_instance()
instance.execute('create table ' + self.TEST_TABLE_NAME + ' (myKey int, myText text, myDate date)')
instance.commit()
instance.execute('insert into ' + self.TEST_TABLE_NAME + " values (1, 'Test Item One', '2016-02-14')")
instance.execute('insert into ' + self.TEST_TABLE_NAME + " values (2, 'Test Item Two', '2016-02-14')")
instance.commit()
def test_run(self):
#Testing loading an Excel spreadsheet into an sqlite3 database.
# print ("Creating temporary database %s" % self.TEST_DATABASE)
self.assertEqual(config.get_environment(),'unittest') # Distructive Tests must run in unittest enviornment
dbu = DatabaseUtilities()
dbc = DatabaseCreate()
loader = Loader()
dbu.delete_all_tables()
dbc.create_all()
dbi = config.get_database_instance()
loader.connect()
loader.open_excel(self.TEST_SPREADSHEET)
shower = Shower()
shower.connect()
#Test that the worksheet has x number of tabs
self.assertEqual(len(loader.wb.get_sheet_names()), 2)
#Test that each tab has x number of columns
self.assertEqual(len(loader.wb['Well'].columns), 12)
self.assertEqual(len(loader.wb['Royalty Master'].columns), 11)
#Test that each tab has x number of rows
self.assertEqual(len(loader.wb['Well'].rows), 9)
self.assertEqual(len(loader.wb['Royalty Master'].rows), 11)
# print(dbi.get_table_names())
# self.assertEqual(len(dbi.get_table_names()), 0)
#
# #Test that we have x number of tables
loader.load_all_sheets()
# self.assertEqual(len(dbi.get_table_names()), 2)
#test that each table has x number of columns
self.assertEqual(len(shower.show_table('Well')), 8)
self.assertEqual(len(shower.show_table('RoyaltyMaster')), 10)
#test that each table has x number of row
self.assertEqual(len(shower.show_columns('Well')), 12)
self.assertEqual(len(shower.show_columns('RoyaltyMaster')), 11)
#test column type
self.assertEqual(shower.column_type('Well', 'ID'), 'integer')
loader.commit()
def setUp(self):
self.assertEqual(
config.get_environment(),
'unittest') # Distructive Tests must run in unittest enviornment
db_utils = DatabaseUtilities()
db_utils.delete_all_tables()
instance = config.get_database_instance()
instance.execute('create table ' + self.TEST_TABLE_NAME +
' (myKey int, myText text, myDate date)')
instance.commit()
instance.execute('insert into ' + self.TEST_TABLE_NAME +
" values (1, 'Test Item One', '2016-02-14')")
instance.execute('insert into ' + self.TEST_TABLE_NAME +
" values (2, 'Test Item Two', '2016-02-14')")
instance.commit()
def test_run(self):
self.assertEqual(
config.get_environment(),
'unittest') # Distructive Tests must run in unittest enviornment
dbu = DatabaseUtilities()
# dbc = DatabaseCreate()
loader = Loader()
dbu.delete_all_tables()
dbi = config.get_database_instance()
db = config.get_database()
loader.open_excel(self.TEST_SPREADSHEET)
# Test that the worksheet has x number of tabs
self.assertEqual(len(loader.wb.get_sheet_names()), 2)
# Test that each tab has x number of columns
self.assertEqual(len(loader.wb['Well'].columns), 12)
self.assertEqual(len(loader.wb['Royalty Master'].columns), 11)
# Test that each tab has x number of rows
self.assertEqual(len(loader.wb['Well'].rows), 9)
self.assertEqual(len(loader.wb['Royalty Master'].rows), 11)
loader.load_all_sheets()
# Test that we have x number of tables
self.assertEqual(len(dbi.get_table_names()), 2)
# check the rows and columns for well
rows = db.select('Well')
self.assertEqual(len(rows), 8)
self.assertEqual(len(dbi.get_column_names()), 12)
# check the rows and columns for royalty master
rows = db.select('RoyaltyMaster')
self.assertEqual(len(rows), 10)
self.assertEqual(len(dbi.get_column_names()), 11)
def test_execute_statement_delete_table(self):
self.assertEqual(config.get_environment(), 'unittest') # Distructive Tests must run in unittest enviornment
dbi = config.get_database_instance()
dbu = DatabaseUtilities()
dbu.delete_all_tables()
self.assertEqual(len(dbi.get_table_names()), 0, 'These should be no tables in the database.')
statement = """
CREATE TABLE table1 ('ID' int, 'Name' text);
CREATE TABLE table2 ('ID' int, 'Name' text);
CREATE TABLE table3 ('ID' int, 'Name' text);
"""
dbi.execute_statement(statement)
self.assertEqual(len(dbi.get_table_names()), 3)
self.assertIn('table2', dbi.get_table_names())
dbu.delete_table('table2')
self.assertEqual(len(dbi.get_table_names()), 2)
self.assertNotIn('table2', dbi.get_table_names())
dbu.delete_all_tables()
self.assertEqual(len(dbi.get_table_names()), 0, 'These should be no tables in the database.')
def test_run(self):
self.assertEqual(config.get_environment(), 'unittest') # Distructive Tests must run in unittest enviornment
dbu = DatabaseUtilities()
# dbc = DatabaseCreate()
loader = Loader()
dbu.delete_all_tables()
dbi = config.get_database_instance()
db = config.get_database()
loader.open_excel(self.TEST_SPREADSHEET)
# Test that the worksheet has x number of tabs
self.assertEqual(len(loader.wb.get_sheet_names()), 2)
# Test that each tab has x number of columns
self.assertEqual(len(loader.wb['Well'].columns), 12)
self.assertEqual(len(loader.wb['Royalty Master'].columns), 11)
# Test that each tab has x number of rows
self.assertEqual(len(loader.wb['Well'].rows), 9)
self.assertEqual(len(loader.wb['Royalty Master'].rows), 11)
loader.load_all_sheets()
# Test that we have x number of tables
self.assertEqual(len(dbi.get_table_names()), 2)
# check the rows and columns for well
rows = db.select('Well')
self.assertEqual(len(rows), 8)
self.assertEqual(len(dbi.get_column_names()), 12)
# check the rows and columns for royalty master
rows = db.select('RoyaltyMaster')
self.assertEqual(len(rows), 10)
self.assertEqual(len(dbi.get_column_names()), 11)
def setUp(self):
self.assertEqual(config.get_environment(),'unittest') # Distructive Tests must run in unittest enviornment
self.myapp = app.test_client()
def setUp(self):
self.assertEqual(config.get_environment(), "unittest") # Distructive Tests must run in unittest enviornment
def setUp(self):
# Destructive Tests must run in unittest environment
self.assertEqual("unittest", config.get_environment())
def setUp(self):
self.assertEqual(
config.get_environment(),
'unittest') # Distructive Tests must run in unittest enviornment
def send_task_2_worker(task_id):
"""
定时任务响应函数，负责把任务按账号拆解成job, 并发送给最适合的队列
:param task_id: 任务id
:return: 成功返回True, 失败返回False
"""
try:
jobs = []
time_it_beg = datetime.datetime.now()
db_scoped_session = ScopedSession()
task = db_scoped_session.query(
Task.category, Task.configure, Task.limit_counts,
Task.succeed_counts,
Task.scheduler).filter(Task.id == task_id).first()
if not task:
logger.error(
'send_task_2_worker can not find the task, id={}. '.format(
task_id))
return False
category, task_configure, limit_counts, succeed_counts, sch_id = task
sch_mode = db_scoped_session.query(
Scheduler.mode).filter(Scheduler.id == sch_id).first()
# 对于周期性任务,每次产生的job会严格控制, 但对于一次性任务, 用户指定多少个账号，就用多少个账号
if sch_mode[0] in [1, 2]:
if limit_counts:
# 如果当前任务的成功数大于需求数, 或者成功数加上正在运行的job数目大于用于需求数110%, 则不需要继续产生job
if succeed_counts >= int(limit_counts * 1.2):
logger.warning(
'send_task_2_worker ignore, task already finished, task id={}, succeed jobs({}) >= limit counts({})*1.2'
.format(task_id, succeed_counts, limit_counts))
return True
task_running_jobs = db_scoped_session.query(Job).filter(
and_(Job.task == task_id,
Job.status == 'running')).count()
if task_running_jobs + succeed_counts >= int(
limit_counts * 1.2):
logger.warning(
'send_task_2_worker ignore, task will finish, task id={}, succeed jobs({})+running jobs({}) >= limit counts({})*1.2'
.format(task_id, succeed_counts, task_running_jobs,
limit_counts))
return True
# 一个任务正在运行job积压过多时, 暂时停止产生新的jobs
if task_running_jobs >= 10000:
logger.warning(
'task({}) jobs num={} has reached jobs limit 10000'.
format(task_id, task_running_jobs))
return True
# 根据task的类别，找到task对应的处理函数
tcg = db_scoped_session.query(TaskCategory.processor).filter(
TaskCategory.category == category).first()
if not tcg:
return False
# 每一个类型的任务都对应一个处理器
task_processor = tcg[0]
if not task_processor:
logger.error(
'Task(id={}) have no processor, ignore processing.'.format(
task_id))
return False
logger.info(
'---------send_task_2_worker task id={}. --------'.format(task_id))
# 找到任务的所有账号
res = db_scoped_session.query(TaskAccountGroup.account_id).filter(
TaskAccountGroup.task_id == task_id).all()
account_ids = [x[0] for x in res]
accounts = db_scoped_session.query(
Account.id, Account.status, Account.account, Account.password,
Account.email, Account.email_pwd, Account.gender,
Account.phone_number, Account.birthday, Account.national_id,
Account.name, Account.active_area, Account.active_browser,
Account.profile_path,
Account.configure).filter(Account.id.in_(account_ids)).all()
# agents = db_scoped_session.query(Agent.id, Agent.active_area).filter(Agent.status != -1).order_by(Agent.status).all()
# 一个任务会有多个账号， 按照账号对任务进行第一次拆分
real_accounts_num = 0
for acc in accounts:
acc_id, status, account, password, email, email_pwd, gender, phone_number, birthday, national_id, name, \
active_area, active_browser_id, profile_path, account_configure = acc
if status == 'invalid':
logger.warning(
'account status in invalid. task id={}, account id={}'.
format(task_id, acc_id))
continue
area = db_scoped_session.query(Area).filter(
Area.id == active_area).first()
queue_name = 'default'
area_id = None
if area:
area_id, queue_name = area.id, area.name
else:
logger.warning(
'There have no optimal agent for task, task id={}, account id={}, account area={}'
.format(task_id, acc_id, active_area))
active_browser = db_scoped_session.query(FingerPrint.value).filter(
FingerPrint.id == active_browser_id).first()
if get_system_args()["force_display"] == 0:
headless = True if get_environment() == 'pro' else False
else:
headless = False
# 构建任务执行必备参数
inputs = {
'system': {
'headless': headless
},
'task': {
'task_id': task_id,
'configure':
json.loads(task_configure) if task_configure else {},
},
'account': {
'account': account,
'password': password,
'status': status,
'email': email,
'email_pwd': email_pwd,
'gender': gender,
'phone_number': phone_number,
'birthday': birthday,
'national_id': national_id,
'name': name,
'active_area': active_area,
'active_browser':
json.loads(active_browser[0]) if active_browser else {},
'profile_path': profile_path,
'configure':
json.loads(account_configure) if account_configure else {}
}
}
celery_task_name = "tasks.tasks.{}".format(task_processor)
real_accounts_num += 1
track = app.send_task(celery_task_name,
args=(inputs, ),
queue=queue_name,
routing_key=queue_name)
logger.info(
'-----send sub task to worker, celery task name={}, area id={}, queue={}, '
'task id={}, account id={}, track id={}'.format(
celery_task_name, area_id, queue_name, task_id, acc_id,
track.id))
job = Job()
job.task = task_id
job.task = task_id
job.account = acc_id
job.area = area_id
job.status = 'running'
job.track_id = track.id
job.start_time = datetime.datetime.now()
jobs.append(job)
if sch_mode[0] in [1, 2]:
# 如果已经在运行的jobs,加上当前产生的jobs数量超过用户需求数量,则break, 停止生产jobs, 下个调度周期重新检测再试
total_running_jobs = task_running_jobs + real_accounts_num
if (limit_counts and total_running_jobs >= int(
limit_counts * 1.2)) or total_running_jobs >= 10000:
logger.warning(
'task({}) total running jobs num({}) is already more than limit counts({})*1.2'
.format(task_id, total_running_jobs, limit_counts))
break
# 更新任务状态为running
# task实际可用的账号数目, 会根据每次轮循时account状态的不同而变化
db_scoped_session.query(Task).filter(and_(Task.id == task_id, Task.status.in_(['new', 'pending'])))\
.update({Task.status: "running", Task.start_time: datetime.datetime.now(),
Task.real_accounts_num: real_accounts_num, Task.last_update: datetime.datetime.now()}, synchronize_session=False)
if jobs:
db_scoped_session.add_all(jobs)
db_scoped_session.commit()
logger.info(
'----send_task_2_worker send task {}, produce jobs={}, used {} seconds. '
.format(task_id, real_accounts_num,
(datetime.datetime.now() - time_it_beg).seconds))
except BaseException as e:
logger.exception(
'send_task_2_worker exception task id={}, e={}'.format(task_id, e))
db_scoped_session.rollback()
finally:
ScopedSession.remove()
return True