def __init__(self):
self.tm = TaskManager()
s = socket.socket()
s.bind((host,port))
s.listen()
print("listen")
while True:
client,addr = s.accept()
_thread.start_new_thread(self.workThread,(client,))
def test_saving(self):
manager_params, browser_params = self.get_config()
manager = TaskManager.TaskManager(manager_params, browser_params)
manager.get('http://example.com')
manager.close()
assert isfile(join(browser_params[0]['profile_archive_dir'],
'profile.tar.gz'))
def test_save_screenshot_valid(self, tmpdir):
"""Check that 'save_screenshot' works"""
# Run the test crawl
manager_params, browser_params = self.get_config(str(tmpdir))
manager = TaskManager.TaskManager(manager_params, browser_params)
cs = CommandSequence.CommandSequence(url_a)
cs.get(sleep=1)
cs.save_screenshot('test')
cs.screenshot_full_page('test_full')
manager.execute_command_sequence(cs)
manager.close()
# Check that viewport image is not blank
pattern = os.path.join(str(tmpdir), 'screenshots', '1-*-test.png')
screenshot = glob.glob(pattern)[0]
im = Image.open(screenshot)
bands = im.split()
is_blank = all(band.getextrema() == (255, 255) for band in bands)
assert not is_blank
# Check that full page screenshot is not blank
pattern = os.path.join(str(tmpdir), 'screenshots', '1-*-test_full.png')
screenshot = glob.glob(pattern)[0]
im = Image.open(screenshot)
bands = im.split()
is_blank = all(band.getextrema() == (255, 255) for band in bands)
assert not is_blank
def test_get_site_visits_table_valid(self):
"""Check that get works and populates db correctly."""
# Run the test crawl
manager_params, browser_params = self.get_config()
manager = TaskManager.TaskManager(manager_params, browser_params)
# Set up two sequential get commands to two URLS
cs_a = CommandSequence.CommandSequence(url_a)
cs_a.get(sleep=1)
cs_b = CommandSequence.CommandSequence(url_b)
cs_b.get(sleep=1)
# Perform the get commands
manager.execute_command_sequence(cs_a)
manager.execute_command_sequence(cs_b)
manager.close()
qry_res = db_utils.query_db(manager_params['db'],
"SELECT site_url FROM site_visits")
# We had two separate page visits
assert len(qry_res) == 2
assert qry_res[0][0] == url_a
assert qry_res[1][0] == url_b
def test_content_saving(self, tmpdir):
""" check that content is saved and hashed correctly """
test_url = utilities.BASE_TEST_URL + '/http_test_page.html'
manager_params, browser_params = self.get_test_config(str(tmpdir))
browser_params[0]['http_instrument'] = True
browser_params[0]['save_all_content'] = True
manager = TaskManager.TaskManager(manager_params, browser_params)
manager.get(url=test_url, sleep=1)
manager.close()
db = manager_params['db']
rows = db_utils.query_db(db, "SELECT * FROM http_responses;")
disk_content = dict()
for row in rows:
if 'MAGIC_REDIRECT' in row['url'] or '404' in row['url']:
continue
path = urlparse(row['url']).path
with open(os.path.join(BASE_PATH, path[1:]), 'rb') as f:
content = f.read()
chash = sha256(content).hexdigest()
# TODO: webext instrumentation doesn't save the content_hash yet.
# assert chash == row['content_hash']
disk_content[chash] = content
ldb_content = dict()
for chash, content in db_utils.get_javascript_content(str(tmpdir)):
chash = chash.decode('ascii')
ldb_content[chash] = content
for k, v in disk_content.items():
assert v == ldb_content[k]
def test_extension_gets_correct_visit_id(self):
manager_params, browser_params = self.get_config()
manager = TaskManager.TaskManager(manager_params, browser_params)
url_a = utilities.BASE_TEST_URL + '/simple_a.html'
url_b = utilities.BASE_TEST_URL + '/simple_b.html'
manager.get(url_a)
manager.get(url_b)
manager.close()
qry_res = db_utils.query_db(
manager_params['db'], "SELECT visit_id, site_url FROM site_visits")
# Construct dict mapping site_url to visit_id
visit_ids = dict()
for row in qry_res:
visit_ids[row[1]] = row[0]
simple_a_visit_id = db_utils.query_db(
manager_params['db'], "SELECT visit_id FROM javascript WHERE "
"symbol=?", ("window.navigator.userAgent", ))
simple_b_visit_id = db_utils.query_db(
manager_params['db'], "SELECT visit_id FROM javascript WHERE "
"symbol=?", ("window.navigator.platform", ))
assert visit_ids[url_a] == simple_a_visit_id[0][0]
assert visit_ids[url_b] == simple_b_visit_id[0][0]
def test_crash(self):
manager_params, browser_params = self.get_config()
manager_params['failure_limit'] = 0
manager = TaskManager.TaskManager(manager_params, browser_params)
with pytest.raises(CommandExecutionError):
manager.get('http://example.com') # So we have a profile
manager.get('example.com') # Selenium requires scheme prefix
manager.get('example.com') # Requires two commands to shut down
def visit(self, page_url, data_dir="", sleep_after=0):
"""Visit a test page with the given parameters."""
manager_params, browser_params = self.get_config(data_dir)
manager = TaskManager.TaskManager(manager_params, browser_params)
if not page_url.startswith("http"):
page_url = utilities.BASE_TEST_URL + page_url
manager.get(url=page_url, sleep=sleep_after)
manager.close()
return manager_params['db']
def setUp(self):
self.env = environment.MockEnvironment()
self.env.removeTestDirs() # just in case
self.env.makeDirs()
mysqlarchiver._deleteMySQLArchiveDb() # just in case
(self.env.con,self.env.cur) = mysqlarchiver.initMySQLArchiveDb()
self.framedb = archiver.SimFrameDBint(self.env)
#import test_archiver
#self.framedb = test_archiver.SimTestFrameDBint(self.env)
self.iaq = 'SUKA'
self.archmon = archiver.ArchiveMonitor(self.framedb,self.env)
self.archivemgr = archiver.ArchiveMgr(self.framedb,self.archmon, self.env)
self.framesrcSUKA = self.archivemgr.createFrameSourceA("SUK")
self.basic = ['SIMPLE','BITPIX','NAXIS','EXTEND','PROP-ID',
'FRAMEID','RANDOM','OBCPMODE']
self.basic.sort()
self.archivemgr.registerKeywords(self.iaq,self.basic)
import random
#randokm.seed(0) # force a random but repeatable sequence
self.random = str(random.randrange(1000,1010))
self.mode = "mode_"+str(random.randrange(0,10))
self.obcpnum = 9
self.ev_quit = threading.Event()
# Timeout value for this task
self.timeout = 20.0
# Create monitor and other allocations
self.monitor_name = 'monitor'
self.monitor = Monitor.Minimon(self.monitor_name, logger=logger)
myhost = SOSSrpc.get_myhost(short=True)
statusDict = {}
# Create and start the instrument interface
self.insint = INSint.ocsInsInt(self.obcpnum, myhost, self.env.ARCH_PROP,
statusDict, db=self.monitor,
logger=logger, ev_quit=self.ev_quit)
self.insint.start()
# This is our set of allocations
self.alloc = Bunch.Bunch(monitor=self.monitor,
INSint9=self.insint,
archmgr=self.archivemgr,
status=None)
# Create and start SIMCAM
ocsint = DAQtk.ocsInt(self.obcpnum, logger=logger, ev_quit=self.ev_quit)
simcam = SIMCAM.SIMCAM(logger, self.env, ev_quit=self.ev_quit)
self.simcam = SIMCAM.OCSInterface(logger, ev_quit=self.ev_quit,
ocsint=ocsint, instrument=simcam)
self.simcam.start()
# Create and start our task manager
self.tm = TaskManager.TaskManager(logger=logger, internal_allocs=self.alloc)
self.tm.setAllocs(['monitor', 'INSint9', 'archmgr'])
self.tm.start()
def setUp(self):
self.task_manager = TaskManager.TaskManager('test.db')
self.username = '******'
self.password = '******'
self.email = '*****@*****.**'
self.task = ('Write more tests!', '1993-12-11')
self.unique_task = ('Do not re-use me!', '{year}-{month}-{day}'.format(
year=TimeManager.CURRENT_YEAR,
month=TimeManager.CURRENT_MONTH,
day=TimeManager.CURRENT_DAY))
def load_modules(self):
'''
载入7个模块
'''
self.node_check = NodeCheck.NodeCheck()
self.node_connect = NodeConnect.NodeConnect()
self.node_manager = NodeManager.NodeManager()
self.node_source_manager = NodeSourceManager.NodeSourceManager()
self.task_manager = TaskManager.TaskManager()
self.task_schedule = TaskSchedule.TaskSchedule()
self.web = Web.Web()
def test_createAddTask():
taskManager = TaskManager.TaskManager([])
nbTask = len(taskManager.tasks)
input = "+ Finish my book"
action = taskManager.parseCommand(input)
assert action.type == "add"
if (action.type == "add"):
task = taskManager.createTask(nbTask, action)
assert task.id == 0
assert task.title == "Finish my book"
assert taskManager.tasks[task.id] == task
assert len(taskManager.tasks) == 1
def remove_task():
username = current_user.username
task_manager = TaskManager.TaskManager()
id = request.args['id']
task_manager.remove_task_by_id(id)
all_tasks = task_manager.retrieve_tasks(username)
table = None
if all_tasks:
items = ItemTable.objectify(all_tasks)
table = ItemTable.ItemTable(items)
return redirect(url_for('index', username=username, table=table))
return redirect(url_for('index', username=username))
def get_a_task():
# 获取一个task
TM = TaskManager.TaskManager()
a = TM.get_task_in_queue()
# 将task中的部分内容转换成json字符串
data = {
'id': a.index,
'passcode': a.passcode,
'sourcefile_path': a.sourcefile_path,
}
json_str = json.dumps(data)
# 将json字符串返回
emit('GET_TASK_response', json_str)
def test_deleteTask():
taskManager = TaskManager.TaskManager([])
nbTask = len(taskManager.tasks)
input = "+ Make a backup of my VPS"
action = taskManager.parseCommand(input)
assert action.type == "add"
if (action.type == "add"):
task = taskManager.createTask(nbTask, action)
assert task.id == 0
assert task.title == "Make a backup of my VPS"
assert taskManager.tasks[task.id] == task
print(taskManager.tasks[task.id])
taskManager.deleteTask(0)
assert len(taskManager.tasks) == 0
def test_parseInValidCommand():
taskManager = TaskManager.TaskManager([])
inputMinus = "- Learn C++"
action = taskManager.parseCommand(inputMinus)
assert action is False
inputMinus = "x Learn C++"
action = taskManager.parseCommand(inputMinus)
assert action is False
inputMinus = "o Learn C++"
action = taskManager.parseCommand(inputMinus)
assert action is False
def test_UpdateTaskToDone():
taskManager = TaskManager.TaskManager([])
nbTask = len(taskManager.tasks)
input = "+ Watch the Synder's Cut"
action = taskManager.parseCommand(input)
assert action.type == "add"
if (action.type == "add"):
task = taskManager.createTask(nbTask, action)
assert task.id == 0
assert task.title == "Watch the Synder's Cut"
assert taskManager.tasks[task.id] == task
assert task.status is None
taskManager.changeTaskStatusToDone(task.id)
assert task.status == "Done"
def test_UpdateTaskToDo():
taskManager = TaskManager.TaskManager([])
nbTask = len(taskManager.tasks)
input = "+ I really have to do this one"
action = taskManager.parseCommand(input)
assert action.type == "add"
if (action.type == "add"):
task = taskManager.createTask(nbTask, action)
assert task.id == 0
assert task.title == "I really have to do this one"
assert taskManager.tasks[task.id] == task
assert task.status is None
taskManager.changeTaskStatusToDo(task.id)
assert task.status == "toDo"
def setUp(self):
import rpc
self.ev_quit = threading.Event()
# Create monitor and other allocations
self.monitor = Monitor.Minimon(TestTCSCmds.monitorName, logger)
#
# commandManager = CommandManager.CommandManager(None)
# commandSender = CommandManager.CommandSender(commandManager=commandManager,
# timeout=0.1,
# host='localhost',
# prog=0x20000011,
# uid=CommandManager.TCS_UID,
# gid=CommandManager.TCS_GID )
# #humm...
# commandManager.commandQueue = commandSender.commandQueue
# commandReceiver = CommandManager.ReplyReceiver(host="",
# prog=0x20000012,
# vers=1,
# port=0,
# commandManager=commandManager)
commandManager = TCSintTestCase.MockCommandManager()
commandManager.mockResult = True
commandSender = TCSintTestCase.MockTCSClient()
commandReceiver = TCSintTestCase.MockReplyReceiver(rpc.Packer(),
rpc.Unpacker('\x00\x00\x00=CEOBS%TSC%00002006020719514370028000005930020000000COMPLETE%%\x00\x00\x00'),
commandManager)
ddSequenceGenerator = TCSint.FakeSequenceGenerator()
self.tCSint = TCSint.TCSint(self.monitor,
commandManager,
commandSender,
commandReceiver,
ddSequenceGenerator)
self.tCSint.start()
self.alloc = Bunch.Bunch(monitor=self.monitor,
TSC=self.tCSint,
status=None)
# Create and start our task manager
self.tm = TaskManager.TaskManager(logger=logger, ev_quit=self.ev_quit,
internal_allocs=self.alloc)
self.tm.setAllocs([TestTCSCmds.monitorName,
TestTCSCmds.intName])
self.tm.start()
def test_crash_profile(self):
manager_params, browser_params = self.get_config()
manager_params['failure_limit'] = 2
manager = TaskManager.TaskManager(manager_params, browser_params)
try:
manager.get('http://example.com') # So we have a profile
manager.get('example.com') # Selenium requires scheme prefix
manager.get('example.com') # Selenium requires scheme prefix
manager.get('example.com') # Selenium requires scheme prefix
manager.get('example.com') # Requires two commands to shut down
except CommandExecutionError:
pass
assert isfile(join(browser_params[0]['profile_archive_dir'],
'profile.tar.gz'))
def test_TaskAndDatabase():
# On vide la table task de la base de données
cur.execute("DELETE FROM task")
conn.commit()
taskManager = TaskManager.TaskManager([])
nbTask = len(taskManager.tasks)
input = "+ Upload this task to database"
action = taskManager.parseCommand(input)
task = taskManager.createTask(nbTask, action)
assert task.id == 0
assert task.title == "Upload this task to database"
assert taskManager.tasks[task.id] == task
taskManager.InsertTaskDatabase(task.id, task.title, task.status)
assert len(taskManager.tasks) == 1
idData = cur.execute("SELECT id FROM task WHERE id = 0")
assert (idData.fetchone())[:][0] == task.id
titleData = cur.execute("SELECT title FROM task where id = ?", (task.id, ))
assert (titleData.fetchone())[:][0] == task.title
statusData = cur.execute("SELECT status FROM task where id = ?",
(task.id, ))
assert (statusData.fetchone())[:][0] == task.status
taskManager.changeTaskStatusToDo(task.id)
assert task.status == "toDo"
taskManager.UpdateTaskDatabase(task.id, task.title, task.status)
idData = cur.execute("SELECT id FROM task where id = ?", (task.id, ))
assert (idData.fetchone())[:][0] == task.id
titleData = cur.execute("SELECT title FROM task where id = ?", (task.id, ))
assert (titleData.fetchone())[:][0] == task.title
statusData = cur.execute("SELECT status FROM task where id = ?",
(task.id, ))
assert (statusData.fetchone())[:][0] == task.status
conn.commit()
assert len(taskManager.tasks) == 1
# On imagine une déconnexion de l'utilisateur, la liste local est donc vidée pour simuler un relancement du programme
taskManager.tasks = []
assert len(taskManager.tasks) == 0
taskManager.getTaskFromDatabase()
assert len(taskManager.tasks) != 0
task = taskManager.tasks[0]
assert task.id == 0
assert task.title == "Upload this task to database"
assert task.status == "toDo"
# On vide la table task de la base de données
# Bien sûr on imagine qu'on aurait une base pour les tests afin de ne pas
# supprimer les données en production
cur.execute("DELETE FROM task")
conn.commit()
def index():
username = current_user.username.title()
task_manager = TaskManager.TaskManager()
table = None
form = forms.CreateTaskForm(request.form)
all_tasks = task_manager.retrieve_tasks(username)
if all_tasks:
items = ItemTable.objectify(all_tasks)
table = ItemTable.ItemTable(items)
if request.method == 'GET':
if table:
return render_template('index.html',
username=username,
table=table,
title='Home',
form=form)
return render_template('index.html',
username=username,
title='Home',
form=form)
if form.validate_on_submit():
error = None
time_manager = TimeManager.TimeManager()
task = form.task.data
timestamp = form.timestamp.data
given_date, given_time = str(timestamp).split(' ')
if not time_manager.create_timestamp(given_date, given_time):
error = 'Date/Time must be after current Date/Time'
return render_template('index.html',
username=username,
error=error,
title='Home',
form=form)
timestamp = time_manager.timestamp
task_manager.insert_new_task(task, timestamp, username)
all_tasks = task_manager.retrieve_tasks(username)
if all_tasks:
items = ItemTable.objectify(all_tasks)
table = ItemTable.ItemTable(items)
return render_template('index.html',
username=username,
table=table,
title='Home',
form=form)
return render_template('index.html',
username=username,
title='Home',
form=form)
def __init__(self):
QtCore.QObject.__init__(self)
self.tcpServer = QtNetwork.QTcpServer(self)
self.clientConnection = None
self.leftovers = ""
self.readlength = 0
self.connect(self.tcpServer, QtCore.SIGNAL("newConnection()"),
self.newConnection)
global server
server = self
self.tm = TaskManager(self)
self.timer = QtCore.QTimer()
self.status = None
self.connect(self.timer, QtCore.SIGNAL("timeout()"), self.refresh)
def handle(self):
self.logger.debug('handle')
data = self.request.recv(1024)
self.logger.debug(data)
clearData = data.replace('\n', '')
if clearData == 'shutdown':
self.logger.debug('server shutdown!')
response = 'server shutdown!'
self.request.sendall(response)
self.server.shutdown()
return
if clearData == 'echo':
response = 'echo'
self.request.sendall(response)
return
match = re.match(
ur'get state (?P<uid>[\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12})',
clearData)
if match != None:
uid = match.group('uid')
for task in self.server.taskList:
print 'task getuid ' + str(task.getUid())
print 'requect uid ' + uid
if str(task.getUid()) == uid:
response = task.getState()
self.request.sendall(response)
return
response = 'task not found'
self.request.sendall(response)
return
schema = CollectorTarget(data)
taskManager = TaskManager.TaskManager(schema)
self.server.taskList = taskManager.start()
for task in self.server.taskList:
response = "task " + str(task.getUid()) + " acepted!"
self.request.send(response)
self.request.close()
def test_flash_cookies(self):
""" Check that some Flash LSOs are saved and
are properly keyed in db."""
# Run the test crawl
manager_params, browser_params = self.get_config()
browser_params[0]['disable_flash'] = False
manager = TaskManager.TaskManager(manager_params, browser_params)
# Get a site we know sets Flash cookies and visit it twice
lso_value_a = utilities.rand_str(8)
expected_lso_content_a[5] = lso_value_a # expected to be present
qry_str = '?lso_test_key=%s&lso_test_value=%s' % ("test_key",
lso_value_a)
test_url_a = utilities.BASE_TEST_URL + '/lso/setlso.html' + qry_str
cs = CommandSequence.CommandSequence(test_url_a)
cs.get(sleep=3, timeout=120)
cs.dump_flash_cookies()
manager.execute_command_sequence(cs)
lso_value_b = utilities.rand_str(8)
expected_lso_content_b[5] = lso_value_b # expected to be present
qry_str = '?lso_test_key=%s&lso_test_value=%s' % ("test_key",
lso_value_b)
test_url_b = utilities.BASE_TEST_URL + '/lso/setlso.html' + qry_str
cs = CommandSequence.CommandSequence(test_url_b)
cs.get(sleep=3, timeout=120)
cs.dump_flash_cookies()
manager.execute_command_sequence(cs)
manager.close()
# Check that some flash cookies are recorded
qry_res = db_utils.query_db(manager_params['db'],
"SELECT * FROM flash_cookies",
as_tuple=True)
lso_count = len(qry_res)
assert lso_count == 2
lso_content_a = list(qry_res[0][2:]) # Remove first two items
lso_content_b = list(qry_res[1][2:]) # Remove first two items
# remove randomly generated LSO directory name
# e.g. TY2FOJUG/localtest.me/Flash.sol -> localtest.me/Flash.sol
lso_content_a[3] = lso_content_a[3].split("/", 1)[-1] # rm LSO dirname
lso_content_b[3] = lso_content_b[3].split("/", 1)[-1] # rm LSO dirname
assert lso_content_a == expected_lso_content_a
assert lso_content_b == expected_lso_content_b
def test_printTaskList():
# Pour simplifier le test nous allons créés plusieurs tâches
# qui seront toujours correctement déclarées par l'utilisateur
taskManager = TaskManager.TaskManager([])
nbTask = len(taskManager.tasks)
input_1 = "+ Task one"
action_1 = taskManager.parseCommand(input_1)
task_1 = taskManager.createTask(nbTask, action_1)
assert len(taskManager.tasks) == 1 and taskManager.tasks[0] == task_1
input_2 = "+ Task two"
action_2 = taskManager.parseCommand(input_2)
nbTask = len(taskManager.tasks)
task_2 = taskManager.createTask(nbTask, action_2)
assert len(taskManager.tasks) == 2 and taskManager.tasks[1] == task_2
taskManager.changeTaskStatusToDone(task_1.id)
assert task_1.status == "Done"
taskManager.changeTaskStatusToDone(task_2.id)
assert task_2.status == "Done"
def test_checkValidUserInput():
taskManager = TaskManager.TaskManager([])
inputPlusGood = "+ Add fantastic task"
boolTrue = taskManager.checkUserInput(inputPlusGood.split(" "))
assert boolTrue == True
inputMinusGood = "- 1"
boolTrue = taskManager.checkUserInput(inputMinusGood.split(" "))
assert boolTrue == True
inputCrossGood = "x 1"
boolTrue = taskManager.checkUserInput(inputCrossGood.split(" "))
assert boolTrue == True
inputRoundGood = "o 1"
boolTrue = taskManager.checkUserInput(inputRoundGood.split(" "))
assert boolTrue == True
def test_profile_cookies(self):
""" Check that some profile cookies are saved """
# Run the test crawl
manager_params, browser_params = self.get_config()
manager = TaskManager.TaskManager(manager_params, browser_params)
# TODO update this to local test site
url = 'http://www.yahoo.com'
cs = CommandSequence.CommandSequence(url)
cs.get(sleep=3, timeout=120)
cs.dump_profile_cookies()
manager.execute_command_sequence(cs)
manager.close()
# Check that some flash cookies are recorded
qry_res = db_utils.query_db(manager_params['db'],
"SELECT COUNT(*) FROM profile_cookies")
prof_cookie_count = qry_res[0][0]
assert prof_cookie_count > 0
def test_get_http_tables_valid(self):
"""Check that get works and populates http tables correctly."""
# Run the test crawl
manager_params, browser_params = self.get_config()
manager = TaskManager.TaskManager(manager_params, browser_params)
# Set up two sequential get commands to two URLS
cs_a = CommandSequence.CommandSequence(url_a)
cs_a.get(sleep=1)
cs_b = CommandSequence.CommandSequence(url_b)
cs_b.get(sleep=1)
manager.execute_command_sequence(cs_a)
manager.execute_command_sequence(cs_b)
manager.close()
qry_res = db_utils.query_db(
manager_params['db'],
"SELECT visit_id, site_url FROM site_visits")
# Construct dict mapping site_url to visit_id
visit_ids = dict()
for row in qry_res:
visit_ids[row[1]] = row[0]
qry_res = db_utils.query_db(manager_params['db'],
"SELECT visit_id FROM http_requests"
" WHERE url = ?", (url_a,))
assert qry_res[0][0] == visit_ids[url_a]
qry_res = db_utils.query_db(manager_params['db'],
"SELECT visit_id FROM http_requests"
" WHERE url = ?", (url_b,))
assert qry_res[0][0] == visit_ids[url_b]
qry_res = db_utils.query_db(manager_params['db'],
"SELECT visit_id FROM http_responses"
" WHERE url = ?", (url_a,))
assert qry_res[0][0] == visit_ids[url_a]
qry_res = db_utils.query_db(manager_params['db'],
"SELECT visit_id FROM http_responses"
" WHERE url = ?", (url_b,))
assert qry_res[0][0] == visit_ids[url_b]
def test_record_file_upload(self):
"""Test that we correctly capture the uploaded file contents.
We upload a CSS file and a PNG file to test both text based and
binary files.
File uploads are not expected in the crawl data, but we make sure we
correctly parse the POST data in this very common scenario.
"""
img_file_path = os.path.abspath("test_pages/shared/test_image.png")
css_file_path = os.path.abspath("test_pages/shared/test_style.css")
def type_filenames_into_form(**kwargs):
"""Simulate typing into the file upload input fields."""
driver = kwargs['driver']
img_file_upload_element = driver.find_element_by_id("upload-img")
css_file_upload_element = driver.find_element_by_id("upload-css")
img_file_upload_element.send_keys(img_file_path)
css_file_upload_element.send_keys(css_file_path)
sleep(5) # wait for the form submission (3 sec after onload)
manager_params, browser_params = self.get_config()
manager = TaskManager.TaskManager(manager_params, browser_params)
test_url = utilities.BASE_TEST_URL + "/post_file_upload.html"
cs = CommandSequence.CommandSequence(test_url)
cs.get(sleep=0, timeout=60)
cs.run_custom_function(type_filenames_into_form, ())
manager.execute_command_sequence(cs)
manager.close()
post_body = self.get_post_request_body_from_db(manager_params['db'])
# Binary strings get put into the database as-if they were latin-1.
with open(img_file_path, 'rb') as f:
img_file_content = f.read().strip().decode('latin-1')
with open(css_file_path, 'rt') as f:
css_file_content = f.read().strip()
# POST data is stored as JSON in the DB
post_body_decoded = json.loads(post_body)
expected_body = {
u"username": u"name surname+",
u"upload-css": css_file_content,
u"upload-img": img_file_content
}
assert expected_body == post_body_decoded
def annotate_from_db(user_id, db, dataset_path):
taskMgr = TaskManager(user_id, db)
if taskMgr.is_cur_task_complete():
print 'Closing batch #%d'%taskMgr.get_cur_batchid()
taskMgr.terminate_cur_task()
print 'Closed'
print '--------------------------------------------'
print 'Annotating batch #%d'%(taskMgr.get_cur_batchid())
exit = False
keep_annotating = True
while keep_annotating:
anno_list, last_pos = taskMgr.get_annotation_list()
for cur_pos in range(last_pos+1, len(anno_list)):
anno_task = anno_list[cur_pos]
local_video_path = os.path.join(dataset_path, anno_task.video_path)
if not os.path.isfile(local_video_path):
# FILE NOT FOUND
print ''
print 'Could not complete annotation, video FILE NOT FOUND'
print '>', local_video_path
if yesno_menu('Would you like to continue? The annotation will be stored as FILE NOT FOUND.'):
#YES answer
print 'Storing annotation to database with FILE NOT FOUND error'
start_frame = -1
end_frame = -1
anno_task.status = AnnotationTask.STATUS_FILE_NOT_FOUND
else:
#NO answer
exit_program() #Terminates program
else:
# Display GUI for annotating
print 'Annotating: %s'%os.path.basename(anno_task.video_path)
print 'Caption: %s'%anno_task.text
exit, skipped, start_frame, end_frame, ss = display_video_capture(local_video_path, caption=anno_task.text, action=anno_task.action.upper())
if skipped:
start_frame = -1
end_frame = -1
anno_task.status = AnnotationTask.STATUS_SKIPPED
else:
anno_task.status = AnnotationTask.STATUS_OK
if exit:
exit_program() #Terminates program
taskMgr.save_annotation(anno_task.video_path, anno_task.dataset, start_frame, end_frame, anno_task.action, user_id, anno_task.text, anno_task.status)
taskMgr.update_annotask_pos(cur_pos)
# END FOR - cycle on list of tasks
if taskMgr.is_cur_task_complete():
print 'Closing batch #%d'%taskMgr.get_cur_batchid()
print 'Closing current batch and opening a new one.'
taskMgr.terminate_cur_task()
print 'Closed'
keep_annotating = yesno_menu('Would you like to keep on annotating?')