def test_receive_get_log_success(self):
program_status = ProgramStatusFactory(running=True)
program = program_status.program
error_status = Status.ok({
'method': 'get_log',
'result': {
'log': 'this is the content of a logfile',
'uuid': program_status.command_uuid,
},
})
# connect webinterface
webinterface = WSClient()
webinterface.join_group('notifications')
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.receive',
path='/commands',
content={'text': error_status.to_json()},
)
# test if the webinterface gets the error message
self.assertEqual(
Status.ok({
'log': 'this is the content of a logfile',
'pid': str(program.id),
}),
Status.from_json(json.dumps(webinterface.receive())),
)
def test_receive_execute_with_error_status(self):
program_status = ProgramStatusFactory(running=True)
program = program_status.program
error_status = Status.err({
'method': 'execute',
'result': str(Exception('foobar')),
})
error_status.uuid = program_status.command_uuid
# connect webinterface
webinterface = WSClient()
webinterface.join_group('notifications')
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.receive',
path='/commands',
content={'text': error_status.to_json()},
)
query = ProgramStatusModel.objects.get(program=program)
self.assertFalse(query.running)
self.assertEqual(query.code, 'foobar')
# test if the webinterface gets the error message
self.assertEqual(
Status.ok({
'program_status': 'finished',
'pid': str(program.id),
'code': 'foobar',
}),
Status.from_json(json.dumps(webinterface.receive())),
)
def filesystem_entry(request, filesystem_id):
"""
Process requests for a single `FilesystemModel`s.
HTTP Methods
------------
DELETE:
Removes the specified entry (in the URL) from the database.
PUT:
Updates the specified entry (in the URL) in the database.
Parameters
----------
request: HttpRequest
The request which should be processed.
Returns
-------
HttpResponse:
If the HTTP method is not supported, then an
`HttpResponseForbidden` is returned.
"""
if request.method == 'DELETE':
try:
filesystem = FilesystemModel.objects.get(id=filesystem_id)
try:
fs_delete(filesystem)
return StatusResponse.ok("")
except FsimError as err:
return StatusResponse(err)
except FilesystemModel.DoesNotExist as err:
return StatusResponse(FilesystemNotExistError(err, filesystem_id))
elif request.method == 'PUT':
# create form from a new QueryDict made from the request body
# (request.PUT is unsupported) as an update (instance) of the
# existing slave
try:
model = FilesystemModel.objects.get(id=filesystem_id)
form = FilesystemForm(QueryDict(request.body), instance=model)
if form.is_valid():
filesystem = form.save(commit=False)
try:
filesystem.full_clean()
form.save()
return StatusResponse.ok('')
except ValidationError as _:
error_dict = {
'name': [
'Filesystem with this Name already exists on this Client.'
]
}
return StatusResponse(Status.err(error_dict))
else:
return StatusResponse(Status.err(form.errors))
except FilesystemModel.DoesNotExist as err:
return StatusResponse(FilesystemNotExistError(err, filesystem_id))
else:
return HttpResponseForbidden()
def test_receive_get_log_program_not_exists(self):
error_status = Status.ok({
'method': 'get_log',
'result': {
'log': '',
'uuid': '0'
},
})
# connect webinterface
webinterface = WSClient()
webinterface.join_group('notifications')
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.receive',
path='/commands',
content={'text': error_status.to_json()},
)
# test if the webinterface gets the error message
self.assertEqual(
Status.err('Received log from unknown program!'),
Status.from_json(json.dumps(webinterface.receive())),
)
def test_chain_command_success(self):
if os.name == 'nt':
prog = "C:\\Windows\\System32\\cmd.exe"
args = ["/c", "ECHO %date%"]
else:
prog = "/bin/sh"
args = ["-c", "echo $(date)"]
result = self.loop.run_until_complete(
client.command.chain_execution(commands=[{
'method': 'execute',
'uuid': 'thisisunique',
'arguments': {
'pid': random.choice(string.digits),
'own_uuid': uuid4().hex,
'path': prog,
'arguments': args
},
}]))
response = Status(
Status.ID_OK,
{
'method': 'execute',
'result': '0',
},
'thisisunique',
)
self.assertEqual(Status(**result[0]), response)
def test_receive_filesystem_moved_success(self):
filesystem = FileFactory()
moved = MovedFileFactory.build()
error_status = Status.ok({
'method': 'filesystem_move',
'result': moved.hash_value,
})
error_status.uuid = filesystem.command_uuid
# connect webinterface
webinterface = WSClient()
webinterface.join_group('notifications')
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.receive',
path='/commands',
content={'text': error_status.to_json()},
)
query = FilesystemModel.objects.get(id=filesystem.id)
self.assertEqual(query.hash_value, moved.hash_value)
self.assertEqual(query.error_code, "")
self.assertEqual(
Status.ok({
'filesystem_status': 'moved',
'fid': str(filesystem.id),
}),
Status.from_json(json.dumps(webinterface.receive())),
)
def test_receive_filesystem_restore_with_error_code(self):
filesystem = MovedFileFactory()
error_code = 'any kind of string'
error_status = Status.err({
'method': 'filesystem_restore',
'result': error_code,
})
error_status.uuid = filesystem.command_uuid
# connect webinterface
webinterface = WSClient()
webinterface.join_group('notifications')
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.receive',
path='/commands',
content={'text': error_status.to_json()},
)
query = FilesystemModel.objects.get(id=filesystem.id)
self.assertEqual(query.error_code, error_code)
self.assertEqual(
Status.ok({
'filesystem_status': 'error',
'fid': str(filesystem.id),
'error_code': error_code,
}),
Status.from_json(json.dumps(webinterface.receive())),
)
def test_receive_online_success(self):
slave = SlaveOnlineFactory(online=False)
expected_status = Status.ok({'method': 'online'})
expected_status.uuid = slave.command_uuid
# connect webinterface on /notifications
webinterface = WSClient()
webinterface.join_group('notifications')
# send online answer
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.receive',
path='/commands',
content={'text': expected_status.to_json()},
)
self.assertTrue(SlaveModel.objects.get(id=slave.id).is_online)
# test if a connected message was send on /notifications
self.assertEqual(
Status.ok({
'slave_status': 'connected',
'sid': str(slave.id),
}),
Status.from_json(json.dumps(webinterface.receive())),
)
def test_receive_execute_success(self):
program_status = ProgramStatusFactory(running=True)
program = program_status.program
expected_status = Status.ok({'method': 'execute', 'result': 0})
expected_status.uuid = program_status.command_uuid
# connect webinterface
webinterface = WSClient()
webinterface.send_and_consume(
'websocket.connect',
path='/notifications',
)
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.receive',
path='/commands',
content={'text': expected_status.to_json()},
)
query = ProgramStatusModel.objects.filter(program=program, code=0)
self.assertTrue(query.count() == 1)
self.assertFalse(query.first().running)
# test if the webinterface gets the "finished" message
self.assertEqual(
Status.ok({
'program_status': 'finished',
'pid': str(program.id),
'code': 0,
}),
Status.from_json(json.dumps(webinterface.receive())),
)
def test_receive_online_with_error_status(self):
slave = SlaveOnlineFactory(online=False)
error_status = Status.err({
'method': 'online',
'result': str(Exception('foobar'))
})
error_status.uuid = slave.command_uuid
# connect webinterface on /notifications
webinterface = WSClient()
webinterface.join_group('notifications')
# send online answer
ws_client = WSClient()
ws_client.send_and_consume('websocket.receive',
path='/commands',
content={'text': error_status.to_json()})
self.assertFalse(SlaveModel.objects.get(id=slave.id).is_online)
# test if a connected message was send on /notifications
self.assertEqual(
Status.err(
'An error occurred while connecting to client {}!'.format(
slave.name)),
Status.from_json(json.dumps(webinterface.receive())),
)
class BilibiliBlock():
"""
Bilibili文件块类
Properties:
index -- block序号
path -- name_type.m4s.index
block_size -- block大小，默认为10MB
"""
def __init__(self, index, url, qn, path, spider, block_size=10*1024*1024):
self.index = index
self.url = url
self.qn = qn
self.path = "{}.{:03}".format(path, index)
self.spider = spider
self.block_size = 10*1024*1024
self.name = os.path.split(self.path)[-1]
self.status = Status()
def download(self):
stream = True
chunk_size = 1024
# 更改状态
self.status.switch(Status.DOWNLOADING)
if not os.path.exists(self.path):
# 设置 headers
headers = dict(self.spider.headers)
headers["Range"] = "bytes={}-{}".format(
self.index * self.block_size,
(self.index + 1) * self.block_size - 1)
# 尝试建立连接
connected = False
while not connected:
try:
res = self.spider.get(
self.url, stream=stream, headers=headers)
connected = True
except:
print("[Error] 下载出错，正在重新尝试...")
# 写入文件
with open(self.path, 'wb+') as f:
if stream == True:
for chunk in res.iter_content(chunk_size=chunk_size):
if not chunk:
break
f.write(chunk)
else:
f.write(res.content)
self.status.switch(Status.DONE)
def remove(self):
""" 删除文件 """
if os.path.exists(self.path):
os.remove(self.path)
def __init__(self, index, url, qn, path, spider, block_size=10*1024*1024):
self.index = index
self.url = url
self.qn = qn
self.path = "{}.{:03}".format(path, index)
self.spider = spider
self.block_size = 10*1024*1024
self.name = os.path.split(self.path)[-1]
self.status = Status()
def test_chain_command_one_failed(self):
if os.name == 'nt':
prog = "C:\\Windows\\System32\\cmd.exe"
args = ["/c", "ECHO %date%"]
else:
prog = "/bin/sh"
args = ["-c", "echo $(date)"]
result = self.loop.run_until_complete(
client.command.chain_execution(commands=[{
'method': 'execute',
'uuid': 0,
'arguments': {
'pid': 1,
'own_uuid': 0,
'path': prog,
},
}, {
'method': 'execute',
'uuid': 1,
'arguments': {
'pid': 1,
'own_uuid': 1,
'path': prog,
'arguments': args
},
}]))
response1 = Status(
Status.ID_ERR,
{
'method':
'execute',
'result':
"execute() missing 1 required positional argument: 'arguments'",
},
'uniqueidforfirst',
)
response2 = Status(
Status.ID_ERR,
{
'method':
'execute',
'result':
'Could not execute because earlier command was not successful.',
},
'uniqueidforsecond',
)
self.assertEqual(Status(**result[0]), response1)
self.assertEqual(Status(**result[1]), response2)
def test_rpc_method_raise_exception(self): # pylint: disable=R0201
"""
Tests what happens if a rpc method raises an exception
"""
@Rpc.method
@asyncio.coroutine
def raises_async(): # pylint: disable=R0201,W0612
"""
Simple async rpc function, that raises an Exception.
"""
raise Exception('foo')
cmd = Command('raises_async')
status = Status.err({
'method': 'raises_async',
'result': str(Exception('foo'))
})
status.uuid = cmd.uuid
Server(
[
cmd.to_json(),
],
[
status.to_json(),
],
).run()
def execute_call(cmd):
"""
Handles an incoming message in a seperat task
"""
callable_command = Rpc.get(cmd.method)
logging.debug("Found correct function ... calling.")
try:
result = yield from asyncio.coroutine(callable_command)(
**cmd.arguments)
status_code = Status.ID_OK
logging.debug(
'method %s with args: %s returned %s.',
cmd.method,
cmd.arguments,
result,
)
except Exception as err: # pylint: disable=W0703
result = str(err)
status_code = Status.ID_ERR
logging.info('Function raise Exception(%s)', result)
return Status(status_code, {
'method': cmd.method,
'result': result
}, cmd.uuid)
def test_receive_execute_slave_not_exists(self):
program_status = ProgramStatusFactory(running=True)
program = program_status.program
expected_status = Status.ok({'method': 'execute', 'result': 0})
expected_status.uuid = program_status.command_uuid
# connect webinterface
webinterface = WSClient()
webinterface.send_and_consume(
'websocket.connect',
path='/notifications',
)
program.delete()
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.receive',
path='/commands',
content={'text': expected_status.to_json()},
)
self.assertFalse(
ProgramStatusModel.objects.filter(program=program).exists())
# test if the webinterface gets the "finished" message
self.assertIsNone(webinterface.receive())
def test_rpc_method_cancel_method(self): # pylint: disable=R0201
"""
Tests what happens if you cancel a running method
"""
@Rpc.method
@asyncio.coroutine
def sleep(sec): # pylint: disable=R0201,W0612
"""
Simple async rpc function, that raises an Exception.
"""
try:
yield from asyncio.sleep(sec)
except asyncio.CancelledError:
return -15
sleep_cmd = Command(method='sleep', sec=1)
cancel_cmd = Command(method='sleep')
cancel_cmd.uuid = sleep_cmd.uuid
status = Status.ok({'method': 'sleep', 'result': -15})
status.uuid = sleep_cmd.uuid
Server(
[
sleep_cmd.to_json(),
cancel_cmd.to_json(),
],
[
status.to_json(),
],
).run()
def ws_rpc_receive(message):
"""
Handles incoming requests on the websocket `/commands`. The incoming
message will be parsed to a `Status` object. The appropriate handler is
called with `select_method`.
Parameters
----------
message: channels.message.Message
Contains a message which needs to be JSON encoded string.
"""
try:
try:
status = Status.from_json(message.content['text'])
except ValueError as err:
LOGGER.error(
"Error while parsing json. (cause: %s)",
str(err),
)
return
select_method(status)
except FormatError as err:
LOGGER.error(
"Could not parse Status from incoming request. (cause: %s)",
str(err),
)
except KeyError:
LOGGER.error("No content['text'] in received message.")
def test_math_add(self): # pylint: disable=R0201
"""
Testing simple math add function with async features.
"""
@Rpc.method
def math_add(integer1, integer2): # pylint: disable=R0201,W0612
"""
Simple add function with async.
Arguments
---------
integer1: first operand
integer2: second operand
"""
res = (integer1 + integer2)
return res
cmd = Command("math_add", integer1=1, integer2=2)
status = Status.ok({'method': 'math_add', 'result': 3})
status.uuid = cmd.uuid
Server(
[
cmd.to_json(),
],
[
status.to_json(),
],
).run()
def filesystem_restore(request, filesystem_id):
"""
Processes an method invocation (restore) for an `FilesystemModel`. (see
@frontend.controller.filesystem_restore)
HTTP Methods
------------
POST:
Invokes the method for the `FilesystemModel` (which is
specified in the URL).
Parameters
----------
request: HttpRequest
The request which should be processed.
Returns
-------
HttpResponse:
If the HTTP method is not supported, then an
`HttpResponseForbidden` is returned.
"""
if request.method == 'POST':
try:
filesystem = FilesystemModel.objects.get(id=filesystem_id)
try:
fs_restore(filesystem)
return StatusResponse(Status.ok(""))
except FsimError as err:
return StatusResponse(err)
except FilesystemModel.DoesNotExist as err:
return StatusResponse(FilesystemNotExistError(err, filesystem_id))
else:
return HttpResponseForbidden()
def test_disconnect_success(self):
slave = SlaveOnlineFactory()
program_status = ProgramStatusFactory(program__slave=slave)
program = program_status.program
# connect client on /commands
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.connect',
path='/commands',
content={'client': [slave.ip_address, slave.mac_address]})
# connect webinterface on /notifications
webinterface = WSClient()
webinterface.send_and_consume(
'websocket.connect',
path='/notifications',
)
# throw away connect repsonse
ws_client.receive()
ws_client.send_and_consume('websocket.disconnect', path='/commands')
# test if SlaveStatus was to offline
self.assertFalse(SlaveModel.objects.get(id=slave.id).is_online)
Group('client_{}'.format(slave.id)).send(
{'text': 'ok'},
immediately=True,
)
self.assertIsNone(ws_client.receive())
# test if program status was removed
self.assertFalse(
ProgramStatusModel.objects.filter(program=program).exists())
# test if a "disconnected" message has been send to the webinterface
self.assertEqual(
Status.ok({
'slave_status': 'disconnected',
'sid': str(slave.id)
}), Status.from_json(json.dumps(webinterface.receive())))
def test_receive_unknown_message_type(self):
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.receive',
path='/commands',
content={'text': Status.ok({
'method': ''
}).to_json()},
)
self.assertIsNone(ws_client.receive())
def notify_err(message):
"""
Sending the given `message` to the notification channel. Indicating that
the message is an error message.
Parameters
----------
message: JSON object
This message is send to the web interface with an Status.err()
wrapped around.
"""
Group('notifications').send({'text': Status.err(message).to_json()})
def test_state_next_offline_filesystem(self):
webinterface = WSClient()
webinterface.join_group('notifications')
self.prog1.delete()
self.sched._Scheduler__index = -1
self.sched._Scheduler__event = asyncio.Event(loop=self.sched.loop)
self.sched._Scheduler__script = self.script.id
self.sched._Scheduler__state = SchedulerStatus.NEXT_STEP
self.sched._Scheduler__state_next()
self.assertEqual(
self.sched._Scheduler__state,
SchedulerStatus.ERROR,
)
self.assertStatusRegex(
Status.err(SlaveOfflineError),
Status.err(self.sched._Scheduler__error_code),
)
def test_state_success(self):
webinterface = WSClient()
webinterface.join_group('notifications')
self.sched._Scheduler__event = asyncio.Event(loop=self.sched.loop)
self.sched._Scheduler__script = self.script.id
self.sched._Scheduler__state = SchedulerStatus.SUCCESS
self.sched._Scheduler__state_success()
self.assertEqual(
self.sched._Scheduler__state,
SchedulerStatus.SUCCESS,
)
self.assertEqual(
Status.ok({
'script_status': 'success',
'script_id': self.script.id,
}),
Status.from_json(json.dumps(webinterface.receive())),
)
def websocket_handler(websocket, path):
self.assertEqual('/logs', path)
# receive log from file
json = yield from websocket.recv()
log = Status.from_json(json).payload['log'].encode()
# ack
yield from websocket.send('')
#receive dynamic log
while True:
json = yield from websocket.recv()
# ack
msg = Status.from_json(json).payload['log'].encode()
log += msg
if msg == b'':
break
else:
yield from websocket.send('')
self.assertIn(expected_log, log)
print('finished server')
finished.set_result(None)
def __init__(self, page, name, path, cid, spider):
self.page = page
self.name = name
self.path = path
self.cid = cid
self.spider = spider
self.video = None
self.audio = None
self.total = 0
self.status = Status()
def test_connect_and_disconnect_success(self):
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.connect',
path='/notifications',
)
# test if ws_client is part of 'notifications'
Group('notifications').send({'text': Status.ok('').to_json()})
self.assertEqual(
Status.ok(''),
Status.from_json(json.dumps(ws_client.receive())),
)
ws_client.send_and_consume(
'websocket.disconnect',
path='/notifications',
)
# test if ws_client was removed from 'notifications'
Group('notifications').send({'text': Status.ok('').to_json()})
self.assertIsNone(ws_client.receive())
def test_receive_get_log_with_error_status(self):
error_status = Status.err({
'method': 'get_log',
'result': str(Exception('foobar')),
})
# connect webinterface
webinterface = WSClient()
webinterface.join_group('notifications')
ws_client = WSClient()
ws_client.send_and_consume(
'websocket.receive',
path='/commands',
content={'text': error_status.to_json()},
)
# test if the webinterface gets the error message
self.assertEqual(
Status.err('An error occured while reading a log file!'),
Status.from_json(json.dumps(webinterface.receive())),
)
def enable_remote(self):
"""
Enables remote logging to the websocket located on 'self.__url'. First
the existing log gets send, then updates follow on every request
(an empty message)by the receiver.
"""
with self.__lock:
self.__ws_connection = yield from websockets.connect(self.__url)
log = self.__log_file.read()
msg = {'log': log.decode(), 'pid': self.__pid_on_master}
yield from self.__ws_connection.send(Status.ok(msg).to_json())
yield from self.__ws_connection.recv()
while True:
yield from self.__ws_buffer_has_content.wait()
with self.__lock:
msg = {
'log': self.__ws_buffer.decode(),
'pid': self.__pid_on_master
}
self.__ws_buffer = b''
self.__ws_buffer_has_content.clear()
try:
yield from self.__ws_connection.send(Status.ok(msg).to_json())
yield from self.__ws_connection.recv()
except websockets.exceptions.ConnectionClosed:
break
if msg['log'] == b''.decode():
break
# if the stream from tee has finished the buffer gets send and
# cleared in the next iterations which terminates the loop
if self.__ws_finished:
self.__ws_buffer_has_content.set()
