def test_resume_exception():
"""
/SA920/
"""
try:
user = User.get_user_by_username("resumeuser")
except UserNotFoundException:
user = User("resumeuser", "token")
job = Job('storageA', 'storageB', '~/.data/', [True, False, True], user)
job.set_status(JobStatus.ACTIVE)
job_id = job.get_job_id()
request = {
"job_id": job_id,
}
json_string = json.dumps(request)
with pytest.raises(SemanticException):
app.resume(json_string)
"""
/SA930/
"""
request = {
"job_id":
"87497294723947289374823[7936593653456348756t3487657834658723452345",
}
json_string = json.dumps(request)
with pytest.raises(JobNotFoundException):
app.resume(json_string)
def __init__(self, token=None, active=True, mailbox=None):
"""
The Constructor. Warning: can break singleton if not used carefully
"""
if token is not None:
self._token = token
if active is not None:
self._is_active = active
self._master_communicator = WorkerCommunicator(self._token)
workers = Worker.get_all_workers()
if Job.get_job_highest_priority() is not None:
self._max_priority = Job.get_job_highest_priority().get_priority()
else:
self._max_priority = 0
for worker in workers:
status = self._master_communicator.get_status(worker)
if 'Exception' in status:
worker.delete()
else:
worker.set_status(status['status'])
if worker.get_status(
) == WorkerStatus.WAITING and self._is_active:
job = self.look_for_work(worker)
if job is not None:
self.dequeue(job, worker)
def create_job(self, user, for_user, workspace, target, email):
"""
:return: ID of a new Job
:raises UserNotFoundException: if 'for_user' or 'user' doesn't exist in database
:raises SearchException: Search raises an exception. See exception's message for details
:raises StorageNotAcceptingException: if one of the referenced Storages doesn't accept jobs currently
:raises NoAdministratorRightsException: if 'user' can not create jobs for 'for_user'
:raises PermissionException: if 'for_user' reached his max. amount of jobs
:raises MasterPausedException: if Master is paused
"""
caller = User.get_user_by_username(user)
for_user = User.get_user_by_username(for_user) if for_user else caller
target_storj = Storage.get_storage_by_name(target)
src_workspace = Workspaces.get_by_full_name(workspace)
if src_workspace.username != caller.name and caller.get_user_type() != UserRole.Administrator:
raise PermissionError('Unauthorized to move workspaces of other users.')
src_storj = Storage.get_storage_by_name(src_workspace.storage)
source = src_workspace.full_path
addr = search_class.verify_job(source, target)
target_path = target_storj.mountpoint + addr['source_relative_path']
copy_options = determine_copytool(src_storj, target_storj)
job = Job(addr['source_alias'], target, source, email, for_user, target_path=target_path, copy_options=json.dumps(copy_options))
self.access_handler.check_create_job(job, caller)
self.master.add_to_queue(job)
return job.get_job_id()
def test_done_job():
try:
user = User.get_user_by_username("uyefv")
except UserNotFoundException:
user = User("uyefv", "token")
job = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job.set_status(JobStatus.DONE)
notification_handler.resolve_notification(job)
def test_create_job_config_exception():
"""Test for max. allowed number of active jobs per user"""
"""
/SA040/
"""
try:
user = User.get_user_by_username("createjobuser2")
except UserNotFoundException:
user = User("createjobuser2", "token")
request = {
"source": "source",
"target": "target",
"user": "******",
"for_user": "******",
"email": [True, True, True]
}
json_string = json.dumps(request)
try:
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
job.set_status(JobStatus.QUEUED)
except PermissionException:
pass
try:
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
job.set_status(JobStatus.QUEUED)
except PermissionException:
pass
try:
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
job.set_status(JobStatus.QUEUED)
except PermissionException:
pass
try:
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
job.set_status(JobStatus.QUEUED)
except PermissionException:
pass
try:
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
job.set_status(JobStatus.QUEUED)
assert 1 == 0
except PermissionException:
assert 1 == 1
def set_priority(self, job, new_priority):
"""
Sets a new priority for the job
:param job: the job to update
:param new_priority: the new priority
:type job: Job
:type new_priority: integer
"""
if Job.get_job_by_priority(new_priority):
Job.get_job_by_priority(new_priority).set_priority(new_priority +
1)
job.set_priority(new_priority)
def job_changed_status(
self,
status,
):
# TODO getting job objects twice, because they expire after setting the status
# how to do this better? maybe: set expire_on_commit=False
job = Job.get_job_by_id(status['id'])
if status['status'] in [
'INITIALIZED', 'DELETED', 'COPYING', 'COPIED', 'CHECKED'
]:
job.set_status(JobStatus.ACTIVE)
job = Job.get_job_by_id(status['id'])
Worker.get_worker_by_name(
status['worker_name']).set_active_job(job)
elif status['status'] in ['PAUSED', 'EXCEPTION', 'ERROR']:
job.set_status(JobStatus.PAUSED)
job = Job.get_job_by_id(status['id'])
Worker.get_worker_by_name(
status['worker_name']).set_active_job(job)
elif status['status'] in ['TERMINATED']:
job.set_status(JobStatus.CANCELED)
job = Job.get_job_by_id(status['id'])
Worker.get_worker_by_name(
status['worker_name']).set_active_job(None)
else:
job.set_status(JobStatus.DONE)
w = Workspaces.get_by_path(job.source_path)
if not w:
raise AttributeError(
'No Workspace entry was found. Location couldn\'t be updated.'
)
w.set_full_path(job.target_path)
w.set_storage(job.target_alias)
job = Job.get_job_by_id(status['id'])
Worker.get_worker_by_name(
status['worker_name']).set_active_job(None)
if 'start_time' in status and status['start_time'] is not None:
job.set_start_time(dateutil.parser.parse(status['start_time']))
if 'completion_time' in status and status[
'completion_time'] is not None:
job.set_end_time(dateutil.parser.parse(status['completion_time']))
if 'exceptions' in status and status['exceptions'] is not None:
for e in status['exceptions']:
job.set_error(e)
if 'errors' in status and status['errors'] is not None:
for err in status['errors']:
job.set_error(err)
notification_handler.resolve_notification(job)
def test_create_job_admin():
"""
/SA020/
"""
try:
user = User.get_user_by_username("createjobuser4")
except UserNotFoundException:
user = User("createjobuser4", "token")
try:
admin = User.get_user_by_username("admin1")
except UserNotFoundException:
admin = Administrator("admin1", "token")
request = {
"source": "source",
"target": "target",
"user": "******",
"for_user": "******",
"email": [True, True, False]
}
json_string = json.dumps(request)
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
assert job.get_job_id() == job_id
assert job.has_start_notification()
assert not job.has_end_notification()
assert job.has_error_notification()
assert job.get_target_alias() == "target"
assert job.get_user().get_username() == "createjobuser4"
def get_workers(self, ):
workers = []
worker1 = Worker('worker1', 'WAITING', None, '192.168.0.1:1234',
['storageA', 'storageB'])
workers.append(worker1)
try:
user = User.get_user_by_username("workeruser1")
except UserNotFoundException:
user = User("workeruser1", "token")
job = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
worker2 = Worker('worker2', 'ACTIVE', job.get_job_id(),
'192.168.0.2:1234', None)
workers.append(worker2)
return workers
def test_cancel_job():
try:
johann = User.get_user_by_username('johann')
except UserNotFoundException:
johann = User('johann', 'jtoken')
job = Job('dmd_home', 'dmd1', 'home/centos/testdata/johann-directory-1', [False, False, False], johann)
job_id = job.id
master = Master.get_master()
master.add_to_queue(job)
master.cancel_job(job)
worker = master.get_workers()[0]
assert worker.name == 'worker1'
assert worker.status.name == 'WAITING'
assert Job.get_job_by_id(job_id).status.name == 'CANCELED'
def resume(self, job_id):
"""
Routes resume request to ApplicationFacade
"""
job = Job.get_job_by_id(job_id)
self.access_handler.check_resume(job)
self.master.resume_job(job)
def add_job_stats(self, job_id, stats):
job = Job.get_job_by_id(job_id)
job.set_n_of_files(stats.get('n_of_files'))
job.set_n_of_dirs(stats.get('n_of_dirs'))
job.set_workpool_size(stats.get('total_size'))
job.set_compression_rate(stats.get('compression_rate'))
return
def check_create_job(self, job, calling_user):
"""
Checks permissions for job creation. Checks:
1) If calling_user can create job for for_user
2) If user did not exceed the allowed amount of jobs
:param job: job to be checked
"""
if job.get_user().get_username() != calling_user.get_username():
if calling_user.get_user_type() != UserRole.Administrator:
raise NoAdministratorRightsException(
calling_user.get_username() + " doesn't have "
"rights to create job for " +
job.get_user().get_username())
allowed_jobs_number = self._config_handler.get_max_tasks_number_for_user(
)
if len(Job.get_jobs([job.get_user().get_username()],
[JobStatus.ACTIVE, JobStatus.QUEUED, JobStatus.PAUSED])) \
> allowed_jobs_number:
raise PermissionException("Max. number of jobs (" +
str(allowed_jobs_number) +
") for user " +
job.get_user().get_username() +
" exceeded")
return
def get_current_job(self):
"""
Returns current job
:return: Job
"""
if self.current_job_id is not None:
return Job.get_job_by_id(self.current_job_id)
return None
def test_resume():
"""
/SA910/
"""
try:
user = User.get_user_by_username("resumeuser")
except UserNotFoundException:
user = User("resumeuser", "token")
job = Job('storageA', 'storageB', '~/.data/', [True, False, True], user)
job.set_status(JobStatus.PAUSED)
job_id = job.get_job_id()
request = {
"job_id": job_id,
}
json_string = json.dumps(request)
app.resume(json_string)
def get_jobs(self, days, user=None, for_user=None, statuses=None):
"""
:raises UserNotFoundException: if given 'user' or 'for_user' doesn't exist
:raises NoProjectManagerRightsException: @see access_handler
"""
calling_user = User.get_user_by_username(user) if user else None
team = True if for_user == "team" else False
for_user = None if for_user in [None, "team"] else User.get_user_by_username(for_user)
if team:
self.access_handler.check_read_rights(None, calling_user)
jobs = Job.get_jobs(calling_user.get_team_members(), statuses, days)
elif for_user is not None and calling_user is not None:
self.access_handler.check_read_rights(for_user, calling_user)
jobs = Job.get_jobs([for_user.get_username()], statuses, days)
else:
jobs = Job.get_jobs(None, statuses)
return build_log(jobs)
def set_priority(self, job_id, priority):
"""
Routes set_priority request to ApplicationFacade
:raises JobNotFoundException: job with 'job_id' doesn't exist
:raises PermissionException: job is not in the queue
:raises MasterPausedException: if master is paused
"""
job = Job.get_job_by_id(job_id)
self.access_handler.check_set_priority(job)
self.master.set_priority(job, priority)
def cancel_job(self, job_id, user):
"""
:raises JobNotFoundException: if there is no job with 'job_id'
:raises UserNotFoundException: if 'user' doesn't exist
:raises PermissionException: if 'user' is not permitted to cancel given job
:raises SemanticException: if 'job' is already cancelled or done
"""
job = Job.get_job_by_id(job_id)
calling_user = User.get_user_by_username(user)
self.access_handler.check_cancel_job(job, calling_user)
self.master.cancel_job(job)
def get_job(self, user, job_id):
"""
Routes get_job request to ApplicationFacade
:raises UserNotFoundException: if given 'user' doesn't exist
:raises NoProjectManagerRightsException: @see access_handler
"""
calling_user = User.get_user_by_username(user)
job = Job.get_job_by_id(job_id)
for_user = job.get_user()
self.access_handler.check_read_rights(for_user, calling_user)
return build_job(job)
def test_success_job(env):
try:
johann = User.get_user_by_username('johann')
except UserNotFoundException:
johann = User('johann', 'jtoken')
job = Job(source_alias = 'dmd_home', target_alias = 'archive1',
source_relative_path = 'export/master_data/tests/johann-directory-1',
notification = [False, False, False], user = johann)
job_id = job.id
master = Master.get_master()
worker = master.get_workers()[0]
assert worker.name == 'worker1'
assert worker.status.name == 'WAITING'
master.add_to_queue(job)
time.sleep(5)
while (worker.status.name == 'ACTIVE'):
#wait for worker to finish
time.sleep(5)
assert worker.status.name == 'WAITING'
assert Job.get_job_by_id(job_id).status.name == 'DONE'
#make sure the file was actually copied
assert os.path.exists("/export/archive1/1-johann-directory.tar")
def test_cancel_job_semantic_exception():
"""
/SA140/
"""
try:
user = User.get_user_by_username("canceljobuser4")
except UserNotFoundException:
user = User("canceljobuser4", "token")
try:
admin = User.get_user_by_username("canceljobadmin4")
except UserNotFoundException:
admin = Administrator("canceljobadmin4", "token")
job = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job_id = job.get_job_id()
job.set_status(JobStatus.DONE)
request = {"job_id": job_id, "user": "******"}
json_string = json.dumps(request)
with pytest.raises(SemanticException):
app.cancel_job(json_string)
def look_for_work(self, worker):
"""
Looks wether a job is available for the given worker
:param worker: The worker to assign a job to
:type worker: Worker
:returns: a job
"""
storages = list(storage.alias for storage in worker.get_storages())
jobs = Job.get_jobs(statuses=[JobStatus.QUEUED], storages=storages)
if not jobs:
return None
else:
return jobs[0]
def test_cancel_job_wrong_administrator():
"""
/SA130/
"""
try:
user = User.get_user_by_username("canceljobuser3")
except UserNotFoundException:
user = User("canceljobuser3", "token")
try:
admin = User.get_user_by_username("canceljobfalseadmin33")
except UserNotFoundException:
admin = User("canceljobfalseadmin33", "token")
job = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job_id = job.get_job_id()
job.set_status(JobStatus.ACTIVE)
request = {"job_id": job_id, "user": "******"}
json_string = json.dumps(request)
with pytest.raises(PermissionException):
app.cancel_job(json_string)
def test_remove():
job = Job('mystorage', 'storageB', '~/.data/', [True, True, True], user)
job_id = job.id
job_get = Job.get_job_by_id(job_id)
assert job_get is not None
job.remove()
try:
Job.get_job_by_id(job_id)
except JobNotFoundException:
assert 1 == 1
return
assert 0 == 0
def test_error_job():
try:
user = User.get_user_by_username("uyefv")
except UserNotFoundException:
user = User("uyefv", "token")
job = Job('storageA', 'storageB', '~/.data/', [False, False, False], user)
job.set_error("Target not mounted")
job.set_status(JobStatus.PAUSED)
notification_handler.resolve_notification(job)
def test_set_priority():
"""
/SA1010/
"""
try:
user = User.get_user_by_username("priorityuser1")
except UserNotFoundException:
user = Administrator("priorityuser1", "token")
job = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job_id = job.get_job_id()
job.set_status(JobStatus.QUEUED)
request = {"job_id": job_id, "priority": 3}
json_string = json.dumps(request)
app.set_priority(json_string)
job = Job.get_job_by_id(job_id)
assert job.get_priority() == 3
def test_cancel_job():
"""
/SA110/
"""
try:
user = User.get_user_by_username("canceljobuser1")
except UserNotFoundException:
user = User("canceljobuser1", "token")
job = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job_id = job.get_job_id()
job.set_status(JobStatus.ACTIVE)
request = {"job_id": job_id, "user": "******"}
json_string = json.dumps(request)
app.cancel_job(json_string)
assert Job.get_job_by_id(job_id).get_status() == JobStatus.CANCELED
def test_get_log_different_days():
"""
/SA240/
"""
try:
user1 = User.get_user_by_username("loguser2")
jobs = Job.get_jobs([user1.get_username()], [
JobStatus.PAUSED, JobStatus.QUEUED, JobStatus.ACTIVE,
JobStatus.DONE
])
for current_job in jobs:
current_job.set_status(JobStatus.ACTIVE)
except UserNotFoundException:
user1 = User("loguser2", "token")
timestamp = datetime.date.today() - datetime.timedelta(days=2)
active_job_ids_user1 = []
job1 = Job('storageA', 'storageB', '~/.data/', [True, True, True], user1)
job1.set_status(JobStatus.DONE)
job1.set_start_time(timestamp)
job1.set_end_time(timestamp)
job1.set_enqueue_time(timestamp)
active_job_ids_user1.append(job1.get_job_id())
job2 = Job('storageA', 'storageB', '~/.data/', [True, True, True], user1)
job2.set_status(JobStatus.DONE)
job2.set_start_time(timestamp)
job2.set_end_time(timestamp)
job2.set_enqueue_time(timestamp)
active_job_ids_user1.append(job2.get_job_id())
job3 = Job('storageA', 'storageB', '~/.data/', [True, True, True], user1)
job3.set_status(JobStatus.DONE)
job3.set_start_time(timestamp)
job3.set_end_time(timestamp)
job3.set_enqueue_time(timestamp)
active_job_ids_user1.append(job3.get_job_id())
# Log for last 3 days
request = {"days": "3", "user": "******", "for_user": "******"}
json_string = json.dumps(request)
response = json.loads(app.get_log(json_string))
assert len(response) == 3
for job in response:
assert job['job_id'] in active_job_ids_user1
# Log for last day
request = {"days": "1", "user": "******", "for_user": "******"}
json_string = json.dumps(request)
response = json.loads(app.get_log(json_string))
assert len(response) == 0
def test_get_queue():
"""
/SA310/
"""
jobs = Job.get_jobs(None, [JobStatus.QUEUED])
for current_job in jobs:
current_job.set_status(JobStatus.DONE)
assert len(Job.get_jobs(None, [JobStatus.QUEUED])) == 0
timestamp = datetime.datetime.now()
try:
admin = User.get_user_by_username("queueadmin1")
except UserNotFoundException:
admin = Administrator("queueadmin1", "token")
job1 = Job('storageA', 'storageB', '~/.data/', [True, True, True], admin)
job1_id = job1.get_job_id()
job1.set_status(JobStatus.QUEUED)
job1.set_enqueue_time(timestamp)
job1.set_priority(1)
try:
user = User.get_user_by_username("queueuser1")
except UserNotFoundException:
user = Administrator("queueuser1", "token")
job2 = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job2_id = job2.get_job_id()
job2.set_status(JobStatus.QUEUED)
job2.set_enqueue_time(timestamp)
job2.set_priority(2)
job3 = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job3_id = job3.get_job_id()
job3.set_status(JobStatus.QUEUED)
job3.set_enqueue_time(timestamp)
job3.set_priority(3)
response = json.loads(app.get_queue())
assert len(response) == 3
for job_json in response:
job = Job.get_job_by_id(job_json['job_id'])
assert job_json['job_id'] == job1_id or job_json[
'job_id'] == job2_id or job_json['job_id'] == job3_id
assert job_json['job_id'] == job.get_job_id()
assert job_json['source'] == job.get_source_alias(
) + ":" + job.get_source_relative_path()
assert job_json['enqueue_time'] == job.get_enqueue_time().isoformat()
assert job_json['creator'] == job.get_user().get_username()
assert job_json['priority'] == job.get_priority()
assert job_json['status'] == job.get_status().name
import pytest
from exceptions.user_not_found_exception import UserNotFoundException
from master.master import Master
from master.worker import Worker
from application.system.job import Job
from application.system.user import User
"""
This test assumes an empty database
"""
try:
user = User.get_user_by_username("xyz")
except UserNotFoundException:
user = User("xyz", "token")
jobA = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
jobB = Job('storageB', 'storageC', '~/.data/', [True, True, True], user)
jobC = Job('storageC', 'storageD', '~/.data/', [True, True, True], user)
workerA = Worker('A', 'WAITING', None, 'host:1234', ['storageA', 'storageB'])
workerB = Worker('B', 'WAITING', None, 'host:1234', ['storageA', 'storageB'])
@pytest.mark.order1
def test_constructor():
master = Master.get_master()
assert (type(master) is Master)
master2 = Master.get_master()
assert (master == master2)
