def test_create_job_config_exception():
"""Test for max. allowed number of active jobs per user"""
"""
/SA040/
"""
try:
user = User.get_user_by_username("createjobuser2")
except UserNotFoundException:
user = User("createjobuser2", "token")
request = {
"source": "source",
"target": "target",
"user": "******",
"for_user": "******",
"email": [True, True, True]
}
json_string = json.dumps(request)
try:
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
job.set_status(JobStatus.QUEUED)
except PermissionException:
pass
try:
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
job.set_status(JobStatus.QUEUED)
except PermissionException:
pass
try:
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
job.set_status(JobStatus.QUEUED)
except PermissionException:
pass
try:
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
job.set_status(JobStatus.QUEUED)
except PermissionException:
pass
try:
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
job.set_status(JobStatus.QUEUED)
assert 1 == 0
except PermissionException:
assert 1 == 1
def test_remove():
job = Job('mystorage', 'storageB', '~/.data/', [True, True, True], user)
job_id = job.id
job_get = Job.get_job_by_id(job_id)
assert job_get is not None
job.remove()
try:
Job.get_job_by_id(job_id)
except JobNotFoundException:
assert 1 == 1
return
assert 0 == 0
def job_changed_status(
self,
status,
):
# TODO getting job objects twice, because they expire after setting the status
# how to do this better? maybe: set expire_on_commit=False
job = Job.get_job_by_id(status['id'])
if status['status'] in [
'INITIALIZED', 'DELETED', 'COPYING', 'COPIED', 'CHECKED'
]:
job.set_status(JobStatus.ACTIVE)
job = Job.get_job_by_id(status['id'])
Worker.get_worker_by_name(
status['worker_name']).set_active_job(job)
elif status['status'] in ['PAUSED', 'EXCEPTION', 'ERROR']:
job.set_status(JobStatus.PAUSED)
job = Job.get_job_by_id(status['id'])
Worker.get_worker_by_name(
status['worker_name']).set_active_job(job)
elif status['status'] in ['TERMINATED']:
job.set_status(JobStatus.CANCELED)
job = Job.get_job_by_id(status['id'])
Worker.get_worker_by_name(
status['worker_name']).set_active_job(None)
else:
job.set_status(JobStatus.DONE)
w = Workspaces.get_by_path(job.source_path)
if not w:
raise AttributeError(
'No Workspace entry was found. Location couldn\'t be updated.'
)
w.set_full_path(job.target_path)
w.set_storage(job.target_alias)
job = Job.get_job_by_id(status['id'])
Worker.get_worker_by_name(
status['worker_name']).set_active_job(None)
if 'start_time' in status and status['start_time'] is not None:
job.set_start_time(dateutil.parser.parse(status['start_time']))
if 'completion_time' in status and status[
'completion_time'] is not None:
job.set_end_time(dateutil.parser.parse(status['completion_time']))
if 'exceptions' in status and status['exceptions'] is not None:
for e in status['exceptions']:
job.set_error(e)
if 'errors' in status and status['errors'] is not None:
for err in status['errors']:
job.set_error(err)
notification_handler.resolve_notification(job)
def add_job_stats(self, job_id, stats):
job = Job.get_job_by_id(job_id)
job.set_n_of_files(stats.get('n_of_files'))
job.set_n_of_dirs(stats.get('n_of_dirs'))
job.set_workpool_size(stats.get('total_size'))
job.set_compression_rate(stats.get('compression_rate'))
return
def resume(self, job_id):
"""
Routes resume request to ApplicationFacade
"""
job = Job.get_job_by_id(job_id)
self.access_handler.check_resume(job)
self.master.resume_job(job)
def test_create_job_admin():
"""
/SA020/
"""
try:
user = User.get_user_by_username("createjobuser4")
except UserNotFoundException:
user = User("createjobuser4", "token")
try:
admin = User.get_user_by_username("admin1")
except UserNotFoundException:
admin = Administrator("admin1", "token")
request = {
"source": "source",
"target": "target",
"user": "******",
"for_user": "******",
"email": [True, True, False]
}
json_string = json.dumps(request)
job_id = app.create_job(json_string)
job = Job.get_job_by_id(job_id)
assert job.get_job_id() == job_id
assert job.has_start_notification()
assert not job.has_end_notification()
assert job.has_error_notification()
assert job.get_target_alias() == "target"
assert job.get_user().get_username() == "createjobuser4"
def get_current_job(self):
"""
Returns current job
:return: Job
"""
if self.current_job_id is not None:
return Job.get_job_by_id(self.current_job_id)
return None
def test_build_log():
try:
user = User.get_user_by_username("rbuser")
except UserNotFoundException:
user = User("rbuser", "token")
job1 = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job1_id = job1.get_job_id()
job1.set_status(JobStatus.DONE)
timestamp = datetime.datetime.now()
job1.set_enqueue_time(timestamp)
job1.set_end_time(timestamp)
job2 = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job2_id = job2.get_job_id()
job2.set_status(JobStatus.DONE)
timestamp = datetime.datetime.now()
job2.set_enqueue_time(timestamp)
job2.set_end_time(timestamp)
job1 = Job.get_job_by_id(job1_id)
job2 = Job.get_job_by_id(job2_id)
answer = json.loads(rb.build_log([job1, job2]))
job1 = Job.get_job_by_id(job1_id)
assert answer[0]["job_id"] == job1.get_job_id()
assert answer[0]["source"] == job1.get_source_alias(
) + ":" + job1.get_source_relative_path()
assert answer[0]["target"] == job1.get_target_alias()
assert answer[0]["enqueue_time"] == job1.get_enqueue_time().isoformat()
assert answer[0]["end_time"] == job1.get_end_time().isoformat()
assert answer[0]["creator"] == job1.get_user().get_username()
assert answer[0]["status"] == job1.get_status().name
assert answer[0]["error"] == job1.get_error()
job2 = Job.get_job_by_id(job2_id)
assert answer[1]["job_id"] == job2.get_job_id()
assert answer[1]["source"] == job2.get_source_alias(
) + ":" + job2.get_source_relative_path()
assert answer[1]["target"] == job2.get_target_alias()
assert answer[1]["enqueue_time"] == job2.get_enqueue_time().isoformat()
assert answer[1]["end_time"] == job2.get_end_time().isoformat()
assert answer[1]["creator"] == job2.get_user().get_username()
assert answer[1]["status"] == job2.get_status().name
assert answer[1]["error"] == job2.get_error()
def set_priority(self, job_id, priority):
"""
Routes set_priority request to ApplicationFacade
:raises JobNotFoundException: job with 'job_id' doesn't exist
:raises PermissionException: job is not in the queue
:raises MasterPausedException: if master is paused
"""
job = Job.get_job_by_id(job_id)
self.access_handler.check_set_priority(job)
self.master.set_priority(job, priority)
def cancel_job(self, job_id, user):
"""
:raises JobNotFoundException: if there is no job with 'job_id'
:raises UserNotFoundException: if 'user' doesn't exist
:raises PermissionException: if 'user' is not permitted to cancel given job
:raises SemanticException: if 'job' is already cancelled or done
"""
job = Job.get_job_by_id(job_id)
calling_user = User.get_user_by_username(user)
self.access_handler.check_cancel_job(job, calling_user)
self.master.cancel_job(job)
def get_job(self, user, job_id):
"""
Routes get_job request to ApplicationFacade
:raises UserNotFoundException: if given 'user' doesn't exist
:raises NoProjectManagerRightsException: @see access_handler
"""
calling_user = User.get_user_by_username(user)
job = Job.get_job_by_id(job_id)
for_user = job.get_user()
self.access_handler.check_read_rights(for_user, calling_user)
return build_job(job)
def test_get_queue():
"""
/SA310/
"""
jobs = Job.get_jobs(None, [JobStatus.QUEUED])
for current_job in jobs:
current_job.set_status(JobStatus.DONE)
assert len(Job.get_jobs(None, [JobStatus.QUEUED])) == 0
timestamp = datetime.datetime.now()
try:
admin = User.get_user_by_username("queueadmin1")
except UserNotFoundException:
admin = Administrator("queueadmin1", "token")
job1 = Job('storageA', 'storageB', '~/.data/', [True, True, True], admin)
job1_id = job1.get_job_id()
job1.set_status(JobStatus.QUEUED)
job1.set_enqueue_time(timestamp)
job1.set_priority(1)
try:
user = User.get_user_by_username("queueuser1")
except UserNotFoundException:
user = Administrator("queueuser1", "token")
job2 = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job2_id = job2.get_job_id()
job2.set_status(JobStatus.QUEUED)
job2.set_enqueue_time(timestamp)
job2.set_priority(2)
job3 = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job3_id = job3.get_job_id()
job3.set_status(JobStatus.QUEUED)
job3.set_enqueue_time(timestamp)
job3.set_priority(3)
response = json.loads(app.get_queue())
assert len(response) == 3
for job_json in response:
job = Job.get_job_by_id(job_json['job_id'])
assert job_json['job_id'] == job1_id or job_json[
'job_id'] == job2_id or job_json['job_id'] == job3_id
assert job_json['job_id'] == job.get_job_id()
assert job_json['source'] == job.get_source_alias(
) + ":" + job.get_source_relative_path()
assert job_json['enqueue_time'] == job.get_enqueue_time().isoformat()
assert job_json['creator'] == job.get_user().get_username()
assert job_json['priority'] == job.get_priority()
assert job_json['status'] == job.get_status().name
def test_cancel_job():
try:
johann = User.get_user_by_username('johann')
except UserNotFoundException:
johann = User('johann', 'jtoken')
job = Job('dmd_home', 'dmd1', 'home/centos/testdata/johann-directory-1', [False, False, False], johann)
job_id = job.id
master = Master.get_master()
master.add_to_queue(job)
master.cancel_job(job)
worker = master.get_workers()[0]
assert worker.name == 'worker1'
assert worker.status.name == 'WAITING'
assert Job.get_job_by_id(job_id).status.name == 'CANCELED'
def test_cancel_job():
"""
/SA110/
"""
try:
user = User.get_user_by_username("canceljobuser1")
except UserNotFoundException:
user = User("canceljobuser1", "token")
job = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job_id = job.get_job_id()
job.set_status(JobStatus.ACTIVE)
request = {"job_id": job_id, "user": "******"}
json_string = json.dumps(request)
app.cancel_job(json_string)
assert Job.get_job_by_id(job_id).get_status() == JobStatus.CANCELED
def test_getters():
job = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
another_job = Job.get_job_by_id(job.get_job_id())
assert another_job.get_job_id() == job.get_job_id()
assert another_job.get_source_alias() == job.get_source_alias()
assert another_job.get_target_alias() == job.get_target_alias()
assert another_job.get_source_relative_path(
) == job.get_source_relative_path()
assert another_job.get_start_time() == job.get_start_time()
assert another_job.get_end_time() == job.get_end_time()
assert another_job.get_enqueue_time() == job.get_enqueue_time()
assert another_job.get_status() == job.get_status()
assert another_job.has_start_notification() == job.has_start_notification()
assert another_job.has_end_notification() == job.has_end_notification()
assert another_job.has_error_notification() == job.has_error_notification()
assert another_job.get_user().get_username() == job.get_user(
).get_username()
def test_set_priority():
"""
/SA1010/
"""
try:
user = User.get_user_by_username("priorityuser1")
except UserNotFoundException:
user = Administrator("priorityuser1", "token")
job = Job('storageA', 'storageB', '~/.data/', [True, True, True], user)
job_id = job.get_job_id()
job.set_status(JobStatus.QUEUED)
request = {"job_id": job_id, "priority": 3}
json_string = json.dumps(request)
app.set_priority(json_string)
job = Job.get_job_by_id(job_id)
assert job.get_priority() == 3
def test_success_job(env):
try:
johann = User.get_user_by_username('johann')
except UserNotFoundException:
johann = User('johann', 'jtoken')
job = Job(source_alias = 'dmd_home', target_alias = 'archive1',
source_relative_path = 'export/master_data/tests/johann-directory-1',
notification = [False, False, False], user = johann)
job_id = job.id
master = Master.get_master()
worker = master.get_workers()[0]
assert worker.name == 'worker1'
assert worker.status.name == 'WAITING'
master.add_to_queue(job)
time.sleep(5)
while (worker.status.name == 'ACTIVE'):
#wait for worker to finish
time.sleep(5)
assert worker.status.name == 'WAITING'
assert Job.get_job_by_id(job_id).status.name == 'DONE'
#make sure the file was actually copied
assert os.path.exists("/export/archive1/1-johann-directory.tar")