def new_session(self):
if hasattr(self, 'session') and self.session:
self.session.close()
self.Session.close_all()
self.session = self.Session()
ActivityLog.attach_to(self.session)
return self.session
def setup(env):
settings = env['request'].registry.settings
env['models'] = aybu.manager.models
env['engine'] = engine_from_config(settings, 'sqlalchemy.')
env['request'].set_db_engine = env['engine']
aybu.manager.models.Base.metadata.bind = env['engine']
aybu.manager.models.Environment.initialize(settings)
env['session'] = env['request'].db_session
ActivityLog.attach_to(env['session'])
def test_render(self):
al = ActivityLog()
instance = namedtuple('Instance', ['paths', 'environment'])(
paths=namedtuple('Paths', ['pyramid_config', 'alembic_config'])(
pyramid_config='MYDUMMYCONFIG',
alembic_config='MYDUMMYCONFIG'
),
environment= namedtuple('Environment', ['settings',
'smtp_config',
'uwsgi_config',
'os_config'])(
smtp_config=None,
uwsgi_config=None,
os_config=None,
settings=None
)
)
template_name = 'main.py.mako'
target = os.path.join(self.tempdir, 'main.py')
al.add(render, template_name, target, instance=instance)
self.assertTrue(os.path.exists(target))
with open(target) as f:
self.assertIn('MYDUMMYCONFIG', f.read())
al.rollback()
self.assertFalse(os.path.exists(target))
al.add(render, template_name, target, deferred=True, instance=instance)
self.assertFalse(os.path.exists(target))
al.commit()
self.assertTrue(os.path.exists(target))
def test_error_on_exists(self):
al = ActivityLog()
dir_ = os.path.join(self.tempdir, 'test')
al.add(mkdir, dir_)
al.commit()
al.add(mkdir, dir_, error_on_exists=False)
al.rollback()
self.assertTrue(os.path.exists(dir_))
def test_transaction(self):
al = ActivityLog()
dir_ = os.path.join(self.tempdir, 'test')
join = os.path.join
def dostuff():
al.add(mkdir, dir_)
al.add(create, join(dir_, 'testfile.txt'), content="Test")
al.add(copy, join(dir_, 'testfile.txt'), join(dir_, 'test2.txt'))
dostuff()
al.rollback()
self.assertFalse(os.path.exists(join(dir_, 'test2.txt')))
self.assertFalse(os.path.exists(join(dir_, 'testfile.txt')))
self.assertFalse(os.path.exists(dir_))
dostuff()
al.commit()
self.assertTrue(os.path.exists(dir_))
self.assertTrue(os.path.exists(join(dir_, 'testfile.txt')))
self.assertTrue(os.path.exists(join(dir_, 'test2.txt')))
def test_failed_rollback(self):
al = ActivityLog()
dir_ = os.path.join(self.tempdir, 'test')
inner_dir = os.path.join(dir_, 'inner')
al.add(mkdir, dir_)
al.add(mkdir, inner_dir)
os.chmod(dir_, stat.S_IRUSR|stat.S_IXUSR)
with self.assertRaises(OSError):
al.rollback()
self.assertTrue(os.path.exists(dir_))
self.assertTrue(os.path.exists(inner_dir))
os.chmod(dir_, stat.S_IRWXU | stat.S_IRWXG)
def run(self):
self.log.debug("Worker starting ... ")
self.socket = self.context.socket(zmq.SUB)
self.socket.setsockopt(zmq.SUBSCRIBE, "") # subscribe to all
self.socket.connect('inproc://tasks')
log = logging.getLogger('aybu')
while True:
task = Task(uuid=self.socket.recv(),
redis_client=self.redis,
started=datetime.datetime.now())
session = self.Session()
if not hasattr(session, 'activity_log'):
ActivityLog.attach_to(session)
level = int(task.get('log_level', logging.DEBUG))
handler = RedisPUBHandler(self.config, self.pub_socket,
self.context, level=level)
handler.set_task(task)
log.addHandler(handler)
log.setLevel(level)
result = None
try:
module_name = 'aybu.manager.daemon.commands.{}'\
.format(task.command_module)
module = __import__(module_name,
fromlist=[task.command_name])
function = getattr(module, task.command_name)
log.debug('Task received: %s: %s', task, task.command_args)
result = function(session, task, **task.command_args)
session.commit()
except ImportError:
session.rollback()
task.status = taskstatus.FAILED
task.result = "Cannot find resource {}"\
.format(task.command_module)
log.exception(task.result)
except AttributeError:
session.rollback()
task.status = taskstatus.FAILED
task.result = "Cannot find action {} on {}"\
.format(task.command_name, task.command_module)
log.critical(task.result)
except Exception:
session.rollback()
log.exception('Error while executing task')
task.status = taskstatus.FAILED
task.result = str('Error')
else:
task.status = taskstatus.FINISHED
log.info("Task completed successfully")
finally:
task['finished'] = datetime.datetime.now()
task.result = result or ''
self.pub_socket.send_multipart(["{}.finished".format(task.uuid),
"task endend"])
session.close()
log.removeHandler(handler)
del handler
self.socket.close()
def test_transaction_status(self):
al = ActivityLog(autobegin=False)
with self.assertRaises(TransactionError):
al.commit()
with self.assertRaises(TransactionError):
al.rollback()
al.begin()
al.commit()
with self.assertRaises(TransactionError):
al.commit()
def test_create(self):
al = ActivityLog()
# test rollback
file_= os.path.join(self.tempdir, 'test.txt')
al.add(create, file_)
self.assertTrue(os.path.exists(file_))
al.rollback()
self.assertFalse(os.path.exists(file_))
# test successfull create
al.add(create, file_)
al.commit()
self.assertTrue(os.path.exists(file_))
# test unsuccessfull create
with self.assertRaises(OSError):
al.add(create, file_)
self.assertTrue(os.path.exists(file_))
def test_mv(self):
al = ActivityLog()
source = os.path.join(self.tempdir, "source")
destination = os.path.join(self.tempdir, "destination")
os.mkdir(source)
os.mkdir(destination)
with self.assertRaises(OSError):
al.add(mv, source, destination)
shutil.rmtree(destination)
al.add(mv, source, destination)
self.assertFalse(os.path.exists(source))
self.assertTrue(os.path.exists(destination))
al.rollback()
self.assertTrue(os.path.exists(source))
self.assertFalse(os.path.exists(destination))
al.add(mv, source, destination)
al.commit()
self.assertFalse(os.path.exists(source))
self.assertTrue(os.path.exists(destination))
def test_delete(self):
al = ActivityLog()
testfile = os.path.join(self.tempdir, 'test.txt')
with self.assertRaises(OSError):
al.add(rm, testfile)
al.add(rm, testfile, error_on_not_exists=False)
al.commit()
with open(testfile, "w") as f:
f.write("###")
al.add(rm, testfile)
self.assertFalse(os.path.exists(testfile))
al.rollback()
self.assertTrue(os.path.exists(testfile))
al.add(rm, testfile)
self.assertFalse(os.path.exists(testfile))
al.commit()
self.assertFalse(os.path.exists(testfile))
testdir = os.path.join(self.tempdir, 'test')
al.add(mkdir, testdir)
al.commit()
# test rmdir
al.add(rmdir, testdir)
self.assertFalse(os.path.exists(testdir))
al.rollback()
self.assertTrue(os.path.exists(testdir))
al.add(rmdir, testdir)
al.commit()
self.assertFalse(os.path.exists(testdir))
# test rmtree
al.add(mkdir, testdir)
inner = os.path.join(testdir, 'inner')
al.add(mkdir, inner)
al.commit()
al.add(rmtree, testdir)
self.assertFalse(os.path.exists(testdir))
al.rollback()
self.assertTrue(os.path.exists(testdir))
al.add(rmtree, testdir)
al.commit()
self.assertFalse(os.path.exists(testdir))
