def daemonize(args, callback):
with DaemonContext():
from touchandgo.logger import log_set_up
log_set_up(True)
log = logging.getLogger('touchandgo.daemon')
log.info("running daemon")
create_process = False
lock = Lock(LOCKFILE, os.getpid(), args.name, args.sea_ep[0],
args.sea_ep[1], args.port)
if lock.is_locked():
log.debug("lock active")
lock_pid = lock.get_pid()
if not lock.is_same_file(args.name, args.sea_ep[0],
args.sea_ep[1]) \
or not is_process_running(lock_pid):
try:
log.debug("killing process %s" % lock_pid)
os.kill(lock_pid, signal.SIGQUIT)
except OSError:
pass
except TypeError:
pass
lock.break_lock()
create_process = True
else:
create_process = True
if create_process:
log.debug("creating proccess")
lock.acquire()
callback()
lock.release()
else:
log.debug("same daemon process")
def daemonize(args, callback):
with DaemonContext():
from touchandgo.logger import log_set_up
log_set_up(True)
log = logging.getLogger('touchandgo.daemon')
log.info("running daemon")
create_process = False
lock = Lock(LOCKFILE, os.getpid(), args.name, args.sea_ep[0],
args.sea_ep[1], args.port)
if lock.is_locked():
log.debug("lock active")
lock_pid = lock.get_pid()
if not lock.is_same_file(args.name, args.sea_ep[0],
args.sea_ep[1]) \
or not is_process_running(lock_pid):
try:
log.debug("killing process %s" % lock_pid)
os.kill(lock_pid, signal.SIGQUIT)
except OSError:
pass
except TypeError:
pass
lock.break_lock()
create_process = True
else:
create_process = True
if create_process:
log.debug("creating proccess")
lock.acquire()
callback()
lock.release()
else:
log.debug("same daemon process")
def main():
lock = Lock('/tmp/poll_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warn('Lock file {} acquired'.format(lock.lock_file))
url = properties.PASTA_BASE_URL + '/changes/eml?'
qm = QueueManager()
fromDate = None
dt = qm.get_last_datetime()
if dt is not None:
fromDate = datetime.strftime(dt, '%Y-%m-%dT%H:%M:%S.%f')
if fromDate is None:
bootstrap(url=url)
else:
parse(url=url, fromDate=fromDate)
lock.release()
logger.warn('Lock file {} released'.format(lock.lock_file))
return 0
def main(package_id, dryrun):
logger.info(f'package_id={package_id}')
lock = Lock('/tmp/poll_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warning('Lock file {} acquired'.format(lock.lock_file))
try:
scope, identifier, revision = package_id.split('.')
connection = connect()
event = get_package_info(connection, scope, identifier, revision)
if event:
qm = QueueManager()
msg = f"Enqueue: {event.package} - {event.datetime} - " + \
f"{event.owner} - {event.doi} - {event.method}"
logger.warning(msg)
if not dryrun:
qm.enqueue(event=event)
else:
msg = f"DRYRUN: qm.enqueue(event=event)"
logger.info(msg)
except AdapterRequestFailureException as e:
logger.error(e)
lock.release()
logger.warning('Lock file {} released'.format(lock.lock_file))
return 0
class Election:
def __init__(self, name, is_master_callback, lost_master_callback):
self.lock = Lock(name,
lock_callback=self._lock,
lock_lost_callback=self._lost_lock)
self.master_callback = is_master_callback
self.lost_master_callback = lost_master_callback
self.running = False
self.condition = threading.Condition()
def shutdown(self):
self.running = False
self.condition.acquire()
self.condition.notify()
self.condition.release()
def run(self):
self.running = True
while self.running:
self.lock.acquire()
self.condition.acquire()
self.condition.wait()
self.condition.release()
self.lock.release()
def _lock(self):
self.master_callback()
def _lost_lock(self):
self.lost_master_callback()
class PhysicalView(Base):
def __init__(self, dfs):
Base.__init__(self, dfs)
self.lock_ = Lock(dfs)
self.createBaseFolder()
def read(self, fileName, buf, offset, bufsize):
# TODO add thread safetly
filePath = os.path.join(self.getBasePath(), fileName)
size = self.getFileSize(fileName)
if offset + bufsize > size:
self.log_.w('tried to read ' + fileName + ' but size is ' + str(size) +
' and bufsize + offset = ' + str(offset + bufsize))
return err.InvalidBufferSize
self.lock_.acquire()
try:
f = open(filePath, "r")
except Exception, ex:
self.log_.e('error opening file in read mode ' + filePath + ': ' + str(ex))
self.lock_.release()
return err.FileNotFound
status = err.OK
f.seek(offset)
try:
data = f.read(bufsize)
for i, d in enumerate(data):
buf[i] = d
except Exception, ex:
self.log_.e('failed to read ' + filePath + ' from ' + str(offset) + ' to ' + str(offset + bufsize) + ': ' + str(ex))
status = err.CannotReadFile
def detachThin(session, lvmCache, srUuid, vdiUuid):
"""Shrink the VDI to the minimal size if no one is using it"""
lvName = LV_PREFIX[VDI_TYPE_VHD] + vdiUuid
path = os.path.join(VG_LOCATION, VG_PREFIX + srUuid, lvName)
lock = Lock(vhdutil.LOCK_TYPE_SR, srUuid)
lock.acquire()
vdiRef = session.xenapi.VDI.get_by_uuid(vdiUuid)
vbds = session.xenapi.VBD.get_all_records_where( \
"field \"VDI\" = \"%s\"" % vdiRef)
numPlugged = 0
for vbdRec in vbds.values():
if vbdRec["currently_attached"]:
numPlugged += 1
if numPlugged > 1:
raise util.SMException("%s still in use by %d others" % \
(vdiUuid, numPlugged - 1))
lvmCache.activate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
try:
newSize = calcSizeLV(vhdutil.getSizePhys(path))
deflate(lvmCache, lvName, newSize)
finally:
lvmCache.deactivate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
lock.release()
class TestLock(unittest.TestCase):
def setUp(self):
self.client = Redis()
self.client.flushdb()
self.lock = Lock(self.client, 'lock')
def test_only_one_lock_can_be_acquire(self):
self.assertTrue(
self.lock.acquire()
)
self.assertFalse(
self.lock.acquire()
)
def test_release_works(self):
self.lock.acquire()
self.assertTrue(
self.lock.release()
)
def test_release_return_false_when_lock_not_acquired(self):
self.assertFalse(
self.lock.release()
)
def daemonize(args, callback):
with DaemonContext():
create_process = False
lock = Lock(LOCKFILE, os.getpid(), args.name, args.sea_ep[0],
args.sea_ep[1], args.port)
if lock.is_locked():
lock_pid = lock.get_pid()
if not lock.is_same_file(args.name, args.sea_ep[0],
args.sea_ep[1]) \
or not is_process_running(lock_pid):
try:
os.kill(lock_pid, signal.SIGQUIT)
except OSError:
pass
except TypeError:
pass
lock.break_lock()
create_process = True
else:
create_process = True
if create_process:
lock.acquire()
callback(args.name, season=args.sea_ep[0], episode=args.sea_ep[1],
serve=True, port=args.port)
lock.release()
def checkLocked(obj, ns):
"""Lock-protected access"""
lock = Lock(obj, ns)
lock.acquire()
try:
return RefCounter.check(obj, ns)
finally:
lock.release()
def checkLocked(obj, ns):
"""Lock-protected access"""
lock = Lock(obj, ns)
lock.acquire()
try:
return RefCounter.check(obj, ns)
finally:
lock.release()
def setReadonly(self, lvName, readonly):
path = self._getPath(lvName)
if self.lvs[lvName].readonly != readonly:
uuids = util.findall_uuid(path)
ns = lvhdutil.NS_PREFIX_LVM + uuids[0]
lock = Lock(uuids[1], ns)
lock.acquire()
lvutil.setReadonly(path, readonly)
lock.release()
self.lvs[lvName].readonly = readonly
def setReadonly(self, lvName, readonly):
path = self._getPath(lvName)
if self.lvs[lvName].readonly != readonly:
uuids = util.findall_uuid(path)
ns = lvhdutil.NS_PREFIX_LVM + uuids[0]
lock = Lock(uuids[1], ns)
lock.acquire()
lvutil.setReadonly(path, readonly)
lock.release()
self.lvs[lvName].readonly = readonly
def setReadonly(self, lvName, readonly):
path = self._getPath(lvName)
if self.lvs[lvName].readonly != readonly:
uuids = util.findall_uuid(path)
ns = lvhdutil.NS_PREFIX_LVM + uuids[0]
# Taking this lock is needed to avoid a race condition
# with tap-ctl open (which is now taking the same lock)
lock = Lock("lvchange-p", ns)
lock.acquire()
lvutil.setReadonly(path, readonly)
lock.release()
self.lvs[lvName].readonly = readonly
def setReadonly(self, lvName, readonly):
path = self._getPath(lvName)
if self.lvs[lvName].readonly != readonly:
uuids = util.findall_uuid(path)
ns = lvhdutil.NS_PREFIX_LVM + uuids[0]
# Taking this lock is needed to avoid a race condition
# with tap-ctl open (which is now taking the same lock)
lock = Lock("lvchange-p", ns)
lock.acquire()
lvutil.setReadonly(path, readonly)
lock.release()
self.lvs[lvName].readonly = readonly
def activate(self, ns, ref, lvName, binary):
lock = Lock(ref, ns)
lock.acquire()
try:
count = RefCounter.get(ref, binary, ns)
if count == 1:
try:
self.activateNoRefcount(lvName)
except util.CommandException:
RefCounter.put(ref, binary, ns)
raise
finally:
lock.release()
def activate(self, ns, ref, lvName, binary):
lock = Lock(ref, ns)
lock.acquire()
try:
count = RefCounter.get(ref, binary, ns)
if count == 1:
try:
self.activateNoRefcount(lvName)
except util.CommandException:
RefCounter.put(ref, binary, ns)
raise
finally:
lock.release()
def _cbt_op(self, uuid, func, *args):
# Lock cbtlog operations
from lock import Lock
lock = Lock("cbtlog", str(uuid))
lock.acquire()
try:
logname = self._get_cbt_logname(uuid)
activated = self._activate_cbt_log(logname)
ret = func(*args)
if activated:
self._deactivate_cbt_log(logname)
return ret
finally:
lock.release()
def _cbt_op(self, uuid, func, *args):
# Lock cbtlog operations
from lock import Lock
lock = Lock("cbtlog", str(uuid))
lock.acquire()
try:
logname = self._get_cbt_logname(uuid)
activated = self._activate_cbt_log(logname)
ret = func(*args)
if activated:
self._deactivate_cbt_log(logname)
return ret
finally:
lock.release()
def deactivate(self, sr_uuid, vdi_uuid):
"""Deactivate VDI - called post tapdisk close"""
if self._get_blocktracking_status():
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
logpath = self._get_cbt_logpath(vdi_uuid)
logname = self._get_cbt_logname(vdi_uuid)
self._cbt_op(vdi_uuid, cbtutil.set_cbt_consistency, logpath, True)
# Finally deactivate log file
self._deactivate_cbt_log(logname)
finally:
lock.release()
def deactivateVdi(sr_uuid, vdi_uuid, vhd_path):
name_space = lvhdutil.NS_PREFIX_LVM + sr_uuid
lock = Lock(vdi_uuid, name_space)
lock.acquire()
try:
count = RefCounter.put(vdi_uuid, False, name_space)
if count > 0:
return
try:
lvutil.deactivateNoRefcount(vhd_path)
except Exception as e:
util.SMlog(" lv de-activate failed for %s with error %s" %
(vhd_path, str(e)))
RefCounter.get(vdi_uuid, False, name_space)
finally:
lock.release()
def deactivateVdi(sr_uuid, vdi_uuid, vhd_path):
name_space = lvhdutil.NS_PREFIX_LVM + sr_uuid
lock = Lock(vdi_uuid, name_space)
lock.acquire()
try:
count = RefCounter.put(vdi_uuid, False, name_space)
if count > 0:
return
try:
lvutil.deactivateNoRefcount(vhd_path)
except Exception, e:
util.SMlog(" lv de-activate failed for %s with error %s" %
(vhd_path, str(e)))
RefCounter.get(vdi_uuid, False, name_space)
finally:
lock.release()
def deactivate(self, sr_uuid, vdi_uuid):
"""Deactivate VDI - called post tapdisk close"""
if self._get_blocktracking_status():
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
logpath = self._get_cbt_logpath(vdi_uuid)
logname = self._get_cbt_logname(vdi_uuid)
self._cbt_op(vdi_uuid, cbtutil.set_cbt_consistency, logpath,
True)
# Finally deactivate log file
self._deactivate_cbt_log(logname)
finally:
lock.release()
def main():
lock = Lock()
try:
lock.acquire()
except LockError as exc:
error("Couldn't create lock file: %s" % exc)
lock = None
else:
try:
gainer = Gainer(_get_options())
gainer.process()
except KeyboardInterrupt:
error("Interrupted by user.")
finally:
if lock:
lock.release()
def main():
lock = Lock('/tmp/package_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warning('Lock file {} acquired'.format(lock.lock_file))
qm = QueueManager()
head = qm.get_head()
while head is not None:
logger.warning('Active package: {p}'.format(p=head.package))
skip = False
if properties.CHECK_PRE_EXISTENCE_IN_GMN and head.method in [properties.CREATE, properties.UPDATE]:
skip = gmn_exists(properties.PASTA_BASE_URL + 'metadata/eml/' + head.package.replace('.', '/'))
if skip:
logger.warning('Package already exists: {}. Skipping {}.'.format(head.package, head.method))
else:
p = Package(head)
if p.public:
logger.warning('Processing: {p}'.format(p=p.package))
resource = p.resources[properties.METADATA]
if p.method == properties.CREATE:
process_create_package(package=p)
elif p.method == properties.UPDATE:
process_update_package(package=p, queue_manager=qm)
elif p.method == properties.DELETE:
process_archive_package(package=p)
else:
msg = 'Unrecognized package event "{event}" for' \
'package: {package}'.format(event=p.method,
package=p.package)
raise(AdapterIncompleteStateException(msg))
else:
logger.warning('Package not public: {p}'.format(p=p.package))
qm.dequeue(package=head.package, method=head.method)
if properties.SLEEP_BETWEEN_PACKAGES:
time.sleep(int(properties.SLEEP_BETWEEN_PACKAGES))
head = qm.get_head()
logger.warning('Queue empty')
lock.release()
logger.warning('Lock file {} released'.format(lock.lock_file))
return 0
class LogicalView(Base):
def __init__(self, dfs):
Base.__init__(self, dfs)
self.lock_ = Lock(dfs)
self.fileList_ = {}
def beginLocalUpdate(self, fileName):
file = self.getFile(fileName)
if file.latestVersion != file.localVersion:
file.localVersion = file.latestVersion.copy()
file.ownNoChunks()
def add(self, fileName, fileSize):
self.lock_.acquire()
f = File(fileName, 1, fileSize, self.dfs_.id.str)
self.fileList_[fileName] = f
self.lock_.release()
def delete(self, fileName):
self.lock_.acquire()
self.fileList_[fileName].isDeleted = True
self.lock_.release()
def exists(self, fileName):
return (fileName in self.fileList_)
def getLocalVersion(self, fileName):
return self.fileList_[fileName].getLocalVersion()
def getLatestVersion(self, fileName):
return self.fileList_[fileName].getLatestVersion()
def setNewVersion(self, fileName, version):
self.fileList_[fileName].setNewVersion(version.copy())
def setLocalVersion(self, fileName, numEdits, fileSize, lastEdited):
self.fileList_[fileName].setLocalVersion(numEdits, fileSize, lastEdited)
def getFileList(self):
return self.fileList_.values()
def getState(self):
return self.fileList_
def getFile(self, fileName):
return self.fileList_[fileName]
def test():
# Create a Lock
lock = Lock("test")
# Should not be yet held.
assert lock.held() == False
# Go get it
lock.acquire()
# Second lock shall throw in debug mode.
try:
lock.acquire()
except AssertionError, e:
if str(e) != flock.WriteLock.ERROR_ISLOCKED:
raise
def attachThin(journaler, srUuid, vdiUuid):
"""Ensure that the VDI LV is expanded to the fully-allocated size"""
lvName = LV_PREFIX[VDI_TYPE_VHD] + vdiUuid
vgName = VG_PREFIX + srUuid
lock = Lock(vhdutil.LOCK_TYPE_SR, srUuid)
lvmCache = journaler.lvmCache
lock.acquire()
vhdInfo = vhdutil.getVHDInfoLVM(lvName, extractUuid, vgName)
newSize = calcSizeVHDLV(vhdInfo.sizeVirt)
currSizeLV = lvmCache.getSize(lvName)
if newSize <= currSizeLV:
return
lvmCache.activate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
try:
inflate(journaler, srUuid, vdiUuid, newSize)
finally:
lvmCache.deactivate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
lock.release()
def activate(self, sr_uuid, vdi_uuid):
"""Activate VDI - called pre tapdisk open"""
if self._get_blocktracking_status():
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
logpath = self._get_cbt_logpath(vdi_uuid)
logname = self._get_cbt_logname(vdi_uuid)
# Activate CBT log file, if required
self._activate_cbt_log(logname)
finally:
lock.release()
# Check and update consistency
consistent = self._cbt_op(vdi_uuid, cbtutil.get_cbt_consistency,
logpath)
if not consistent:
lock.acquire()
try:
self._delete_cbt_log()
finally:
lock.release()
vdi_ref = self.sr.srcmd.params['vdi_ref']
self.sr.session.xenapi.VDI.set_cbt_enabled(vdi_ref, False)
alert_name = "VDI_CBT_METADATA_INCONSISTENT"
alert_prio_warning = "3"
alert_obj = "VDI"
alert_uuid = str(vdi_uuid)
alert_str = ("Changed Block Tracking metadata is inconsistent"
" for disk %s." % vdi_uuid)
util.SMlog(alert_str)
self.sr.session.xenapi.message.create(alert_name,
alert_prio_warning,
alert_obj, alert_uuid,
alert_str)
return None
self._cbt_op(self.uuid, cbtutil.set_cbt_consistency, logpath,
False)
return {'cbtlog': logpath}
return None
def deactivate(self, ns, ref, lvName, binary):
lock = Lock(ref, ns)
lock.acquire()
try:
count = RefCounter.put(ref, binary, ns)
if count > 0:
return
refreshed = False
while True:
lvInfo = self.getLVInfo(lvName)
if len(lvInfo) != 1:
raise util.SMException("LV info not found for %s" % ref)
info = lvInfo[lvName]
if info.open:
if refreshed:
# should never happen in normal conditions but in some
# failure cases the recovery code may not be able to
# determine what the correct refcount should be, so it
# is not unthinkable that the value might be out of
# sync
util.SMlog("WARNING: deactivate: LV %s open" % lvName)
return
# check again in case the cached value is stale
self.refresh()
refreshed = True
else:
break
try:
self.deactivateNoRefcount(lvName)
except util.CommandException:
self.refresh()
if self.getLVInfo(lvName):
util.SMlog("LV %s could not be deactivated" % lvName)
if lvInfo[lvName].active:
util.SMlog("Reverting the refcount change")
RefCounter.get(ref, binary, ns)
raise
else:
util.SMlog("LV %s not found" % lvName)
finally:
lock.release()
def deactivate(self, ns, ref, lvName, binary):
lock = Lock(ref, ns)
lock.acquire()
try:
count = RefCounter.put(ref, binary, ns)
if count > 0:
return
refreshed = False
while True:
lvInfo = self.getLVInfo(lvName)
if len(lvInfo) != 1:
raise util.SMException("LV info not found for %s" % ref)
info = lvInfo[lvName]
if info.open:
if refreshed:
# should never happen in normal conditions but in some
# failure cases the recovery code may not be able to
# determine what the correct refcount should be, so it
# is not unthinkable that the value might be out of
# sync
util.SMlog("WARNING: deactivate: LV %s open" % lvName)
return
# check again in case the cached value is stale
self.refresh()
refreshed = True
else:
break
try:
self.deactivateNoRefcount(lvName)
except util.CommandException:
self.refresh()
if self.getLVInfo(lvName):
util.SMlog("LV %s could not be deactivated" % lvName)
if lvInfo[lvName].active:
util.SMlog("Reverting the refcount change")
RefCounter.get(ref, binary, ns)
raise
else:
util.SMlog("LV %s not found" % lvName)
finally:
lock.release()
def main():
lock = Lock('/tmp/package_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warn('Lock file {} acquired'.format(lock.lock_file))
qm = QueueManager()
head = qm.get_head()
while head is not None:
logger.warn('Active package: {p}'.format(p=head.package))
p = Package(head)
if p.public:
logger.warn('Processing: {p}'.format(p=p.package))
if p.method == properties.CREATE:
process_create_package(package=p)
elif p.method == properties.UPDATE:
process_update_package(package=p, queue_manager=qm)
elif p.method == properties.DELETE:
process_archive_package(package=p)
else:
msg = 'Unrecognized package event "{event}" for' \
'package: {package}'.format(event=p.method,
package=p.package)
raise (AdapterIncompleteStateException(msg))
else:
logger.warn('Package not public: {p}'.format(p=p.package))
qm.dequeue(package=p.package, method=p.method)
head = qm.get_head()
logger.warn('Queue empty')
lock.release()
logger.warn('Lock file {} released'.format(lock.lock_file))
return 0
class TestLock(unittest.TestCase):
def setUp(self):
self.lock = Lock('bozo.lock')
def tearDown(self):
try:
self.lock.release()
except IOError as e:
logger.error(e)
def test_acquire_release(self):
print(self.lock.acquire())
try:
self.lock.release()
except IOError as e:
logger.error(e)
self.fail()
self.lock.acquire()
def test_locked(self):
self.lock.acquire()
self.assertTrue(self.lock.locked)
def test():
# Create a Lock
lock = Lock("test")
# Should not be yet held.
assert lock.held() == False
# Go get it
lock.acquire()
# Second lock shall throw in debug mode.
try:
lock.acquire()
except AssertionError as e:
if str(e) != flock.WriteLock.ERROR_ISLOCKED:
raise
else:
raise AssertionError("Reaquired a locked lock")
lock.release()
Lock.cleanup()
def main():
lock = Lock('/tmp/poll_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warning('Lock file {} acquired'.format(lock.lock_file))
url = properties.PASTA_BASE_URL + 'changes/eml?'
qm = QueueManager()
# queue fromDate (fallback): effective but not efficient
fromDate = qm.get_last_datetime()
logger.info(f'"fromDate" from QueueManager: {fromDate}')
if fromDate is None: # Empty adapter_queue database
bootstrap(url=url)
else:
fromDate = pendulum.instance(dt=fromDate, tz='US/Mountain')
last_query_date = adapter_utilities.get_last_query_date()
if last_query_date is not None:
# pickled fromDate: effective and efficient
fromDate = last_query_date.in_tz('US/Mountain')
logger.info(f'"fromDate" from adapter_utilities: {fromDate}')
try:
query_date = pendulum.now(tz='UTC')
parse(url=url, fromDate=fromDate, scope=properties.SCOPE)
adapter_utilities.save_last_query_date(query_date)
except AdapterRequestFailureException as e:
logger.error(e)
lock.release()
logger.warning('Lock file {} released'.format(lock.lock_file))
return 0
def activate(self, sr_uuid, vdi_uuid):
"""Activate VDI - called pre tapdisk open"""
if self._get_blocktracking_status():
if self.sr.srcmd.params.has_key('args'):
read_write = self.sr.srcmd.params['args'][0]
if read_write == "false":
# Disk is being attached in RO mode,
# don't attach metadata log file
return None
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
logpath = self._get_cbt_logpath(vdi_uuid)
logname = self._get_cbt_logname(vdi_uuid)
# Activate CBT log file, if required
self._activate_cbt_log(logname)
finally:
lock.release()
# Check and update consistency
consistent = self._cbt_op(vdi_uuid, cbtutil.get_cbt_consistency,
logpath)
if not consistent:
alert_name = "VDI_CBT_METADATA_INCONSISTENT"
alert_str = ("Changed Block Tracking metadata is inconsistent"
" for disk %s." % vdi_uuid)
self._disable_cbt_on_error(alert_name, alert_str)
return None
self._cbt_op(self.uuid, cbtutil.set_cbt_consistency, logpath,
False)
return {'cbtlog': logpath}
return None
def activate(self, sr_uuid, vdi_uuid):
"""Activate VDI - called pre tapdisk open"""
if self._get_blocktracking_status():
if self.sr.srcmd.params.has_key('args'):
read_write = self.sr.srcmd.params['args'][0]
if read_write == "false":
# Disk is being attached in RO mode,
# don't attach metadata log file
return None
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
logpath = self._get_cbt_logpath(vdi_uuid)
logname = self._get_cbt_logname(vdi_uuid)
# Activate CBT log file, if required
self._activate_cbt_log(logname)
finally:
lock.release()
# Check and update consistency
consistent = self._cbt_op(vdi_uuid, cbtutil.get_cbt_consistency,
logpath)
if not consistent:
alert_name = "VDI_CBT_METADATA_INCONSISTENT"
alert_str = ("Changed Block Tracking metadata is inconsistent"
" for disk %s." % vdi_uuid)
self._disable_cbt_on_error(alert_name, alert_str)
return None
self._cbt_op(self.uuid, cbtutil.set_cbt_consistency,
logpath, False)
return {'cbtlog': logpath}
return None
def lock(self, lockid, blocking=True, timeout=LOCK_TIMEOUT):
# with self.__locks_lock:
lock = Lock(self.client, self.lock_path_prefix + lockid)
try:
acquired = lock.acquire(blocking=blocking, timeout=timeout)
logger.debug('Lock {0} acquired: {1}'.format(lockid, acquired))
if not acquired:
raise LockFailedError(lock_id=lockid)
yield
except LockTimeout:
logger.info('Failed to acquire lock {0} due to timeout '
'({1} seconds)'.format(lockid, timeout))
raise LockFailedError(lock_id=lockid)
except LockFailedError:
raise
except Exception as e:
logger.error('Failed to acquire lock {0}: {1}\n{2}'.format(
lockid, e, traceback.format_exc()))
raise
finally:
lock.release()
def lock(self, lockid, blocking=True, timeout=LOCK_TIMEOUT):
lock = Lock(self.client, self.lock_path_prefix + lockid)
try:
acquired = lock.acquire(blocking=blocking, timeout=timeout)
logger.debug('Lock {0} acquired: {1}'.format(lockid, acquired))
if not acquired:
# TODO: Change exception time or set all required parameters for
# this type of exception
raise LockAlreadyAcquiredError(lock_id=lockid)
yield
except LockTimeout:
logger.info('Failed to acquire lock {} due to timeout ({} seconds)'.format(
lockid, timeout))
raise LockFailedError(lock_id=lockid)
except LockAlreadyAcquiredError:
raise
except LockError as e:
logger.error('Failed to acquire lock {0}: {1}\n{2}'.format(
lockid, e, traceback.format_exc()))
raise
finally:
lock.release()
def lock(self, lockid, blocking=True, timeout=LOCK_TIMEOUT):
lock = Lock(self.client, self.lock_path_prefix + lockid)
try:
acquired = lock.acquire(blocking=blocking, timeout=timeout)
logger.debug('Lock {0} acquired: {1}'.format(lockid, acquired))
if not acquired:
# TODO: Change exception time or set all required parameters for
# this type of exception
raise LockAlreadyAcquiredError(lock_id=lockid)
yield
except LockTimeout:
logger.info(
'Failed to acquire lock {} due to timeout ({} seconds)'.format(
lockid, timeout))
raise LockFailedError(lock_id=lockid)
except LockAlreadyAcquiredError:
raise
except LockError as e:
logger.error('Failed to acquire lock {0}: {1}\n{2}'.format(
lockid, e, traceback.format_exc()))
raise
finally:
lock.release()
#coding:utf-8
from redis import Redis
from lock import Lock
from uuid import uuid4
client = Redis()
client.flushdb()
lock = Lock(client, 'test-lock')
identifier = str(uuid4())
# acquire test
assert (lock.acquire(identifier) is True)
assert (lock.acquire('another-identifier') is False)
# release test
assert (lock.release("another-identifier") is False)
assert (lock.release(identifier) is True)
assert (lock.release("lock is empty now") is None)
# acquire after release test
assert (lock.acquire(identifier) is True)
def delete(self, sr_uuid, vdi_uuid, data_only=False):
"""Delete this VDI.
This operation IS idempotent and should succeed if the VDI
exists and can be deleted or if the VDI does not exist. It is
the responsibility of the higher-level management tool to
ensure that the detach() operation has been explicitly called
prior to deletion, otherwise the delete() will fail if the
disk is still attached.
"""
import blktap2
from lock import Lock
if data_only == False and self._get_blocktracking_status():
logpath = self._get_cbt_logpath(vdi_uuid)
parent_uuid = self._cbt_op(vdi_uuid, cbtutil.get_cbt_parent,
logpath)
parent_path = self._get_cbt_logpath(parent_uuid)
child_uuid = self._cbt_op(vdi_uuid, cbtutil.get_cbt_child, logpath)
child_path = self._get_cbt_logpath(child_uuid)
lock = Lock("cbtlog", str(vdi_uuid))
if self._cbt_log_exists(parent_path):
self._cbt_op(parent_uuid, cbtutil.set_cbt_child, parent_path,
child_uuid)
if self._cbt_log_exists(child_path):
self._cbt_op(child_uuid, cbtutil.set_cbt_parent, child_path,
parent_uuid)
lock.acquire()
try:
# Coalesce contents of bitmap with child's bitmap
# Check if child bitmap is currently attached
paused_for_coalesce = False
consistent = self._cbt_op(child_uuid,
cbtutil.get_cbt_consistency,
child_path)
if not consistent:
if not blktap2.VDI.tap_pause(self.session, sr_uuid,
child_uuid):
raise util.SMException("failed to pause VDI %s")
paused_for_coalesce = True
self._activate_cbt_log(self._get_cbt_logname(vdi_uuid))
self._cbt_op(child_uuid, cbtutil.coalesce_bitmap, logpath,
child_path)
lock.release()
except util.CommandException:
# If there is an exception in coalescing,
# CBT log file is not deleted and pointers are reset
# to what they were
util.SMlog("Exception in coalescing bitmaps on VDI delete,"
" restoring to previous state")
try:
if self._cbt_log_exists(parent_path):
self._cbt_op(parent_uuid, cbtutil.set_cbt_child,
parent_path, vdi_uuid)
if self._cbt_log_exists(child_path):
self._cbt_op(child_uuid, cbtutil.set_cbt_parent,
child_path, vdi_uuid)
finally:
lock.release()
lock.cleanup("cbtlog", str(vdi_uuid))
return
finally:
# Unpause tapdisk if it wasn't originally paused
if paused_for_coalesce:
blktap2.VDI.tap_unpause(self.session, sr_uuid,
child_uuid)
lock.acquire()
try:
self._delete_cbt_log()
finally:
lock.release()
lock.cleanup("cbtlog", str(vdi_uuid))
def configure_blocktracking(self, sr_uuid, vdi_uuid, enable):
"""Function for configuring blocktracking"""
import blktap2
vdi_ref = self.sr.srcmd.params['vdi_ref']
# Check if raw VDI or snapshot
if self.vdi_type == vhdutil.VDI_TYPE_RAW or \
self.session.xenapi.VDI.get_is_a_snapshot(vdi_ref):
raise xs_errors.XenError(
'VDIType', opterr='Raw VDI or snapshot not permitted')
# Check if already enabled
if self._get_blocktracking_status() == enable:
return
# Save disk state before pause
disk_state = blktap2.VDI.tap_status(self.session, vdi_uuid)
if not blktap2.VDI.tap_pause(self.session, sr_uuid, vdi_uuid):
error = "Failed to pause VDI %s" % vdi_uuid
raise xs_errors.XenError('CBTActivateFailed', opterr=error)
logfile = None
try:
if enable:
try:
# Check available space
self._ensure_cbt_space()
logfile = self._create_cbt_log()
# Set consistency
if disk_state:
util.SMlog(
"Setting consistency of cbtlog file to False for VDI: %s"
% self.uuid)
logpath = self._get_cbt_logpath(self.uuid)
self._cbt_op(self.uuid, cbtutil.set_cbt_consistency,
logpath, False)
except Exception as error:
self._delete_cbt_log()
raise xs_errors.XenError('CBTActivateFailed',
opterr=str(error))
else:
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
# Find parent of leaf metadata file, if any,
# and nullify its successor
logpath = self._get_cbt_logpath(self.uuid)
parent = self._cbt_op(self.uuid, cbtutil.get_cbt_parent,
logpath)
self._delete_cbt_log()
parent_path = self._get_cbt_logpath(parent)
if self._cbt_log_exists(parent_path):
self._cbt_op(parent, cbtutil.set_cbt_child,
parent_path, uuid.UUID(int=0))
except Exception as error:
raise xs_errors.XenError('CBTDeactivateFailed', str(error))
finally:
lock.release()
lock.cleanup("cbtlog", str(vdi_uuid))
finally:
blktap2.VDI.tap_unpause(self.session, sr_uuid, vdi_uuid)
def clientthread(conn, client_address):
#infinite loop so that function do not terminate and thread do not end.
while True:
#Receiving from client
try:
data = conn.recv(RECV_BUFFER) #release or lock resource_name
except:
#release all locked resources for the client address
client_lock = Lock("", client_address)
client_lock.release_by_client_address()
break
else:
#split the message Received to get the command and the resource name
tmp = data.split()
# check if the message Received follows the correct format
if tmp is not None and len(tmp) == 2 and (tmp[0] == "release"
or tmp[0] == "lock"):
# operation has to be lock or release
operation = tmp[0]
resource_name = tmp[1]
#targeted resource name and status
client_lock = Lock(resource_name, client_address)
resource_status = client_lock.check_status()
if not data:
break
elif resource_status is None:
# targeted resource is not listed in the database
reply = "required resource is not listed"
elif operation == 'lock' and resource_status == "free":
# you gain access to the resource
reply = "You have an exclusive access to resource " + resource_name
client_lock.acquire()
elif operation == 'lock' and resource_status == "busy":
# the resource is busy, you will wait TIMEOUT secs and retry
time.sleep(TIMEOUT)
# checking the resource status again after TIMEOUT
resource_status_new = client_lock.check_status()
if resource_status_new == "busy":
# it is still busy, try again later
reply = "required resource is busy now, you have to wait a while"
else:
#it's free now, you will gain access
reply = "You have an exclusive access to resource " + resource_name
client_lock.acquire()
elif operation == 'release':
if resource_status == "free":
#trying to release a free resource, not allowed
reply = "resource is already free."
#checking if the client who request release is the same client who lock it in the first place
elif client_address in client_lock.get_client_address():
reply = "lock released from resource " + resource_name
client_lock.release()
else:
# trying to release someone else's resource, not allowed
reply = 'it is not allowed to release someone else resource'
else:
#the message Received does not follow the correct format so send an error message
reply = "wrong message, you must send release or lock as the first word then space then the resource_name"
#send the message to the client
conn.sendall(reply)
#came out of loop and close the connection
conn.close()
def configure_blocktracking(self, sr_uuid, vdi_uuid, enable):
"""Function for configuring blocktracking"""
import blktap2
vdi_ref = self.sr.srcmd.params['vdi_ref']
# Check if raw VDI or snapshot
if self.vdi_type == vhdutil.VDI_TYPE_RAW or \
self.session.xenapi.VDI.get_is_a_snapshot(vdi_ref):
raise xs_errors.XenError('VDIType',
opterr='Raw VDI or snapshot not permitted')
# Check if already enabled
if self._get_blocktracking_status() == enable:
return
# Save disk state before pause
disk_state = blktap2.VDI.tap_status(self.session, vdi_uuid)
if not blktap2.VDI.tap_pause(self.session, sr_uuid, vdi_uuid):
error = "Failed to pause VDI %s" % vdi_uuid
raise xs_errors.XenError('CBTActivateFailed', opterr=error)
logfile = None
try:
if enable:
try:
# Check available space
self._ensure_cbt_space()
logfile = self._create_cbt_log()
# Set consistency
if disk_state:
util.SMlog("Setting consistency of cbtlog file to False for VDI: %s"
% self.uuid)
logpath = self._get_cbt_logpath(self.uuid)
self._cbt_op(self.uuid, cbtutil.set_cbt_consistency,
logpath, False)
except Exception as error:
self._delete_cbt_log()
raise xs_errors.XenError('CBTActivateFailed',
opterr=str(error))
else:
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
# Find parent of leaf metadata file, if any,
# and nullify its successor
logpath = self._get_cbt_logpath(self.uuid)
parent = self._cbt_op(self.uuid,
cbtutil.get_cbt_parent, logpath)
self._delete_cbt_log()
parent_path = self._get_cbt_logpath(parent)
if self._cbt_log_exists(parent_path):
self._cbt_op(parent, cbtutil.set_cbt_child,
parent_path, uuid.UUID(int=0))
except Exception as error:
raise xs_errors.XenError('CBTDeactivateFailed', str(error))
finally:
lock.release()
lock.cleanup("cbtlog", str(vdi_uuid))
finally:
blktap2.VDI.tap_unpause(self.session, sr_uuid, vdi_uuid)
class SenderThread(NetworkThread):
def __init__(self, dfs, fileSystem):
NetworkThread.__init__(self, dfs)
self.fileSystem_ = fileSystem
self.listeners_ = []
self.peerLock_ = Lock(dfs, 'peer')
self.workQueue_ = []
self.work_ = None
self.knownPeers_ = set()
self.fileFetchStatus = []
def isDoneFileFetch(self):
return len(self.fileFetchStatus) == 0
def getPeers(self):
return self.knownPeers_
def connectToMultiple(self, dfsList):
for dfs in dfsList:
self.connectTo(dfs)
def connectTo(self, dfs):
if self.isConnectedTo(dfs):
self.log_.v('already connected to ' + str(dfs.id))
return
self.registerConnDFS(dfs)
lt = ListenerThread(self.dfs_, self.addWork)
status = lt.connect(dfs)
if status < 0:
return status
self.log_.v('connected to ' + str(dfs.id))
lt.start()
self.addListener(lt)
def addListener(self, listener):
self.peerLock_.acquire()
self.listeners_.append(listener)
self.peerLock_.release()
self.addUpdateWork(listener)
def updateAll(self):
self.peerLock_.acquire()
for lt in self.listeners_:
self.addUpdateWork(lt)
self.peerLock_.release()
def beginFileFetch(self, fileName):
ltList = []
self.peerLock_.acquire()
self.fileFetchStatus = []
for lt in self.listeners_:
self.fileFetchStatus.append(lt)
ltList.append(lt)
self.peerLock_.release()
for lt in ltList:
self.addChunkRequestWork(lt, fileName)
def editPropagated(self):
# due to time constraints, I'm using this rather simple implementation
time.sleep(1)
return True
def isConnectedTo(self, dfs):
if dfs == self.dfs_:
return True
val = False
self.peerLock_.acquire()
for lt in self.listeners_:
if lt.getConnDFS() == dfs:
val = True
break
self.peerLock_.release()
return val
def registerConnDFS(self, dfs):
self.log_.v('registering new peer: ' + dfs.id.str)
self.knownPeers_.add(dfs)
def addWork(self, work):
self.workQueue_.append(work)
def doWork(self):
if len(self.workQueue_) > 0:
self.work_ = self.workQueue_.pop(0)
self.processWork()
else:
self.removeDisconnectedListeners()
time.sleep(.03)
def removeDisconnectedListeners(self):
self.peerLock_.acquire()
for lt in self.listeners_:
if not lt.active_:
self.log_.v('detected that ' + lt.getConnDFS().id.str + ' has disconnected')
self.removeListener(lt)
self.peerLock_.release()
def processWork(self):
if self.work_.source.id == self.dfs_.id:
self.sendWork()
else:
lt = self.work_.dest
if lt.getConnDFS().isInit():
self.knownPeers_.add(lt.getConnDFS())
if self.work_.type == work.UPDATE:
self.handleUpdate()
elif self.work_.type == work.CHUNK_REQUEST:
self.handleChunkRequest()
elif self.work_.type == work.CHUNK_RESPONSE:
self.handleChunkResponse()
elif self.work_.type == work.NO_CHUNK:
self.handleNoChunk()
else:
self.log_.e('received work unkonwn type: ' + self.work_.type)
def sendWork(self):
lt = self.work_.dest
status = lt.sendWork(self.work_)
if status < 0:
self.peerLock_.acquire()
self.log_.v('trying to send work, but ' + lt.getConnDFS().id.str + ' has disconnected')
self.removeListener(lt)
self.peerLock_.release()
def removeListener(self, lt):
if lt in self.fileFetchStatus:
self.fileFetchStatus.remove(lt)
if lt in self.listeners_:
self.listeners_.remove(lt) # other peer has disconnected
def handleUpdate(self):
lt = self.work_.dest
self.log_.v('received update from ' + str(lt.getConnDFS().id))
fs, ns = self.work_.data
status = self.fileSystem_.updateFiles(fs)
self.connectToMultiple(ns)
if status == err.CausedConflict:
self.log_.v('update from ' + str(lt.getConnDFS().id) + ' caused conflict, updating all peers')
self.updateAll()
def handleNoChunk(self):
lt = self.work_.dest
self.peerLock_.acquire()
if lt in self.fileFetchStatus:
self.log_.v(lt.getConnDFS().id.str + ' is no longer providing chunks')
self.fileFetchStatus.remove(lt)
self.peerLock_.release()
def handleChunkRequest(self):
fileName, missingChunks = self.work_.data
lt = self.work_.dest
chunkInfo = self.fileSystem_.getRandomChunk(fileName, missingChunks)
self.addChunkResponseWork(chunkInfo, fileName)
def handleChunkResponse(self):
lt = self.work_.dest
fileName, chunkNum, chunkData = self.work_.data
self.log_.v(lt.getConnDFS().id.str + ' received chunk ' + str(chunkNum) + ' of ' + fileName)
self.fileSystem_.writeChunk(fileName, chunkNum, chunkData)
self.addChunkRequestWork(lt, fileName)
def addUpdateWork(self, lt):
state = (self.fileSystem_.getState(), self.getPeers())
w = work.Work(work.UPDATE, self.dfs_, lt, state)
self.log_.v('sending an update to ' + lt.getConnDFS().id.str)
self.addWork(w)
def addChunkRequestWork(self, lt, fileName):
missingChunks = self.fileSystem_.getMissingChunks(fileName)
if missingChunks == None:
self.log_.v(fileName + ' is no longer missing chunks. File retrieval complete')
self.handleNoChunk()
return
data = (fileName, missingChunks)
w = work.Work(work.CHUNK_REQUEST, self.dfs_, lt, data)
self.log_.v('requesting a chunk of ' + fileName + ' from ' + lt.getConnDFS().id.str)
self.addWork(w)
def addChunkResponseWork(self, chunkInfo, fileName):
lt = self.work_.dest
type = ''
if chunkInfo:
type = work.CHUNK_RESPONSE
fileName, chunkNum, chunkData = chunkInfo
self.log_.v('sending chunk ' + str(chunkNum) + ' of ' + fileName + ' to ' + lt.getConnDFS().id.str)
else:
self.log_.v('do not have ' + fileName + ' chunk for ' + lt.getConnDFS().id.str)
type = work.NO_CHUNK
w = work.Work(type, self.dfs_, lt, chunkInfo)
self.addWork(w)
def close(self):
self.log_.d('close started')
self.peerLock_.acquire()
for listener in self.listeners_:
listener.close()
self.peerLock_.release()
self.waitForListenersToClose()
NetworkThread.close(self)
self.log_.d('close finished')
def waitForListenersToClose(self):
self.log_.d('waitForListenersToClose started')
self.peerLock_.acquire()
lts = list(self.listeners_)
self.peerLock_.release()
for lt in lts:
lt.join()
self.log_.d('waitForListenersToClose finished')
def delete(self, sr_uuid, vdi_uuid, data_only=False):
"""Delete this VDI.
This operation IS idempotent and should succeed if the VDI
exists and can be deleted or if the VDI does not exist. It is
the responsibility of the higher-level management tool to
ensure that the detach() operation has been explicitly called
prior to deletion, otherwise the delete() will fail if the
disk is still attached.
"""
import blktap2
from lock import Lock
if data_only == False and self._get_blocktracking_status():
logpath = self._get_cbt_logpath(vdi_uuid)
parent_uuid = self._cbt_op(vdi_uuid, cbtutil.get_cbt_parent,
logpath)
parent_path = self._get_cbt_logpath(parent_uuid)
child_uuid = self._cbt_op(vdi_uuid, cbtutil.get_cbt_child, logpath)
child_path = self._get_cbt_logpath(child_uuid)
lock = Lock("cbtlog", str(vdi_uuid))
if self._cbt_log_exists(parent_path):
self._cbt_op(parent_uuid, cbtutil.set_cbt_child,
parent_path, child_uuid)
if self._cbt_log_exists(child_path):
self._cbt_op(child_uuid, cbtutil.set_cbt_parent,
child_path, parent_uuid)
lock.acquire()
try:
# Coalesce contents of bitmap with child's bitmap
# Check if child bitmap is currently attached
paused_for_coalesce = False
consistent = self._cbt_op(child_uuid,
cbtutil.get_cbt_consistency,
child_path)
if not consistent:
if not blktap2.VDI.tap_pause(self.session,
sr_uuid, child_uuid):
raise util.SMException("failed to pause VDI %s")
paused_for_coalesce = True
self._activate_cbt_log(self._get_cbt_logname(vdi_uuid))
self._cbt_op(child_uuid, cbtutil.coalesce_bitmap,
logpath, child_path)
lock.release()
except util.CommandException:
# If there is an exception in coalescing,
# CBT log file is not deleted and pointers are reset
# to what they were
util.SMlog("Exception in coalescing bitmaps on VDI delete,"
" restoring to previous state")
try:
if self._cbt_log_exists(parent_path):
self._cbt_op(parent_uuid, cbtutil.set_cbt_child,
parent_path, vdi_uuid)
if self._cbt_log_exists(child_path):
self._cbt_op(child_uuid, cbtutil.set_cbt_parent,
child_path, vdi_uuid)
finally:
lock.release()
lock.cleanup("cbtlog", str(vdi_uuid))
return
finally:
# Unpause tapdisk if it wasn't originally paused
if paused_for_coalesce:
blktap2.VDI.tap_unpause(self.session, sr_uuid,
child_uuid)
lock.acquire()
try:
self._delete_cbt_log()
finally:
lock.release()
lock.cleanup("cbtlog", str(vdi_uuid))
class Logger(object):
def __init__(self, dry=False, name="scapy-tgen", logs_dir=None):
self.dry = dry
self.dbg = 1
self.log_file = None
self.lock = Lock()
self.fmt = LogFormatter()
self.logger = logging.getLogger(name)
stdlog = logging.StreamHandler(sys.stdout)
stdlog.setLevel(logging.ERROR if not self.dry else logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR if not self.dry else logging.DEBUG)
ch.setFormatter(self.fmt)
self.logger.addHandler(ch)
self.logger.removeHandler(stdlog)
self.logger.propagate = False
self.log_file = None
self.file_handler = None
if logs_dir:
self.logs_dir = logs_dir
else:
self.logs_dir = os.getenv("SCAPY_TGEN_LOGS_PATH", "server")
self.set_log_file(None)
@staticmethod
def ensure_parent(filename):
path = os.path.dirname(filename)
path = os.path.abspath(path)
if not os.path.exists(path):
os.makedirs(path)
def set_node_name(self, name):
if self.fmt:
self.fmt.node_name = name
def set_log_file(self, log_file):
if self.file_handler:
self.logger.removeHandler(self.file_handler)
self.file_handler = None
if log_file:
if self.logs_dir:
log_file = os.path.join(self.logs_dir, log_file)
self.ensure_parent(log_file)
self.file_handler = logging.FileHandler(log_file)
self.file_handler.setFormatter(self.fmt)
self.file_handler.setLevel(logging.DEBUG)
self.logger.addHandler(self.file_handler)
self.log_file = log_file
def get_log(self, filename=None):
if not filename:
filename = self.log_file
elif not os.path.exists(filename) and self.logs_dir:
filename = os.path.join(self.logs_dir, filename)
if not filename:
return ""
try:
fh = open(filename, 'r')
data = fh.readlines()
fh.close()
data = map(str.strip, data)
return "\n".join(data)
except Exception:
return ""
def todo(self, etype, name, value):
msg = "{}: {} = {}".format(etype, name, value)
self.error(msg)
raise ValueError(msg)
def debug(self, *args, **kwargs):
msg = " ".join(map(str, args))
self._log(logging.DEBUG, msg)
def _log(self, lvl, msg):
if not msg.strip(): return
self.lock.acquire()
for line in msg.split("\n"):
self.logger.log(lvl, line)
self.lock.release()
def log(self, *args, **kwargs):
msg = " ".join(map(str, args))
self._log(logging.INFO, msg)
def info(self, *args, **kwargs):
msg = " ".join(map(str, args))
self._log(logging.INFO, msg)
def error(self, *args, **kwargs):
msg = ""
#msg = msg + "=================================== "
msg = msg + " ".join(map(str, args))
#msg = msg + "=================================== "
self._log(logging.ERROR, msg)
def log_exception(self, e, msg):
self.logger.error("=========== exception ==================")
self.logger.error(msg)
self.logger.error("=========== exception ==================")
self.logger.error(msg, traceback.format_exc())
self.lock.acquire()
self.logger.exception(e)
self.lock.release()
@staticmethod
def setup():
logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
