def _kickGC(self):
# don't bother if an instance already running (this is just an
# optimization to reduce the overhead of forking a new process if we
# don't have to, but the process will check the lock anyways)
lockRunning = Lock(cleanup.LOCK_TYPE_RUNNING, self.uuid)
if not lockRunning.acquireNoblock():
if cleanup.should_preempt(self.session, self.uuid):
util.SMlog(
"Aborting currently-running coalesce of garbage VDI")
try:
if not cleanup.abort(self.uuid, soft=True):
util.SMlog("The GC has already been scheduled to "
"re-start")
except util.CommandException as e:
if e.code != errno.ETIMEDOUT:
raise
util.SMlog('failed to abort the GC')
finally:
return
else:
util.SMlog("A GC instance already running, not kicking")
return
else:
lockRunning.release()
util.SMlog("Kicking GC")
cleanup.gc(self.session, self.uuid, True)
def daemonize(args, callback):
with DaemonContext():
create_process = False
lock = Lock(LOCKFILE, os.getpid(), args.name, args.sea_ep[0],
args.sea_ep[1], args.port)
if lock.is_locked():
lock_pid = lock.get_pid()
if not lock.is_same_file(args.name, args.sea_ep[0],
args.sea_ep[1]) \
or not is_process_running(lock_pid):
try:
os.kill(lock_pid, signal.SIGQUIT)
except OSError:
pass
except TypeError:
pass
lock.break_lock()
create_process = True
else:
create_process = True
if create_process:
lock.acquire()
callback(args.name, season=args.sea_ep[0], episode=args.sea_ep[1],
serve=True, port=args.port)
lock.release()
class TestLock(unittest.TestCase):
def setUp(self):
self.client = Redis()
self.client.flushdb()
self.lock = Lock(self.client, 'lock')
def test_only_one_lock_can_be_acquire(self):
self.assertTrue(
self.lock.acquire()
)
self.assertFalse(
self.lock.acquire()
)
def test_release_works(self):
self.lock.acquire()
self.assertTrue(
self.lock.release()
)
def test_release_return_false_when_lock_not_acquired(self):
self.assertFalse(
self.lock.release()
)
def main():
lock = Lock('/tmp/poll_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warn('Lock file {} acquired'.format(lock.lock_file))
url = properties.PASTA_BASE_URL + '/changes/eml?'
qm = QueueManager()
fromDate = None
dt = qm.get_last_datetime()
if dt is not None:
fromDate = datetime.strftime(dt, '%Y-%m-%dT%H:%M:%S.%f')
if fromDate is None:
bootstrap(url=url)
else:
parse(url=url, fromDate=fromDate)
lock.release()
logger.warn('Lock file {} released'.format(lock.lock_file))
return 0
def detachThin(session, lvmCache, srUuid, vdiUuid):
"""Shrink the VDI to the minimal size if no one is using it"""
lvName = LV_PREFIX[vhdutil.VDI_TYPE_VHD] + vdiUuid
path = os.path.join(VG_LOCATION, VG_PREFIX + srUuid, lvName)
lock = Lock(vhdutil.LOCK_TYPE_SR, srUuid)
_tryAcquire(lock)
vdiRef = session.xenapi.VDI.get_by_uuid(vdiUuid)
vbds = session.xenapi.VBD.get_all_records_where( \
"field \"VDI\" = \"%s\"" % vdiRef)
numPlugged = 0
for vbdRec in vbds.values():
if vbdRec["currently_attached"]:
numPlugged += 1
if numPlugged > 1:
raise util.SMException("%s still in use by %d others" % \
(vdiUuid, numPlugged - 1))
lvmCache.activate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
try:
newSize = calcSizeLV(vhdutil.getSizePhys(path))
deflate(lvmCache, lvName, newSize)
finally:
lvmCache.deactivate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
lock.release()
def daemonize(args, callback):
with DaemonContext():
from touchandgo.logger import log_set_up
log_set_up(True)
log = logging.getLogger('touchandgo.daemon')
log.info("running daemon")
create_process = False
lock = Lock(LOCKFILE, os.getpid(), args.name, args.sea_ep[0],
args.sea_ep[1], args.port)
if lock.is_locked():
log.debug("lock active")
lock_pid = lock.get_pid()
if not lock.is_same_file(args.name, args.sea_ep[0],
args.sea_ep[1]) \
or not is_process_running(lock_pid):
try:
log.debug("killing process %s" % lock_pid)
os.kill(lock_pid, signal.SIGQUIT)
except OSError:
pass
except TypeError:
pass
lock.break_lock()
create_process = True
else:
create_process = True
if create_process:
log.debug("creating proccess")
lock.acquire()
callback()
lock.release()
else:
log.debug("same daemon process")
def main(package_id, dryrun):
logger.info(f'package_id={package_id}')
lock = Lock('/tmp/poll_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warning('Lock file {} acquired'.format(lock.lock_file))
try:
scope, identifier, revision = package_id.split('.')
connection = connect()
event = get_package_info(connection, scope, identifier, revision)
if event:
qm = QueueManager()
msg = f"Enqueue: {event.package} - {event.datetime} - " + \
f"{event.owner} - {event.doi} - {event.method}"
logger.warning(msg)
if not dryrun:
qm.enqueue(event=event)
else:
msg = f"DRYRUN: qm.enqueue(event=event)"
logger.info(msg)
except AdapterRequestFailureException as e:
logger.error(e)
lock.release()
logger.warning('Lock file {} released'.format(lock.lock_file))
return 0
class Election:
def __init__(self, name, is_master_callback, lost_master_callback):
self.lock = Lock(name,
lock_callback=self._lock,
lock_lost_callback=self._lost_lock)
self.master_callback = is_master_callback
self.lost_master_callback = lost_master_callback
self.running = False
self.condition = threading.Condition()
def shutdown(self):
self.running = False
self.condition.acquire()
self.condition.notify()
self.condition.release()
def run(self):
self.running = True
while self.running:
self.lock.acquire()
self.condition.acquire()
self.condition.wait()
self.condition.release()
self.lock.release()
def _lock(self):
self.master_callback()
def _lost_lock(self):
self.lost_master_callback()
def detachThin(session, lvmCache, srUuid, vdiUuid):
"""Shrink the VDI to the minimal size if no one is using it"""
lvName = LV_PREFIX[vhdutil.VDI_TYPE_VHD] + vdiUuid
path = os.path.join(VG_LOCATION, VG_PREFIX + srUuid, lvName)
lock = Lock(vhdutil.LOCK_TYPE_SR, srUuid)
_tryAcquire(lock)
vdiRef = session.xenapi.VDI.get_by_uuid(vdiUuid)
vbds = session.xenapi.VBD.get_all_records_where( \
"field \"VDI\" = \"%s\"" % vdiRef)
numPlugged = 0
for vbdRec in vbds.values():
if vbdRec["currently_attached"]:
numPlugged += 1
if numPlugged > 1:
raise util.SMException("%s still in use by %d others" % \
(vdiUuid, numPlugged - 1))
lvmCache.activate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
try:
newSize = calcSizeLV(vhdutil.getSizePhys(path))
deflate(lvmCache, lvName, newSize)
finally:
lvmCache.deactivate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
lock.release()
def daemonize(args, callback):
with DaemonContext():
from touchandgo.logger import log_set_up
log_set_up(True)
log = logging.getLogger('touchandgo.daemon')
log.info("running daemon")
create_process = False
lock = Lock(LOCKFILE, os.getpid(), args.name, args.sea_ep[0],
args.sea_ep[1], args.port)
if lock.is_locked():
log.debug("lock active")
lock_pid = lock.get_pid()
if not lock.is_same_file(args.name, args.sea_ep[0],
args.sea_ep[1]) \
or not is_process_running(lock_pid):
try:
log.debug("killing process %s" % lock_pid)
os.kill(lock_pid, signal.SIGQUIT)
except OSError:
pass
except TypeError:
pass
lock.break_lock()
create_process = True
else:
create_process = True
if create_process:
log.debug("creating proccess")
lock.acquire()
callback()
lock.release()
else:
log.debug("same daemon process")
class PhysicalView(Base):
def __init__(self, dfs):
Base.__init__(self, dfs)
self.lock_ = Lock(dfs)
self.createBaseFolder()
def read(self, fileName, buf, offset, bufsize):
# TODO add thread safetly
filePath = os.path.join(self.getBasePath(), fileName)
size = self.getFileSize(fileName)
if offset + bufsize > size:
self.log_.w('tried to read ' + fileName + ' but size is ' + str(size) +
' and bufsize + offset = ' + str(offset + bufsize))
return err.InvalidBufferSize
self.lock_.acquire()
try:
f = open(filePath, "r")
except Exception, ex:
self.log_.e('error opening file in read mode ' + filePath + ': ' + str(ex))
self.lock_.release()
return err.FileNotFound
status = err.OK
f.seek(offset)
try:
data = f.read(bufsize)
for i, d in enumerate(data):
buf[i] = d
except Exception, ex:
self.log_.e('failed to read ' + filePath + ' from ' + str(offset) + ' to ' + str(offset + bufsize) + ': ' + str(ex))
status = err.CannotReadFile
def checkLocked(obj, ns):
"""Lock-protected access"""
lock = Lock(obj, ns)
lock.acquire()
try:
return RefCounter.check(obj, ns)
finally:
lock.release()
def checkLocked(obj, ns):
"""Lock-protected access"""
lock = Lock(obj, ns)
lock.acquire()
try:
return RefCounter.check(obj, ns)
finally:
lock.release()
def setReadonly(self, lvName, readonly):
path = self._getPath(lvName)
if self.lvs[lvName].readonly != readonly:
uuids = util.findall_uuid(path)
ns = lvhdutil.NS_PREFIX_LVM + uuids[0]
lock = Lock(uuids[1], ns)
lock.acquire()
lvutil.setReadonly(path, readonly)
lock.release()
self.lvs[lvName].readonly = readonly
def setReadonly(self, lvName, readonly):
path = self._getPath(lvName)
if self.lvs[lvName].readonly != readonly:
uuids = util.findall_uuid(path)
ns = lvhdutil.NS_PREFIX_LVM + uuids[0]
lock = Lock(uuids[1], ns)
lock.acquire()
lvutil.setReadonly(path, readonly)
lock.release()
self.lvs[lvName].readonly = readonly
def setReadonly(self, lvName, readonly):
path = self._getPath(lvName)
if self.lvs[lvName].readonly != readonly:
uuids = util.findall_uuid(path)
ns = lvhdutil.NS_PREFIX_LVM + uuids[0]
# Taking this lock is needed to avoid a race condition
# with tap-ctl open (which is now taking the same lock)
lock = Lock("lvchange-p", ns)
lock.acquire()
lvutil.setReadonly(path, readonly)
lock.release()
self.lvs[lvName].readonly = readonly
def setReadonly(self, lvName, readonly):
path = self._getPath(lvName)
if self.lvs[lvName].readonly != readonly:
uuids = util.findall_uuid(path)
ns = lvhdutil.NS_PREFIX_LVM + uuids[0]
# Taking this lock is needed to avoid a race condition
# with tap-ctl open (which is now taking the same lock)
lock = Lock("lvchange-p", ns)
lock.acquire()
lvutil.setReadonly(path, readonly)
lock.release()
self.lvs[lvName].readonly = readonly
def activate(self, ns, ref, lvName, binary):
lock = Lock(ref, ns)
lock.acquire()
try:
count = RefCounter.get(ref, binary, ns)
if count == 1:
try:
self.activateNoRefcount(lvName)
except util.CommandException:
RefCounter.put(ref, binary, ns)
raise
finally:
lock.release()
def activate(self, ns, ref, lvName, binary):
lock = Lock(ref, ns)
lock.acquire()
try:
count = RefCounter.get(ref, binary, ns)
if count == 1:
try:
self.activateNoRefcount(lvName)
except util.CommandException:
RefCounter.put(ref, binary, ns)
raise
finally:
lock.release()
def deactivate(self, sr_uuid, vdi_uuid):
"""Deactivate VDI - called post tapdisk close"""
if self._get_blocktracking_status():
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
logpath = self._get_cbt_logpath(vdi_uuid)
logname = self._get_cbt_logname(vdi_uuid)
self._cbt_op(vdi_uuid, cbtutil.set_cbt_consistency, logpath, True)
# Finally deactivate log file
self._deactivate_cbt_log(logname)
finally:
lock.release()
def _cbt_op(self, uuid, func, *args):
# Lock cbtlog operations
from lock import Lock
lock = Lock("cbtlog", str(uuid))
lock.acquire()
try:
logname = self._get_cbt_logname(uuid)
activated = self._activate_cbt_log(logname)
ret = func(*args)
if activated:
self._deactivate_cbt_log(logname)
return ret
finally:
lock.release()
def _cbt_op(self, uuid, func, *args):
# Lock cbtlog operations
from lock import Lock
lock = Lock("cbtlog", str(uuid))
lock.acquire()
try:
logname = self._get_cbt_logname(uuid)
activated = self._activate_cbt_log(logname)
ret = func(*args)
if activated:
self._deactivate_cbt_log(logname)
return ret
finally:
lock.release()
def deactivateVdi(sr_uuid, vdi_uuid, vhd_path):
name_space = lvhdutil.NS_PREFIX_LVM + sr_uuid
lock = Lock(vdi_uuid, name_space)
lock.acquire()
try:
count = RefCounter.put(vdi_uuid, False, name_space)
if count > 0:
return
try:
lvutil.deactivateNoRefcount(vhd_path)
except Exception as e:
util.SMlog(" lv de-activate failed for %s with error %s" %
(vhd_path, str(e)))
RefCounter.get(vdi_uuid, False, name_space)
finally:
lock.release()
def deactivate(self, sr_uuid, vdi_uuid):
"""Deactivate VDI - called post tapdisk close"""
if self._get_blocktracking_status():
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
logpath = self._get_cbt_logpath(vdi_uuid)
logname = self._get_cbt_logname(vdi_uuid)
self._cbt_op(vdi_uuid, cbtutil.set_cbt_consistency, logpath,
True)
# Finally deactivate log file
self._deactivate_cbt_log(logname)
finally:
lock.release()
def main():
lock = Lock()
try:
lock.acquire()
except LockError as exc:
error("Couldn't create lock file: %s" % exc)
lock = None
else:
try:
gainer = Gainer(_get_options())
gainer.process()
except KeyboardInterrupt:
error("Interrupted by user.")
finally:
if lock:
lock.release()
def deactivateVdi(sr_uuid, vdi_uuid, vhd_path):
name_space = lvhdutil.NS_PREFIX_LVM + sr_uuid
lock = Lock(vdi_uuid, name_space)
lock.acquire()
try:
count = RefCounter.put(vdi_uuid, False, name_space)
if count > 0:
return
try:
lvutil.deactivateNoRefcount(vhd_path)
except Exception, e:
util.SMlog(" lv de-activate failed for %s with error %s" %
(vhd_path, str(e)))
RefCounter.get(vdi_uuid, False, name_space)
finally:
lock.release()
class LogicalView(Base):
def __init__(self, dfs):
Base.__init__(self, dfs)
self.lock_ = Lock(dfs)
self.fileList_ = {}
def beginLocalUpdate(self, fileName):
file = self.getFile(fileName)
if file.latestVersion != file.localVersion:
file.localVersion = file.latestVersion.copy()
file.ownNoChunks()
def add(self, fileName, fileSize):
self.lock_.acquire()
f = File(fileName, 1, fileSize, self.dfs_.id.str)
self.fileList_[fileName] = f
self.lock_.release()
def delete(self, fileName):
self.lock_.acquire()
self.fileList_[fileName].isDeleted = True
self.lock_.release()
def exists(self, fileName):
return (fileName in self.fileList_)
def getLocalVersion(self, fileName):
return self.fileList_[fileName].getLocalVersion()
def getLatestVersion(self, fileName):
return self.fileList_[fileName].getLatestVersion()
def setNewVersion(self, fileName, version):
self.fileList_[fileName].setNewVersion(version.copy())
def setLocalVersion(self, fileName, numEdits, fileSize, lastEdited):
self.fileList_[fileName].setLocalVersion(numEdits, fileSize, lastEdited)
def getFileList(self):
return self.fileList_.values()
def getState(self):
return self.fileList_
def getFile(self, fileName):
return self.fileList_[fileName]
def main():
lock = Lock('/tmp/package_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warning('Lock file {} acquired'.format(lock.lock_file))
qm = QueueManager()
head = qm.get_head()
while head is not None:
logger.warning('Active package: {p}'.format(p=head.package))
skip = False
if properties.CHECK_PRE_EXISTENCE_IN_GMN and head.method in [properties.CREATE, properties.UPDATE]:
skip = gmn_exists(properties.PASTA_BASE_URL + 'metadata/eml/' + head.package.replace('.', '/'))
if skip:
logger.warning('Package already exists: {}. Skipping {}.'.format(head.package, head.method))
else:
p = Package(head)
if p.public:
logger.warning('Processing: {p}'.format(p=p.package))
resource = p.resources[properties.METADATA]
if p.method == properties.CREATE:
process_create_package(package=p)
elif p.method == properties.UPDATE:
process_update_package(package=p, queue_manager=qm)
elif p.method == properties.DELETE:
process_archive_package(package=p)
else:
msg = 'Unrecognized package event "{event}" for' \
'package: {package}'.format(event=p.method,
package=p.package)
raise(AdapterIncompleteStateException(msg))
else:
logger.warning('Package not public: {p}'.format(p=p.package))
qm.dequeue(package=head.package, method=head.method)
if properties.SLEEP_BETWEEN_PACKAGES:
time.sleep(int(properties.SLEEP_BETWEEN_PACKAGES))
head = qm.get_head()
logger.warning('Queue empty')
lock.release()
logger.warning('Lock file {} released'.format(lock.lock_file))
return 0
def _kickGC(self):
# don't bother if an instance already running (this is just an
# optimization to reduce the overhead of forking a new process if we
# don't have to, but the process will check the lock anyways)
lockRunning = Lock(cleanup.LOCK_TYPE_RUNNING, self.uuid)
if not lockRunning.acquireNoblock():
if cleanup.should_preempt(self.session, self.uuid):
util.SMlog("Aborting currently-running coalesce of garbage VDI")
cleanup.abort(self.uuid)
else:
util.SMlog("A GC instance already running, not kicking")
return
else:
lockRunning.release()
util.SMlog("Kicking GC")
cleanup.gc(self.session, self.uuid, True)
def attachThin(journaler, srUuid, vdiUuid):
"""Ensure that the VDI LV is expanded to the fully-allocated size"""
lvName = LV_PREFIX[VDI_TYPE_VHD] + vdiUuid
vgName = VG_PREFIX + srUuid
lock = Lock(vhdutil.LOCK_TYPE_SR, srUuid)
lvmCache = journaler.lvmCache
lock.acquire()
vhdInfo = vhdutil.getVHDInfoLVM(lvName, extractUuid, vgName)
newSize = calcSizeVHDLV(vhdInfo.sizeVirt)
currSizeLV = lvmCache.getSize(lvName)
if newSize <= currSizeLV:
return
lvmCache.activate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
try:
inflate(journaler, srUuid, vdiUuid, newSize)
finally:
lvmCache.deactivate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
lock.release()
def _kickGC(self):
# don't bother if an instance already running (this is just an
# optimization to reduce the overhead of forking a new process if we
# don't have to, but the process will check the lock anyways)
lockRunning = Lock(cleanup.LOCK_TYPE_RUNNING, self.uuid)
if not lockRunning.acquireNoblock():
if cleanup.should_preempt(self.session, self.uuid):
util.SMlog(
"Aborting currently-running coalesce of garbage VDI")
cleanup.abort(self.uuid)
else:
util.SMlog("A GC instance already running, not kicking")
return
else:
lockRunning.release()
util.SMlog("Kicking GC")
cleanup.gc(self.session, self.uuid, True)
def activate(self, sr_uuid, vdi_uuid):
"""Activate VDI - called pre tapdisk open"""
if self._get_blocktracking_status():
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
logpath = self._get_cbt_logpath(vdi_uuid)
logname = self._get_cbt_logname(vdi_uuid)
# Activate CBT log file, if required
self._activate_cbt_log(logname)
finally:
lock.release()
# Check and update consistency
consistent = self._cbt_op(vdi_uuid, cbtutil.get_cbt_consistency,
logpath)
if not consistent:
lock.acquire()
try:
self._delete_cbt_log()
finally:
lock.release()
vdi_ref = self.sr.srcmd.params['vdi_ref']
self.sr.session.xenapi.VDI.set_cbt_enabled(vdi_ref, False)
alert_name = "VDI_CBT_METADATA_INCONSISTENT"
alert_prio_warning = "3"
alert_obj = "VDI"
alert_uuid = str(vdi_uuid)
alert_str = ("Changed Block Tracking metadata is inconsistent"
" for disk %s." % vdi_uuid)
util.SMlog(alert_str)
self.sr.session.xenapi.message.create(alert_name,
alert_prio_warning,
alert_obj, alert_uuid,
alert_str)
return None
self._cbt_op(self.uuid, cbtutil.set_cbt_consistency, logpath,
False)
return {'cbtlog': logpath}
return None
def attachThin(journaler, srUuid, vdiUuid):
"""Ensure that the VDI LV is expanded to the fully-allocated size"""
lvName = LV_PREFIX[vhdutil.VDI_TYPE_VHD] + vdiUuid
vgName = VG_PREFIX + srUuid
lock = Lock(vhdutil.LOCK_TYPE_SR, srUuid)
lvmCache = journaler.lvmCache
_tryAcquire(lock)
lvmCache.refresh()
vhdInfo = vhdutil.getVHDInfoLVM(lvName, extractUuid, vgName)
newSize = calcSizeVHDLV(vhdInfo.sizeVirt)
currSizeLV = lvmCache.getSize(lvName)
if newSize <= currSizeLV:
return
lvmCache.activate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
try:
inflate(journaler, srUuid, vdiUuid, newSize)
finally:
lvmCache.deactivate(NS_PREFIX_LVM + srUuid, vdiUuid, lvName, False)
lock.release()
def deactivate(self, ns, ref, lvName, binary):
lock = Lock(ref, ns)
lock.acquire()
try:
count = RefCounter.put(ref, binary, ns)
if count > 0:
return
refreshed = False
while True:
lvInfo = self.getLVInfo(lvName)
if len(lvInfo) != 1:
raise util.SMException("LV info not found for %s" % ref)
info = lvInfo[lvName]
if info.open:
if refreshed:
# should never happen in normal conditions but in some
# failure cases the recovery code may not be able to
# determine what the correct refcount should be, so it
# is not unthinkable that the value might be out of
# sync
util.SMlog("WARNING: deactivate: LV %s open" % lvName)
return
# check again in case the cached value is stale
self.refresh()
refreshed = True
else:
break
try:
self.deactivateNoRefcount(lvName)
except util.CommandException:
self.refresh()
if self.getLVInfo(lvName):
util.SMlog("LV %s could not be deactivated" % lvName)
if lvInfo[lvName].active:
util.SMlog("Reverting the refcount change")
RefCounter.get(ref, binary, ns)
raise
else:
util.SMlog("LV %s not found" % lvName)
finally:
lock.release()
def deactivate(self, ns, ref, lvName, binary):
lock = Lock(ref, ns)
lock.acquire()
try:
count = RefCounter.put(ref, binary, ns)
if count > 0:
return
refreshed = False
while True:
lvInfo = self.getLVInfo(lvName)
if len(lvInfo) != 1:
raise util.SMException("LV info not found for %s" % ref)
info = lvInfo[lvName]
if info.open:
if refreshed:
# should never happen in normal conditions but in some
# failure cases the recovery code may not be able to
# determine what the correct refcount should be, so it
# is not unthinkable that the value might be out of
# sync
util.SMlog("WARNING: deactivate: LV %s open" % lvName)
return
# check again in case the cached value is stale
self.refresh()
refreshed = True
else:
break
try:
self.deactivateNoRefcount(lvName)
except util.CommandException:
self.refresh()
if self.getLVInfo(lvName):
util.SMlog("LV %s could not be deactivated" % lvName)
if lvInfo[lvName].active:
util.SMlog("Reverting the refcount change")
RefCounter.get(ref, binary, ns)
raise
else:
util.SMlog("LV %s not found" % lvName)
finally:
lock.release()
def lock(self, lockid, blocking=True, timeout=LOCK_TIMEOUT):
# with self.__locks_lock:
lock = Lock(self.client, self.lock_path_prefix + lockid)
try:
acquired = lock.acquire(blocking=blocking, timeout=timeout)
logger.debug('Lock {0} acquired: {1}'.format(lockid, acquired))
if not acquired:
raise LockFailedError(lock_id=lockid)
yield
except LockTimeout:
logger.info('Failed to acquire lock {0} due to timeout '
'({1} seconds)'.format(lockid, timeout))
raise LockFailedError(lock_id=lockid)
except LockFailedError:
raise
except Exception as e:
logger.error('Failed to acquire lock {0}: {1}\n{2}'.format(
lockid, e, traceback.format_exc()))
raise
finally:
lock.release()
def lock(self, lockid, blocking=True, timeout=LOCK_TIMEOUT):
lock = Lock(self.client, self.lock_path_prefix + lockid)
try:
acquired = lock.acquire(blocking=blocking, timeout=timeout)
logger.debug('Lock {0} acquired: {1}'.format(lockid, acquired))
if not acquired:
# TODO: Change exception time or set all required parameters for
# this type of exception
raise LockAlreadyAcquiredError(lock_id=lockid)
yield
except LockTimeout:
logger.info('Failed to acquire lock {} due to timeout ({} seconds)'.format(
lockid, timeout))
raise LockFailedError(lock_id=lockid)
except LockAlreadyAcquiredError:
raise
except LockError as e:
logger.error('Failed to acquire lock {0}: {1}\n{2}'.format(
lockid, e, traceback.format_exc()))
raise
finally:
lock.release()
def main():
lock = Lock('/tmp/package_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warn('Lock file {} acquired'.format(lock.lock_file))
qm = QueueManager()
head = qm.get_head()
while head is not None:
logger.warn('Active package: {p}'.format(p=head.package))
p = Package(head)
if p.public:
logger.warn('Processing: {p}'.format(p=p.package))
if p.method == properties.CREATE:
process_create_package(package=p)
elif p.method == properties.UPDATE:
process_update_package(package=p, queue_manager=qm)
elif p.method == properties.DELETE:
process_archive_package(package=p)
else:
msg = 'Unrecognized package event "{event}" for' \
'package: {package}'.format(event=p.method,
package=p.package)
raise (AdapterIncompleteStateException(msg))
else:
logger.warn('Package not public: {p}'.format(p=p.package))
qm.dequeue(package=p.package, method=p.method)
head = qm.get_head()
logger.warn('Queue empty')
lock.release()
logger.warn('Lock file {} released'.format(lock.lock_file))
return 0
class TestLock(unittest.TestCase):
def setUp(self):
self.lock = Lock('bozo.lock')
def tearDown(self):
try:
self.lock.release()
except IOError as e:
logger.error(e)
def test_acquire_release(self):
print(self.lock.acquire())
try:
self.lock.release()
except IOError as e:
logger.error(e)
self.fail()
self.lock.acquire()
def test_locked(self):
self.lock.acquire()
self.assertTrue(self.lock.locked)
def lock(self, lockid, blocking=True, timeout=LOCK_TIMEOUT):
lock = Lock(self.client, self.lock_path_prefix + lockid)
try:
acquired = lock.acquire(blocking=blocking, timeout=timeout)
logger.debug('Lock {0} acquired: {1}'.format(lockid, acquired))
if not acquired:
# TODO: Change exception time or set all required parameters for
# this type of exception
raise LockAlreadyAcquiredError(lock_id=lockid)
yield
except LockTimeout:
logger.info(
'Failed to acquire lock {} due to timeout ({} seconds)'.format(
lockid, timeout))
raise LockFailedError(lock_id=lockid)
except LockAlreadyAcquiredError:
raise
except LockError as e:
logger.error('Failed to acquire lock {0}: {1}\n{2}'.format(
lockid, e, traceback.format_exc()))
raise
finally:
lock.release()
def test():
# Create a Lock
lock = Lock("test")
# Should not be yet held.
assert lock.held() == False
# Go get it
lock.acquire()
# Second lock shall throw in debug mode.
try:
lock.acquire()
except AssertionError as e:
if str(e) != flock.WriteLock.ERROR_ISLOCKED:
raise
else:
raise AssertionError("Reaquired a locked lock")
lock.release()
Lock.cleanup()
def main():
lock = Lock('/tmp/poll_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warning('Lock file {} acquired'.format(lock.lock_file))
url = properties.PASTA_BASE_URL + 'changes/eml?'
qm = QueueManager()
# queue fromDate (fallback): effective but not efficient
fromDate = qm.get_last_datetime()
logger.info(f'"fromDate" from QueueManager: {fromDate}')
if fromDate is None: # Empty adapter_queue database
bootstrap(url=url)
else:
fromDate = pendulum.instance(dt=fromDate, tz='US/Mountain')
last_query_date = adapter_utilities.get_last_query_date()
if last_query_date is not None:
# pickled fromDate: effective and efficient
fromDate = last_query_date.in_tz('US/Mountain')
logger.info(f'"fromDate" from adapter_utilities: {fromDate}')
try:
query_date = pendulum.now(tz='UTC')
parse(url=url, fromDate=fromDate, scope=properties.SCOPE)
adapter_utilities.save_last_query_date(query_date)
except AdapterRequestFailureException as e:
logger.error(e)
lock.release()
logger.warning('Lock file {} released'.format(lock.lock_file))
return 0
def activate(self, sr_uuid, vdi_uuid):
"""Activate VDI - called pre tapdisk open"""
if self._get_blocktracking_status():
if self.sr.srcmd.params.has_key('args'):
read_write = self.sr.srcmd.params['args'][0]
if read_write == "false":
# Disk is being attached in RO mode,
# don't attach metadata log file
return None
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
logpath = self._get_cbt_logpath(vdi_uuid)
logname = self._get_cbt_logname(vdi_uuid)
# Activate CBT log file, if required
self._activate_cbt_log(logname)
finally:
lock.release()
# Check and update consistency
consistent = self._cbt_op(vdi_uuid, cbtutil.get_cbt_consistency,
logpath)
if not consistent:
alert_name = "VDI_CBT_METADATA_INCONSISTENT"
alert_str = ("Changed Block Tracking metadata is inconsistent"
" for disk %s." % vdi_uuid)
self._disable_cbt_on_error(alert_name, alert_str)
return None
self._cbt_op(self.uuid, cbtutil.set_cbt_consistency, logpath,
False)
return {'cbtlog': logpath}
return None
def activate(self, sr_uuid, vdi_uuid):
"""Activate VDI - called pre tapdisk open"""
if self._get_blocktracking_status():
if self.sr.srcmd.params.has_key('args'):
read_write = self.sr.srcmd.params['args'][0]
if read_write == "false":
# Disk is being attached in RO mode,
# don't attach metadata log file
return None
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
logpath = self._get_cbt_logpath(vdi_uuid)
logname = self._get_cbt_logname(vdi_uuid)
# Activate CBT log file, if required
self._activate_cbt_log(logname)
finally:
lock.release()
# Check and update consistency
consistent = self._cbt_op(vdi_uuid, cbtutil.get_cbt_consistency,
logpath)
if not consistent:
alert_name = "VDI_CBT_METADATA_INCONSISTENT"
alert_str = ("Changed Block Tracking metadata is inconsistent"
" for disk %s." % vdi_uuid)
self._disable_cbt_on_error(alert_name, alert_str)
return None
self._cbt_op(self.uuid, cbtutil.set_cbt_consistency,
logpath, False)
return {'cbtlog': logpath}
return None
def process_repositories(self, repo_configs, ref, action, request_body):
import os
import time
import logging
from wrappers import GitWrapper
from lock import Lock
import json
logger = logging.getLogger()
data = json.loads(request_body)
# Process each matching repository
for repo_config in repo_configs:
try:
# Verify that all filters matches the request (if any filters are specified)
if 'filters' in repo_config:
# at least one filter must match
for filter in repo_config['filters']:
# all options specified in the filter must match
for filter_key, filter_value in filter.iteritems():
# support for earlier version so it's non-breaking functionality
if filter_key == 'action' and filter_value == action:
continue
if filter_key not in data or filter_value != data[filter_key]:
raise FilterMatchError()
except FilterMatchError as e:
# Filter does not match, do not process this repo config
continue
# In case there is no path configured for the repository, no pull will
# be made.
if not 'path' in repo_config:
GitWrapper.deploy(repo_config)
continue
running_lock = Lock(os.path.join(repo_config['path'], 'status_running'))
waiting_lock = Lock(os.path.join(repo_config['path'], 'status_waiting'))
try:
# Attempt to obtain the status_running lock
while not running_lock.obtain():
# If we're unable, try once to obtain the status_waiting lock
if not waiting_lock.has_lock() and not waiting_lock.obtain():
logger.error("Unable to obtain the status_running lock nor the status_waiting lock. Another process is " +
"already waiting, so we'll ignore the request.")
# If we're unable to obtain the waiting lock, ignore the request
break
# Keep on attempting to obtain the status_running lock until we succeed
time.sleep(5)
n = 4
while 0 < n and 0 != GitWrapper.pull(repo_config):
n -= 1
if 0 < n:
GitWrapper.deploy(repo_config)
except Exception as e:
logger.error('Error during \'pull\' or \'deploy\' operation on path: %s' % repo_config['path'])
logger.error(e)
finally:
# Release the lock if it's ours
if running_lock.has_lock():
running_lock.release()
# Release the lock if it's ours
if waiting_lock.has_lock():
waiting_lock.release()
def clientthread(conn, client_address):
#infinite loop so that function do not terminate and thread do not end.
while True:
#Receiving from client
try:
data = conn.recv(RECV_BUFFER) #release or lock resource_name
except:
#release all locked resources for the client address
client_lock = Lock("", client_address)
client_lock.release_by_client_address()
break
else:
#split the message Received to get the command and the resource name
tmp = data.split()
# check if the message Received follows the correct format
if tmp is not None and len(tmp) == 2 and (tmp[0] == "release"
or tmp[0] == "lock"):
# operation has to be lock or release
operation = tmp[0]
resource_name = tmp[1]
#targeted resource name and status
client_lock = Lock(resource_name, client_address)
resource_status = client_lock.check_status()
if not data:
break
elif resource_status is None:
# targeted resource is not listed in the database
reply = "required resource is not listed"
elif operation == 'lock' and resource_status == "free":
# you gain access to the resource
reply = "You have an exclusive access to resource " + resource_name
client_lock.acquire()
elif operation == 'lock' and resource_status == "busy":
# the resource is busy, you will wait TIMEOUT secs and retry
time.sleep(TIMEOUT)
# checking the resource status again after TIMEOUT
resource_status_new = client_lock.check_status()
if resource_status_new == "busy":
# it is still busy, try again later
reply = "required resource is busy now, you have to wait a while"
else:
#it's free now, you will gain access
reply = "You have an exclusive access to resource " + resource_name
client_lock.acquire()
elif operation == 'release':
if resource_status == "free":
#trying to release a free resource, not allowed
reply = "resource is already free."
#checking if the client who request release is the same client who lock it in the first place
elif client_address in client_lock.get_client_address():
reply = "lock released from resource " + resource_name
client_lock.release()
else:
# trying to release someone else's resource, not allowed
reply = 'it is not allowed to release someone else resource'
else:
#the message Received does not follow the correct format so send an error message
reply = "wrong message, you must send release or lock as the first word then space then the resource_name"
#send the message to the client
conn.sendall(reply)
#came out of loop and close the connection
conn.close()
def process_repositories(self, repo_configs, ref, action, request_body):
"""Verify that the suggested repositories has matching settings and
issue git pull and/or deploy commands."""
import os
import time
import logging
from wrappers import GitWrapper
from lock import Lock
import json
logger = logging.getLogger()
data = json.loads(request_body)
result = []
# Process each matching repository
for repo_config in repo_configs:
repo_result = {}
try:
# Verify that all filters matches the request (if any filters are specified)
if 'filters' in repo_config:
# At least one filter must match
for filter in repo_config['filters']:
# All options specified in the filter must match
for filter_key, filter_value in filter.iteritems():
# Ignore filters with value None (let them pass)
if filter_value == None:
continue
# Support for earlier version so it's non-breaking functionality
if filter_key == 'action' and filter_value == action:
continue
# Interpret dots in filter name as path notations
node_value = data
for node_key in filter_key.split('.'):
# If the path is not valid the filter does not match
if not node_key in node_value:
logger.info("Filter '%s'' does not match since the path is invalid" % (filter_key))
raise FilterMatchError()
node_value = node_value[node_key]
if filter_value == node_value:
continue
# If the filter value is set to True. the filter
# will pass regardless of the actual value
if filter_value == True:
continue
logger.info("Filter '%s'' does not match ('%s' != '%s')" % (filter_key, filter_value, (str(node_value)[:75] + '..') if len(str(node_value)) > 75 else str(node_value)))
raise FilterMatchError()
except FilterMatchError as e:
# Filter does not match, do not process this repo config
continue
# In case there is no path configured for the repository, no pull will
# be made.
if not 'path' in repo_config:
res = GitWrapper.deploy(repo_config)
repo_result['deploy'] = res
result.append(repo_result)
continue
running_lock = Lock(os.path.join(repo_config['path'], 'status_running'))
waiting_lock = Lock(os.path.join(repo_config['path'], 'status_waiting'))
try:
# Attempt to obtain the status_running lock
while not running_lock.obtain():
# If we're unable, try once to obtain the status_waiting lock
if not waiting_lock.has_lock() and not waiting_lock.obtain():
logger.error("Unable to obtain the status_running lock nor the status_waiting lock. Another process is " +
"already waiting, so we'll ignore the request.")
# If we're unable to obtain the waiting lock, ignore the request
break
# Keep on attempting to obtain the status_running lock until we succeed
time.sleep(5)
n = 4
res = None
while n > 0:
# Attempt to pull up a maximum of 4 times
print repo_config
if not repo_config.get('branch'):
if data.get('ref_type') == "tag":
repo_config.update(
{'tag': data.get('ref')})
elif '/' in data.get('ref', ''):
repo_config.update(
{'branch': data.get('ref').split('/')[-1]})
elif data.get('pull_request'):
repo_config.update(
{'branch': data.get(
'pull_request').get('base').get('ref')})
print repo_config
res = GitWrapper.pull(repo_config)
repo_result['git pull'] = res
print 'pull done!'
# Return code indicating success?
if res == 0:
break
n -= 1
if 0 < n:
res = GitWrapper.deploy(repo_config)
repo_result['deploy'] = res
except Exception as e:
logger.error('Error during \'pull\' or \'deploy\' operation on path: %s' % repo_config['path'])
logger.error(e.message)
finally:
# Release the lock if it's ours
if running_lock.has_lock():
running_lock.release()
# Release the lock if it's ours
if waiting_lock.has_lock():
waiting_lock.release()
result.append(repo_result)
return result
class Logger(object):
def __init__(self, dry=False, name="scapy-tgen", logs_dir=None):
self.dry = dry
self.dbg = 1
self.log_file = None
self.lock = Lock()
self.fmt = LogFormatter()
self.logger = logging.getLogger(name)
stdlog = logging.StreamHandler(sys.stdout)
stdlog.setLevel(logging.ERROR if not self.dry else logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR if not self.dry else logging.DEBUG)
ch.setFormatter(self.fmt)
self.logger.addHandler(ch)
self.logger.removeHandler(stdlog)
self.logger.propagate = False
self.log_file = None
self.file_handler = None
if logs_dir:
self.logs_dir = logs_dir
else:
self.logs_dir = os.getenv("SCAPY_TGEN_LOGS_PATH", "server")
self.set_log_file(None)
@staticmethod
def ensure_parent(filename):
path = os.path.dirname(filename)
path = os.path.abspath(path)
if not os.path.exists(path):
os.makedirs(path)
def set_node_name(self, name):
if self.fmt:
self.fmt.node_name = name
def set_log_file(self, log_file):
if self.file_handler:
self.logger.removeHandler(self.file_handler)
self.file_handler = None
if log_file:
if self.logs_dir:
log_file = os.path.join(self.logs_dir, log_file)
self.ensure_parent(log_file)
self.file_handler = logging.FileHandler(log_file)
self.file_handler.setFormatter(self.fmt)
self.file_handler.setLevel(logging.DEBUG)
self.logger.addHandler(self.file_handler)
self.log_file = log_file
def get_log(self, filename=None):
if not filename:
filename = self.log_file
elif not os.path.exists(filename) and self.logs_dir:
filename = os.path.join(self.logs_dir, filename)
if not filename:
return ""
try:
fh = open(filename, 'r')
data = fh.readlines()
fh.close()
data = map(str.strip, data)
return "\n".join(data)
except Exception:
return ""
def todo(self, etype, name, value):
msg = "{}: {} = {}".format(etype, name, value)
self.error(msg)
raise ValueError(msg)
def debug(self, *args, **kwargs):
msg = " ".join(map(str, args))
self._log(logging.DEBUG, msg)
def _log(self, lvl, msg):
if not msg.strip(): return
self.lock.acquire()
for line in msg.split("\n"):
self.logger.log(lvl, line)
self.lock.release()
def log(self, *args, **kwargs):
msg = " ".join(map(str, args))
self._log(logging.INFO, msg)
def info(self, *args, **kwargs):
msg = " ".join(map(str, args))
self._log(logging.INFO, msg)
def error(self, *args, **kwargs):
msg = ""
#msg = msg + "=================================== "
msg = msg + " ".join(map(str, args))
#msg = msg + "=================================== "
self._log(logging.ERROR, msg)
def log_exception(self, e, msg):
self.logger.error("=========== exception ==================")
self.logger.error(msg)
self.logger.error("=========== exception ==================")
self.logger.error(msg, traceback.format_exc())
self.lock.acquire()
self.logger.exception(e)
self.lock.release()
@staticmethod
def setup():
logging.basicConfig()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
def delete(self, sr_uuid, vdi_uuid, data_only=False):
"""Delete this VDI.
This operation IS idempotent and should succeed if the VDI
exists and can be deleted or if the VDI does not exist. It is
the responsibility of the higher-level management tool to
ensure that the detach() operation has been explicitly called
prior to deletion, otherwise the delete() will fail if the
disk is still attached.
"""
import blktap2
from lock import Lock
if data_only == False and self._get_blocktracking_status():
logpath = self._get_cbt_logpath(vdi_uuid)
parent_uuid = self._cbt_op(vdi_uuid, cbtutil.get_cbt_parent,
logpath)
parent_path = self._get_cbt_logpath(parent_uuid)
child_uuid = self._cbt_op(vdi_uuid, cbtutil.get_cbt_child, logpath)
child_path = self._get_cbt_logpath(child_uuid)
lock = Lock("cbtlog", str(vdi_uuid))
if self._cbt_log_exists(parent_path):
self._cbt_op(parent_uuid, cbtutil.set_cbt_child,
parent_path, child_uuid)
if self._cbt_log_exists(child_path):
self._cbt_op(child_uuid, cbtutil.set_cbt_parent,
child_path, parent_uuid)
lock.acquire()
try:
# Coalesce contents of bitmap with child's bitmap
# Check if child bitmap is currently attached
paused_for_coalesce = False
consistent = self._cbt_op(child_uuid,
cbtutil.get_cbt_consistency,
child_path)
if not consistent:
if not blktap2.VDI.tap_pause(self.session,
sr_uuid, child_uuid):
raise util.SMException("failed to pause VDI %s")
paused_for_coalesce = True
self._activate_cbt_log(self._get_cbt_logname(vdi_uuid))
self._cbt_op(child_uuid, cbtutil.coalesce_bitmap,
logpath, child_path)
lock.release()
except util.CommandException:
# If there is an exception in coalescing,
# CBT log file is not deleted and pointers are reset
# to what they were
util.SMlog("Exception in coalescing bitmaps on VDI delete,"
" restoring to previous state")
try:
if self._cbt_log_exists(parent_path):
self._cbt_op(parent_uuid, cbtutil.set_cbt_child,
parent_path, vdi_uuid)
if self._cbt_log_exists(child_path):
self._cbt_op(child_uuid, cbtutil.set_cbt_parent,
child_path, vdi_uuid)
finally:
lock.release()
lock.cleanup("cbtlog", str(vdi_uuid))
return
finally:
# Unpause tapdisk if it wasn't originally paused
if paused_for_coalesce:
blktap2.VDI.tap_unpause(self.session, sr_uuid,
child_uuid)
lock.acquire()
try:
self._delete_cbt_log()
finally:
lock.release()
lock.cleanup("cbtlog", str(vdi_uuid))
def configure_blocktracking(self, sr_uuid, vdi_uuid, enable):
"""Function for configuring blocktracking"""
import blktap2
vdi_ref = self.sr.srcmd.params['vdi_ref']
# Check if raw VDI or snapshot
if self.vdi_type == vhdutil.VDI_TYPE_RAW or \
self.session.xenapi.VDI.get_is_a_snapshot(vdi_ref):
raise xs_errors.XenError('VDIType',
opterr='Raw VDI or snapshot not permitted')
# Check if already enabled
if self._get_blocktracking_status() == enable:
return
# Save disk state before pause
disk_state = blktap2.VDI.tap_status(self.session, vdi_uuid)
if not blktap2.VDI.tap_pause(self.session, sr_uuid, vdi_uuid):
error = "Failed to pause VDI %s" % vdi_uuid
raise xs_errors.XenError('CBTActivateFailed', opterr=error)
logfile = None
try:
if enable:
try:
# Check available space
self._ensure_cbt_space()
logfile = self._create_cbt_log()
# Set consistency
if disk_state:
util.SMlog("Setting consistency of cbtlog file to False for VDI: %s"
% self.uuid)
logpath = self._get_cbt_logpath(self.uuid)
self._cbt_op(self.uuid, cbtutil.set_cbt_consistency,
logpath, False)
except Exception as error:
self._delete_cbt_log()
raise xs_errors.XenError('CBTActivateFailed',
opterr=str(error))
else:
from lock import Lock
lock = Lock("cbtlog", str(vdi_uuid))
lock.acquire()
try:
# Find parent of leaf metadata file, if any,
# and nullify its successor
logpath = self._get_cbt_logpath(self.uuid)
parent = self._cbt_op(self.uuid,
cbtutil.get_cbt_parent, logpath)
self._delete_cbt_log()
parent_path = self._get_cbt_logpath(parent)
if self._cbt_log_exists(parent_path):
self._cbt_op(parent, cbtutil.set_cbt_child,
parent_path, uuid.UUID(int=0))
except Exception as error:
raise xs_errors.XenError('CBTDeactivateFailed', str(error))
finally:
lock.release()
lock.cleanup("cbtlog", str(vdi_uuid))
finally:
blktap2.VDI.tap_unpause(self.session, sr_uuid, vdi_uuid)
return pathsNotInUse
if __name__ == "__main__":
import lvutil
# used by the master changeover script
cmd = sys.argv[1]
if cmd == "fixrefcounts":
from lvmcache import LVMCache
srUuid = sys.argv[2]
try:
vgName = VG_PREFIX + srUuid
lvmCache = LVMCache(vgName)
setInnerNodeRefcounts(lvmCache, srUuid)
except:
util.logException("setInnerNodeRefcounts")
elif cmd == "extend":
ssize = sys.argv[2]
path = sys.argv[3]
vgName = path.split('/')[-2].replace('VG_XenStorage', 'lvm')
lvUuid = path.split('/')[-1].lstrip('VHD-')
lock = Lock(vgName, lvUuid)
util.SMlog("LV Extending %s to %s" % (path, ssize))
_tryAcquire(lock)
if not lvutil.extend(ssize, path):
sys.exit(1)
lock.release()
else:
util.SMlog("Invalid usage")
print "Usage: %s <fixrefcounts|extend> ..." % sys.argv[0]
def process_repositories(self, repo_configs, ref, action, request_body):
"""Verify that the suggested repositories has matching settings and
issue git pull and/or deploy commands."""
import os
import time
import logging
from wrappers import GitWrapper
from lock import Lock
import json
logger = logging.getLogger()
data = json.loads(request_body)
result = []
# Process each matching repository
for repo_config in repo_configs:
repo_result = {}
try:
# Verify that all filters matches the request (if any filters are specified)
if 'filters' in repo_config:
# At least one filter must match
for filter in repo_config['filters']:
# All options specified in the filter must match
for filter_key, filter_value in filter.iteritems():
# Ignore filters with value None (let them pass)
if filter_value == None:
continue
# Support for earlier version so it's non-breaking functionality
if filter_key == 'action' and filter_value == action:
continue
# Interpret dots in filter name as path notations
node_value = data
for node_key in filter_key.split('.'):
# If the path is not valid the filter does not match
if not node_key in node_value:
logger.info(
"Filter '%s'' does not match since the path is invalid"
% (filter_key))
raise FilterMatchError()
node_value = node_value[node_key]
if filter_value == node_value:
continue
# If the filter value is set to True. the filter
# will pass regardless of the actual value
if filter_value == True:
continue
logger.info(
"Filter '%s'' does not match ('%s' != '%s')" %
(filter_key, filter_value,
(str(node_value)[:75] +
'..') if len(str(node_value)) > 75 else
str(node_value)))
raise FilterMatchError()
except FilterMatchError as e:
# Filter does not match, do not process this repo config
continue
# In case there is no path configured for the repository, no pull will
# be made.
if not 'path' in repo_config:
res = GitWrapper.deploy(repo_config)
repo_result['deploy'] = res
result.append(repo_result)
continue
# If the path does not exist, a warning will be raised and no pull or
# deploy will be made.
if not os.path.isdir(repo_config['path']):
logger.error(
"The repository '%s' does not exist locally. Make sure it was pulled "
% repo_config['path'] +
"properly without errors by reviewing the log.")
result.append(repo_result)
continue
# If the path is not writable, a warning will be raised and no pull or
# deploy will be made.
if not os.access(repo_config['path'], os.W_OK):
logger.error(
"The path '%s' is not writable. Make sure that GAD has write access to that path."
% repo_config['path'])
result.append(repo_result)
continue
running_lock = Lock(
os.path.join(repo_config['path'], 'status_running'))
waiting_lock = Lock(
os.path.join(repo_config['path'], 'status_waiting'))
try:
# Attempt to obtain the status_running lock
while not running_lock.obtain():
# If we're unable, try once to obtain the status_waiting lock
if not waiting_lock.has_lock() and not waiting_lock.obtain(
):
logger.error(
"Unable to obtain the status_running lock nor the status_waiting lock. Another process is "
+ "already waiting, so we'll ignore the request.")
# If we're unable to obtain the waiting lock, ignore the request
break
# Keep on attempting to obtain the status_running lock until we succeed
time.sleep(5)
n = 4
res = None
while n > 0:
# Attempt to pull up a maximum of 4 times
res = GitWrapper.pull(repo_config)
repo_result['git pull'] = res
# Return code indicating success?
if res == 0:
break
n -= 1
if 0 < n:
res = GitWrapper.deploy(repo_config)
repo_result['deploy'] = res
#except Exception as e:
# logger.error('Error during \'pull\' or \'deploy\' operation on path: %s' % repo_config['path'])
# logger.error(e)
# raise e
finally:
# Release the lock if it's ours
if running_lock.has_lock():
running_lock.release()
# Release the lock if it's ours
if waiting_lock.has_lock():
waiting_lock.release()
result.append(repo_result)
return result
class SenderThread(NetworkThread):
def __init__(self, dfs, fileSystem):
NetworkThread.__init__(self, dfs)
self.fileSystem_ = fileSystem
self.listeners_ = []
self.peerLock_ = Lock(dfs, 'peer')
self.workQueue_ = []
self.work_ = None
self.knownPeers_ = set()
self.fileFetchStatus = []
def isDoneFileFetch(self):
return len(self.fileFetchStatus) == 0
def getPeers(self):
return self.knownPeers_
def connectToMultiple(self, dfsList):
for dfs in dfsList:
self.connectTo(dfs)
def connectTo(self, dfs):
if self.isConnectedTo(dfs):
self.log_.v('already connected to ' + str(dfs.id))
return
self.registerConnDFS(dfs)
lt = ListenerThread(self.dfs_, self.addWork)
status = lt.connect(dfs)
if status < 0:
return status
self.log_.v('connected to ' + str(dfs.id))
lt.start()
self.addListener(lt)
def addListener(self, listener):
self.peerLock_.acquire()
self.listeners_.append(listener)
self.peerLock_.release()
self.addUpdateWork(listener)
def updateAll(self):
self.peerLock_.acquire()
for lt in self.listeners_:
self.addUpdateWork(lt)
self.peerLock_.release()
def beginFileFetch(self, fileName):
ltList = []
self.peerLock_.acquire()
self.fileFetchStatus = []
for lt in self.listeners_:
self.fileFetchStatus.append(lt)
ltList.append(lt)
self.peerLock_.release()
for lt in ltList:
self.addChunkRequestWork(lt, fileName)
def editPropagated(self):
# due to time constraints, I'm using this rather simple implementation
time.sleep(1)
return True
def isConnectedTo(self, dfs):
if dfs == self.dfs_:
return True
val = False
self.peerLock_.acquire()
for lt in self.listeners_:
if lt.getConnDFS() == dfs:
val = True
break
self.peerLock_.release()
return val
def registerConnDFS(self, dfs):
self.log_.v('registering new peer: ' + dfs.id.str)
self.knownPeers_.add(dfs)
def addWork(self, work):
self.workQueue_.append(work)
def doWork(self):
if len(self.workQueue_) > 0:
self.work_ = self.workQueue_.pop(0)
self.processWork()
else:
self.removeDisconnectedListeners()
time.sleep(.03)
def removeDisconnectedListeners(self):
self.peerLock_.acquire()
for lt in self.listeners_:
if not lt.active_:
self.log_.v('detected that ' + lt.getConnDFS().id.str + ' has disconnected')
self.removeListener(lt)
self.peerLock_.release()
def processWork(self):
if self.work_.source.id == self.dfs_.id:
self.sendWork()
else:
lt = self.work_.dest
if lt.getConnDFS().isInit():
self.knownPeers_.add(lt.getConnDFS())
if self.work_.type == work.UPDATE:
self.handleUpdate()
elif self.work_.type == work.CHUNK_REQUEST:
self.handleChunkRequest()
elif self.work_.type == work.CHUNK_RESPONSE:
self.handleChunkResponse()
elif self.work_.type == work.NO_CHUNK:
self.handleNoChunk()
else:
self.log_.e('received work unkonwn type: ' + self.work_.type)
def sendWork(self):
lt = self.work_.dest
status = lt.sendWork(self.work_)
if status < 0:
self.peerLock_.acquire()
self.log_.v('trying to send work, but ' + lt.getConnDFS().id.str + ' has disconnected')
self.removeListener(lt)
self.peerLock_.release()
def removeListener(self, lt):
if lt in self.fileFetchStatus:
self.fileFetchStatus.remove(lt)
if lt in self.listeners_:
self.listeners_.remove(lt) # other peer has disconnected
def handleUpdate(self):
lt = self.work_.dest
self.log_.v('received update from ' + str(lt.getConnDFS().id))
fs, ns = self.work_.data
status = self.fileSystem_.updateFiles(fs)
self.connectToMultiple(ns)
if status == err.CausedConflict:
self.log_.v('update from ' + str(lt.getConnDFS().id) + ' caused conflict, updating all peers')
self.updateAll()
def handleNoChunk(self):
lt = self.work_.dest
self.peerLock_.acquire()
if lt in self.fileFetchStatus:
self.log_.v(lt.getConnDFS().id.str + ' is no longer providing chunks')
self.fileFetchStatus.remove(lt)
self.peerLock_.release()
def handleChunkRequest(self):
fileName, missingChunks = self.work_.data
lt = self.work_.dest
chunkInfo = self.fileSystem_.getRandomChunk(fileName, missingChunks)
self.addChunkResponseWork(chunkInfo, fileName)
def handleChunkResponse(self):
lt = self.work_.dest
fileName, chunkNum, chunkData = self.work_.data
self.log_.v(lt.getConnDFS().id.str + ' received chunk ' + str(chunkNum) + ' of ' + fileName)
self.fileSystem_.writeChunk(fileName, chunkNum, chunkData)
self.addChunkRequestWork(lt, fileName)
def addUpdateWork(self, lt):
state = (self.fileSystem_.getState(), self.getPeers())
w = work.Work(work.UPDATE, self.dfs_, lt, state)
self.log_.v('sending an update to ' + lt.getConnDFS().id.str)
self.addWork(w)
def addChunkRequestWork(self, lt, fileName):
missingChunks = self.fileSystem_.getMissingChunks(fileName)
if missingChunks == None:
self.log_.v(fileName + ' is no longer missing chunks. File retrieval complete')
self.handleNoChunk()
return
data = (fileName, missingChunks)
w = work.Work(work.CHUNK_REQUEST, self.dfs_, lt, data)
self.log_.v('requesting a chunk of ' + fileName + ' from ' + lt.getConnDFS().id.str)
self.addWork(w)
def addChunkResponseWork(self, chunkInfo, fileName):
lt = self.work_.dest
type = ''
if chunkInfo:
type = work.CHUNK_RESPONSE
fileName, chunkNum, chunkData = chunkInfo
self.log_.v('sending chunk ' + str(chunkNum) + ' of ' + fileName + ' to ' + lt.getConnDFS().id.str)
else:
self.log_.v('do not have ' + fileName + ' chunk for ' + lt.getConnDFS().id.str)
type = work.NO_CHUNK
w = work.Work(type, self.dfs_, lt, chunkInfo)
self.addWork(w)
def close(self):
self.log_.d('close started')
self.peerLock_.acquire()
for listener in self.listeners_:
listener.close()
self.peerLock_.release()
self.waitForListenersToClose()
NetworkThread.close(self)
self.log_.d('close finished')
def waitForListenersToClose(self):
self.log_.d('waitForListenersToClose started')
self.peerLock_.acquire()
lts = list(self.listeners_)
self.peerLock_.release()
for lt in lts:
lt.join()
self.log_.d('waitForListenersToClose finished')
