def __init__(self, hosts, session=None, name=None, default_acl=None,
auth_data=None, rsyncpath=None, rsyncdepth=-1,
netcat=False, dryrun=False, delete=False, excludere=None):
kwargs = {
'hosts' : hosts,
'session' : session,
'name' : name,
'default_acl' : default_acl,
'auth_data' : auth_data,
'rsyncpath' : rsyncpath,
'netcat' : netcat
}
super(RsyncSource, self).__init__(**kwargs)
self.lockpath = self.znode_path(self.session + '/lock')
self.lock = None
self.path_queue = LockingQueue(self, self.znode_path(self.session + '/pathQueue'))
self.completed_queue = LockingQueue(self, self.znode_path(self.session + '/completedQueue'))
self.failed_queue = LockingQueue(self, self.znode_path(self.session + '/failedQueue'))
self.output_queue = LockingQueue(self, self.znode_path(self.session + '/outputQueue'))
if rsyncdepth < 0:
self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth)
else:
self.rsyncdepth = rsyncdepth
self.rsync_delete = delete
self.rsync_dry = dryrun
self.excludere = excludere
def processTransfer():
try:
conn = psycopg2.connect(dbConnectStr)
cur = conn.cursor()
zk = KazooClient(hosts=zkHost)
zk.start()
transferq = LockingQueue(zk, '/transfer/')
while True:
rawCode = transferq.get()
proposal = rawCode.decode().strip()
transferq.consume()
# print(" proposal = {0} ".format(proposal))
ints = datetime.now()
inload = os.getloadavg()[0]
pro1 = Popen(['/usr/bin/python36', './processproptran.py', proposal], stdin=None, stdout=None)
pro1.wait()
outts = datetime.now()
outload = os.getloadavg()[0]
# insert the runtime info into c*
cluster = Cluster(cfg.cassCluster)
session = cluster.connect(cfg.cassKeyspace)
stmt = SimpleStatement("""insert into runstat(id,executable,ints,inload,outts,outload)
values (%s, %s, %s, %s, %s, %s)""", consistency_level=ConsistencyLevel.ANY)
session.execute(stmt, (uuid.uuid4(), executable, ints, inload, outts, outload))
except psycopg2.Error as err:
print("SQLError {0}".format(err))
finally:
zk.stop()
zk.close()
cur.close()
conn.close()
def _work(queue_name, function, resource_name, lock_timeout=10):
logger = setup_logger(queue_name)
logger.info("Worker Started!")
zookeeper_client = get_zookeeper_client()
queue = LockingQueue(zookeeper_client, queue_name)
while True:
try:
message = get_next_message(queue)
except Exception as e:
logger.exception(
"Error while retrieving message from queue: {}".format(e)
)
try:
with get_resource_lock(
resource_name, zookeeper_client, message['name'],
timeout=lock_timeout
):
if queue.consume():
try:
function(message, logger)
except Exception as e:
logger.exception(
"Function raised exception: {}".format(e)
)
except exceptions.LockTimeout:
logger.info("LockTimeout on {}".format(message['name']))
except Exception as e:
logger.exception("Exception: {}".format(e))
def processSymbol():
try:
conn = psycopg2.connect(dbConnectStr)
cur = conn.cursor()
zk = KazooClient(hosts=zkHost)
zk.start()
symbolq = LockingQueue(zk, cfg.symbol)
while True:
rawCode = symbolq.get()
ints = datetime.now()
inload = os.getloadavg()[0]
symbol = rawCode.decode().split('||')[0]
globalId = rawCode.decode().split('||')[1]
symbolq.consume()
alias = ''
while not alias:
print("loop for the alias of {0}".format(globalId))
cur.execute("""
select alias from player0 where "globalId" = %s
""", [globalId])
res = cur.fetchone()
conn.commit()
if res:
alias = res[0]
print("process symbol:{0} alias:{1} globalId:{2}".format(symbol, alias, globalId))
lock0 = zk.Lock(symbol, 'jg')
with lock0:
# the operation
cmd = "cd /{4}/;{3}/openssl-1.0.2o/apps/openssl genrsa 2048| {3}/openssl-1.0.2o/apps/openssl asn1parse|/{4}/parseoutput.pl; /usr/bin/perl /{4}/makeissuer.pl '{0}' '{1}' '{2}'".format(alias, symbol, globalId, baseDir, workshopInstance)
f = os.popen(cmd)
while True:
en = f.readline()
if en == '':
break
cur.execute("""
update symbol_redo set progress= 0, setup=now() where symbol=%s
""", [symbol])
conn.commit()
outts = datetime.now()
outload = os.getloadavg()[0]
# insert the runtime info into c*
cluster = Cluster(cfg.cassCluster)
session = cluster.connect(cfg.cassKeyspace)
stmt = SimpleStatement("""insert into runstat(id,executable,ints,inload,outts,outload)
values (%s, %s, %s, %s, %s, %s)""", consistency_level=ConsistencyLevel.ANY)
session.execute(stmt, (uuid.uuid4(), executable, ints, inload, outts, outload))
except psycopg2.Error as err:
print("SQLError {0}".format(err))
finally:
zk.stop()
zk.close()
cur.close()
conn.close()
def __init__(self, hosts, session=None, name=None, default_acl=None,
auth_data=None, rsyncpath=None, netcat=None, verifypath=True, dropcache=False):
kwargs = {
'hosts' : hosts,
'session' : session,
'name' : name,
'default_acl' : default_acl,
'auth_data' : auth_data,
}
self.netcat = netcat
super(RsyncController, self).__init__(**kwargs)
self.rsyncpath = rsyncpath.rstrip(os.path.sep)
if not netcat:
if verifypath and not self.basepath_ok():
self.log.raiseException('Path does not exists in filesystem: %s' % rsyncpath)
if not os.path.isdir(self.RSDIR):
os.mkdir(self.RSDIR, 0o700)
self.module = 'zkrs-%s' % self.session
self.dest_queue = LockingQueue(self, self.znode_path(self.session + '/destQueue'))
self.verifypath = verifypath
self.rsync_dropcache = dropcache
def __init__(self, hosts, session=None, name=None, default_acl=None,
auth_data=None, rsyncpath=None, rsyncdepth=-1, rsubpaths=None,
netcat=False, dryrun=False, delete=False, checksum=False,
hardlinks=False, verbose=False, dropcache=False, timeout=None,
excludere=None, excl_usr=None, verifypath=True, done_file=None, arbitopts=None):
kwargs = {
'hosts' : hosts,
'session' : session,
'name' : name,
'default_acl' : default_acl,
'auth_data' : auth_data,
'rsyncpath' : rsyncpath,
'verifypath' : verifypath,
'netcat' : netcat,
'dropcache' : dropcache,
}
super(RsyncSource, self).__init__(**kwargs)
self.lockpath = self.znode_path(self.session + '/lock')
self.lock = None
self.path_queue = LockingQueue(self, self.znode_path(self.session + '/pathQueue'))
self.completed_queue = LockingQueue(self, self.znode_path(self.session + '/completedQueue'))
self.failed_queue = LockingQueue(self, self.znode_path(self.session + '/failedQueue'))
self.output_queue = LockingQueue(self, self.znode_path(self.session + '/outputQueue'))
self.stats_path = '%s/stats' % self.session
self.init_stats()
if rsyncdepth < 0:
self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth)
else:
self.rsyncdepth = rsyncdepth
self.rsync_arbitopts = arbitopts
self.rsync_checksum = checksum
self.rsync_delete = delete
self.rsync_dry = dryrun
self.rsync_hardlinks = hardlinks
self.rsync_timeout = timeout
self.rsync_verbose = verbose
self.done_file = done_file
self.excludere = excludere
self.excl_usr = excl_usr
self.rsubpaths = rsubpaths
def wrapper():
try:
conn = psycopg2.connect(dbConnectStr)
cur = conn.cursor()
zk = KazooClient(hosts=zkHost)
zk.start()
propTranq = LockingQueue(zk, cfg.proposeTran)
while True:
rawC = propTranq.get()
ints = datetime.now()
inload = os.getloadavg()[0]
alias = rawC.decode().split('&&')[0]
rawCode = rawC.decode().split('&&')[1]
lastsig = rawC.decode().split('&&')[2]
entryId = rawC.decode().split('&&')[3]
globalId = rawC.decode().split('&&')[4]
pro1 = Popen(['/usr/bin/python36','/{0}/payer{1}.py'.format(workshopInstance, alias), rawCode, lastsig, globalId], stdin=None, stdout=None)
pro1.wait()
cur.execute("""update propose_transfer set progress = 0 , setup = now() where id = %s
""", [entryId])
conn.commit()
propTranq.consume()
outts = datetime.now()
outload = os.getloadavg()[0]
# insert the runtime info into c*
cluster = Cluster(cfg.cassCluster)
session = cluster.connect(cfg.cassKeyspace)
stmt = SimpleStatement("""insert into runstat(id,executable,ints,inload,outts,outload)
values (%s, %s, %s, %s, %s, %s)""", consistency_level=ConsistencyLevel.ANY)
session.execute(stmt, (uuid.uuid4(), executable, ints, inload, outts, outload))
except psycopg2.Error as err:
print("SQLError {0}".format(err))
finally:
zk.stop()
zk.close()
cur.close()
conn.close()
# Start client
# read_only mode must be turned on for the servers in the Zookeeper
# cluster for the client to utilize it.
zk = KazooClient(hosts='127.0.0.1:2181')
zk.start()
print("Client started!\n")
# Clear the nodes at the end or else you migth get a NodeExistsError next time
zk.delete("/app", recursive=True)
# Ensure a path, create if it doesn't exist
zk.ensure_path("/app")
# Initialize queue at specified location
zk_queue = LockingQueue(zk, "/app/queue")
# Thread handler
handler = SequentialThreadingHandler()
# Watchers ---------------------------------------------------------------
@zk.DataWatch("/app/queue")
def watch_queue(data, stat):
if data == "":
pass
elif data is not None or stat is not None:
global input_done
# print("Queue Node:")
# print("%s\n" % data)
def processAlias():
try:
conn = psycopg2.connect(dbConnectStr)
cur = conn.cursor()
zk = KazooClient(hosts=zkHost)
zk.start()
aliasq = LockingQueue(zk, cfg.alias)
while True:
rawCode = aliasq.get()
ints = datetime.now()
inload = os.getloadavg()[0]
alias = rawCode.decode().split('||')[0]
globalId = rawCode.decode().split('||')[1]
aliasq.consume()
print("process alias:{0} globalId:{1}".format(alias, globalId))
lock0 = zk.Lock(alias, 'jg')
with lock0:
# the operation
cmd = "cd /{3}/; {2}/openssl-1.0.2o/apps/openssl genrsa 2048| {2}/openssl-1.0.2o/apps/openssl asn1parse|/{3}/parseoutput.pl; /usr/bin/perl /{3}/makepayer.pl '{0}' '{1}'".format(alias, globalId, baseDir, workshopInstance)
f = os.popen(cmd)
while True:
en = f.readline()
if en == '':
break
cur.execute("""
update alias_redo set progress= 0, setup=now() where alias=%s
""", [alias])
conn.commit()
# insert the entry to symbol_redo
sha256 = hashlib.sha256()
while True:
sha256.update('{0}'.format(globalId).encode('utf-8'))
sha256.update('{0}'.format(alias).encode('utf-8'))
hashCode = sha256.hexdigest()
symbol = hashCode[:5]
symbol = symbol.upper()
cur.execute("""
select symbol from symbol_redo where symbol = %s
union
select symbol from issuer0 where symbol = %s
""", [symbol, symbol])
res = cur.fetchone()
conn.commit()
if not res:
break
cur.execute("""
insert into symbol_redo("globalId",symbol,setup,progress) values(%s,%s,now(), 1)
""", [globalId, symbol])
conn.commit()
# insert the runtime_stat to c*
outload = os.getloadavg()[0]
outts = datetime.now()
stmt = SimpleStatement("""insert into runstat(id,executable,ints,inload,outts,outload) values (%s, %s, %s, %s, %s, %s)""", consistency_level=ConsistencyLevel.ANY)
cluster = Cluster(cfg.cassCluster)
session = cluster.connect(cfg.cassKeyspace)
session.execute(stmt, (uuid.uuid4(), executable, ints, inload, outts, outload))
except psycopg2.Error as err:
print("SQLError {0}".format(err))
finally:
zk.stop()
zk.close()
cur.close()
conn.close()
class RsyncSource(RsyncController):
"""
Class for controlling rsync with Zookeeper.
Builds a tree of paths to devide, and effectively rsyncs the subpaths.
Stops when ready
"""
BASE_PARTIES = RsyncController.BASE_PARTIES + ['sources']
NC_RANGE = 15
SLEEPTIME = 1 # For netcat stub
TIME_OUT = 5 # waiting for destination
WAITTIME = 5 # check interval of closure of other clients
def __init__(self, hosts, session=None, name=None, default_acl=None,
auth_data=None, rsyncpath=None, rsyncdepth=-1,
netcat=False, dryrun=False, delete=False, excludere=None):
kwargs = {
'hosts' : hosts,
'session' : session,
'name' : name,
'default_acl' : default_acl,
'auth_data' : auth_data,
'rsyncpath' : rsyncpath,
'netcat' : netcat
}
super(RsyncSource, self).__init__(**kwargs)
self.lockpath = self.znode_path(self.session + '/lock')
self.lock = None
self.path_queue = LockingQueue(self, self.znode_path(self.session + '/pathQueue'))
self.completed_queue = LockingQueue(self, self.znode_path(self.session + '/completedQueue'))
self.failed_queue = LockingQueue(self, self.znode_path(self.session + '/failedQueue'))
self.output_queue = LockingQueue(self, self.znode_path(self.session + '/outputQueue'))
if rsyncdepth < 0:
self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth)
else:
self.rsyncdepth = rsyncdepth
self.rsync_delete = delete
self.rsync_dry = dryrun
self.excludere = excludere
def get_sources(self):
""" Get all zookeeper clients in this session registered as clients """
hosts = []
for host in self.parties['sources']:
hosts.append(host)
return hosts
def acq_lock(self):
""" Try to acquire lock. Returns true if lock is acquired """
self.lock = self.Lock(self.lockpath, "")
return self.lock.acquire(False)
def release_lock(self):
""" Release the acquired lock """
return self.lock.release()
def start_ready_rwatch(self):
""" Start a watch other clients can register to, but release lock and exit on error """
watch = self.start_ready_watch()
if not watch:
if len(self.get_all_hosts()) == 1: # Fix previous unclean shutdown
self.cleanup()
self.release_lock()
self.exit()
return False
else:
return watch
def build_pathqueue(self):
""" Build a queue of paths that needs to be rsynced """
self.log.debug('removing old queue and building new queue')
if self.exists(self.path_queue.path):
self.delete(self.path_queue.path, recursive=True)
if self.netcat:
paths = [str(i) for i in range(self.NC_RANGE)]
time.sleep(self.SLEEPTIME)
else:
tuplpaths = get_pathlist(self.rsyncpath, self.rsyncdepth, exclude_re=self.excludere,
exclude_usr='******') # Don't exclude user files
paths = encode_paths(tuplpaths)
self.paths_total = len(paths)
for path in paths:
self.path_queue.put(path) # Put_all can issue a zookeeper connection error with big lists
self.log.debug('pathqueue building finished')
return self.paths_total
def isempty_pathqueue(self):
""" Returns true if all paths in pathqueue are done """
return len(self.path_queue) == 0
def len_paths(self):
""" Returns how many elements still in pathQueue """
return len(self.path_queue)
def shutdown_all(self):
""" Send end signal and release lock
Make sure other clients are disconnected, clean up afterwards."""
self.stop_ready_watch()
self.log.debug('watch set to stop')
while len(self.get_all_hosts()) > 1:
# self.log.debug("clients still connected: %s" % self.get_all_hosts())
time.sleep(self.WAITTIME)
self.cleanup()
def cleanup(self):
""" Remove all session nodes in zookeeper after first logging the output queues """
self.delete(self.dest_queue.path, recursive=True)
self.delete(self.path_queue.path, recursive=True)
while (len(self.failed_queue) > 0):
self.log.error('Failed Path %s' % self.failed_queue.get())
self.failed_queue.consume()
while (len(self.completed_queue) > 0):
self.log.info('Completed Path %s' % self.completed_queue.get())
self.completed_queue.consume()
self.log.info('Output:')
while (len(self.output_queue) > 0):
self.log.info(self.output_queue.get())
self.output_queue.consume()
self.delete(self.completed_queue.path, recursive=True)
self.delete(self.failed_queue.path, recursive=True)
self.delete(self.output_queue.path, recursive=True)
self.remove_ready_watch()
self.release_lock()
self.log.debug('Lock, Queues and watch removed')
def generate_file(self, path):
"""
Writes the relative path used for the rsync of this path,
for use by --files-from.
"""
if not path.startswith(self.rsyncpath):
self.log.raiseException('Invalid path! %s is not a subpath of %s!' % (path, self.rsyncpath))
else:
subpath = path[len(self.rsyncpath):]
subpath = subpath.strip(os.path.sep)
fd, name = tempfile.mkstemp(dir=self.RSDIR, text=True)
file = os.fdopen(fd, "w")
file.write('%s/' % subpath)
return name
def attempt_run(self, path, attempts=3):
""" Try to run a command x times, on failure add to failed queue """
attempt = 1
while (attempt <= attempts):
dest = self.get_a_dest(self.TIME_OUT) # Keeps it if not consuming
if dest: # We locked a rsync daemon
self.log.debug('Got destination %s' % dest)
port, host, other = tuple(dest.split(':', 2))
if self.netcat:
code, output = self.run_netcat(path, host, port)
else:
code, output = self.run_rsync(path, host, port)
if code == 0:
self.completed_queue.put(path)
return code, output
attempt += 1
time.sleep(self.WAITTIME) # Wait before new attempt
if dest:
self.log.error('There were issues with path %s!' % path)
self.failed_queue.put(path)
return 0, output # otherwise client get stuck
else:
self.log.debug('Still no destination after %s tries' % attempts)
self.path_queue.put(path, priority=50) # Keep path in queue
self.path_queue.consume() # But stop locking it
time.sleep(self.TIME_OUT) # Wait before new attempt
return 1, None
def run_rsync(self, encpath, host, port):
"""
Runs the rsync command with or without recursion, delete or dry-run option.
It uses the destination module linked with this session.
"""
path, recursive = decode_path(encpath)
file = self.generate_file(path)
# Start rsync recursive or non recursive; archive mode (a) is equivalent to -rlptgoD (see man rsync)
flags = ['--stats', '--numeric-ids', '-lptgoD', '--files-from=%s' % file]
if recursive:
flags.append('-r')
if self.rsync_delete:
flags.append('--delete')
if self.rsync_dry:
flags.append('-n')
self.log.debug('echo %s is sending %s to %s %s' % (self.whoami, path, host, port))
command = 'rsync %s %s/ rsync://%s:%s/%s' % (' '.join(flags), self.rsyncpath,
host, port, self.module)
code, output = RunAsyncLoopLog.run(command)
os.remove(file)
return code, output
def run_netcat(self, path, host, port):
""" Test run with netcat """
time.sleep(self.SLEEPTIME)
command = 'echo %s is sending %s | nc %s %s' % (self.whoami, path, host, port)
return RunAsyncLoopLog.run(command)
def rsync_path(self, path):
""" start rsync session for given path and destination, returns true if successful """
if not path:
self.log.raiseException('Empty path given!')
elif not isinstance(path, basestring):
self.log.raiseException('Invalid path: %s !' % path)
else:
code, output = self.attempt_run(path)
if output:
self.output_queue.put(output)
return (code == 0)
def get_a_dest(self, timeout):
"""
Try to get a destination.
check if destination is still running, otherwise remove
"""
if len(self.dest_queue) == 0:
self.log.debug('Destinations not yet available')
dest = self.dest_queue.get(timeout)
if dest:
port, whoami = tuple(dest.split(':', 1))
if not self.member_of_party(whoami, 'allsd'):
self.log.debug('destination is not found in party')
self.dest_queue.consume()
return None
else:
portmap = '%s/portmap/%s' % (self.session, whoami)
lport, stat = self.get_znode(portmap)
if port != lport:
self.log.error('destination port not valid') # Should not happen
self.dest_queue.consume()
return None
return dest
def rsync(self, timeout=None):
""" Get a destination, a path and call a new rsync iteration """
path = self.path_queue.get(timeout)
if path:
if self.rsync_path(path):
self.path_queue.consume()
class RsyncSource(RsyncController):
"""
Class for controlling rsync with Zookeeper.
Builds a tree of paths to devide, and effectively rsyncs the subpaths.
Stops when ready
"""
BASE_PARTIES = RsyncController.BASE_PARTIES + ['sources']
NC_RANGE = 15
SLEEPTIME = 1 # For netcat stub
TIME_OUT = 5 # waiting for destination
WAITTIME = 5 # check interval of closure of other clients
CHECK_WAIT = 20 # wait for path to be available
RSYNC_STATS = ['Number_of_files', 'Number_of_files_transferred', 'Total_file_size',
'Total_transferred_file_size', 'Literal_data', 'Matched_data', 'File_list_size',
'Total_bytes_sent', 'Total_bytes_received'];
def __init__(self, hosts, session=None, name=None, default_acl=None,
auth_data=None, rsyncpath=None, rsyncdepth=-1, rsubpaths=None,
netcat=False, dryrun=False, delete=False, checksum=False,
hardlinks=False, verbose=False, dropcache=False, timeout=None,
excludere=None, excl_usr=None, verifypath=True, done_file=None, arbitopts=None):
kwargs = {
'hosts' : hosts,
'session' : session,
'name' : name,
'default_acl' : default_acl,
'auth_data' : auth_data,
'rsyncpath' : rsyncpath,
'verifypath' : verifypath,
'netcat' : netcat,
'dropcache' : dropcache,
}
super(RsyncSource, self).__init__(**kwargs)
self.lockpath = self.znode_path(self.session + '/lock')
self.lock = None
self.path_queue = LockingQueue(self, self.znode_path(self.session + '/pathQueue'))
self.completed_queue = LockingQueue(self, self.znode_path(self.session + '/completedQueue'))
self.failed_queue = LockingQueue(self, self.znode_path(self.session + '/failedQueue'))
self.output_queue = LockingQueue(self, self.znode_path(self.session + '/outputQueue'))
self.stats_path = '%s/stats' % self.session
self.init_stats()
if rsyncdepth < 0:
self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth)
else:
self.rsyncdepth = rsyncdepth
self.rsync_arbitopts = arbitopts
self.rsync_checksum = checksum
self.rsync_delete = delete
self.rsync_dry = dryrun
self.rsync_hardlinks = hardlinks
self.rsync_timeout = timeout
self.rsync_verbose = verbose
self.done_file = done_file
self.excludere = excludere
self.excl_usr = excl_usr
self.rsubpaths = rsubpaths
def init_stats(self):
self.ensure_path(self.znode_path(self.stats_path))
self.counters = {};
for stat in self.RSYNC_STATS:
self.counters[stat] = Counter(self, self.znode_path('%s/%s' % (self.stats_path, stat)))
def output_stats(self):
self.stats = dict([(k, v.value) for k, v in self.counters.items()])
jstring = json.dumps(self.stats)
self.log.info('progress stats: %s' % jstring)
return jstring
def get_sources(self):
""" Get all zookeeper clients in this session registered as clients """
hosts = []
for host in self.parties['sources']:
hosts.append(host)
return hosts
def acq_lock(self):
""" Try to acquire lock. Returns true if lock is acquired """
self.lock = self.Lock(self.lockpath, "")
return self.lock.acquire(False)
def release_lock(self):
""" Release the acquired lock """
return self.lock.release()
def start_ready_rwatch(self):
""" Start a watch other clients can register to, but release lock and exit on error """
watch = self.start_ready_watch()
if not watch:
if len(self.get_all_hosts()) == 1: # Fix previous unclean shutdown
self.cleanup()
self.release_lock()
self.exit()
return False
else:
return watch
def build_pathqueue(self):
""" Build a queue of paths that needs to be rsynced """
self.log.info('removing old queue and building new queue')
if self.exists(self.path_queue.path):
self.delete(self.path_queue.path, recursive=True)
if self.netcat:
paths = [str(i) for i in range(self.NC_RANGE)]
time.sleep(self.SLEEPTIME)
else:
tuplpaths = get_pathlist(self.rsyncpath, self.rsyncdepth, exclude_re=self.excludere,
exclude_usr=self.excl_usr, rsubpaths=self.rsubpaths) # By default don't exclude user files
paths = encode_paths(tuplpaths)
self.paths_total = len(paths)
for path in paths:
self.path_queue.put(path) # Put_all can issue a zookeeper connection error with big lists
self.log.info('pathqueue building finished')
return self.paths_total
def isempty_pathqueue(self):
""" Returns true if all paths in pathqueue are done """
return len(self.path_queue) == 0
def output_progress(self, todo):
self.log.info('Progress: %s of %s paths remaining, %s failed' % (todo, self.paths_total, len(self.failed_queue)))
self.output_stats()
def output_clients(self, total, sources):
dests = total - sources
sources = sources - 1
self.log.info('Connected source (slave) clients: %s, connected destination clients: %s', sources, dests)
def wait_and_keep_progress(self):
todo_paths = self.paths_total
total_clients = len(self.get_all_hosts())
total_sources = len(self.get_sources())
while not self.isempty_pathqueue():
todo_new = self.len_paths()
if todo_paths != todo_new: # Output progress state
todo_paths = todo_new
self.output_progress(todo_paths)
tot_clients_new = len(self.get_all_hosts())
src_clients_new = len(self.get_sources())
if total_clients != tot_clients_new or total_sources != src_clients_new:
total_clients = tot_clients_new
total_sources = src_clients_new
self.output_clients(total_clients, total_sources);
time.sleep(self.WAITTIME)
def len_paths(self):
""" Returns how many elements still in pathQueue """
return len(self.path_queue)
def shutdown_all(self):
""" Send end signal and release lock
Make sure other clients are disconnected, clean up afterwards."""
self.stop_ready_watch()
self.log.debug('watch set to stop')
while len(self.get_all_hosts()) > 1:
# self.log.debug("clients still connected: %s" % self.get_all_hosts())
time.sleep(self.WAITTIME)
self.cleanup()
def get_state(self):
"""Get the state of a running session"""
remain = self.len_paths()
if remain > 0:
code = 0
self.log.info('Remaining: %s, Failed: %s' % (remain, len(self.failed_queue)))
else:
code = ZKRS_NO_SUCH_SESSION_EXIT_CODE
self.log.info('No active session')
return code
def write_donefile(self, values):
""" Write a cachefile with some stats about the run when done """
cache_file = FileCache(self.done_file)
cache_file.update('stats', values, 0)
cache_file.close()
def cleanup(self):
""" Remove all session nodes in zookeeper after first logging the output queues """
values = {
'unfinished' : len(self.path_queue),
'failed' : len(self.path_queue),
'completed' : len(self.completed_queue)
}
while (len(self.path_queue) > 0):
self.log.warning('Unfinished Path %s' % self.path_queue.get())
self.path_queue.consume()
self.delete(self.dest_queue.path, recursive=True)
self.delete(self.path_queue.path, recursive=True)
self.output_stats()
self.delete(self.znode_path(self.stats_path), recursive=True)
while (len(self.failed_queue) > 0):
self.log.error('Failed Path %s' % self.failed_queue.get())
self.failed_queue.consume()
while (len(self.completed_queue) > 0):
self.log.info('Completed Path %s' % self.completed_queue.get())
self.completed_queue.consume()
self.log.info('Output:')
while (len(self.output_queue) > 0):
self.log.info(self.output_queue.get())
self.output_queue.consume()
self.delete(self.completed_queue.path, recursive=True)
self.delete(self.failed_queue.path, recursive=True)
self.delete(self.output_queue.path, recursive=True)
self.remove_ready_watch()
self.release_lock()
self.log.info('Cleanup done: Lock, Queues and watch removed')
if self.done_file:
self.write_donefile(values)
def generate_file(self, path):
"""
Writes the relative path used for the rsync of this path,
for use by --files-from.
"""
if not path.startswith(self.rsyncpath):
self.log.raiseException('Invalid path! %s is not a subpath of %s!' % (path, self.rsyncpath))
else:
subpath = path[len(self.rsyncpath):]
subpath = subpath.strip(os.path.sep)
fd, name = tempfile.mkstemp(dir=self.RSDIR, text=True)
wfile = os.fdopen(fd, "w")
wfile.write('%s/' % subpath)
return name
def attempt_run(self, path, attempts=3):
""" Try to run a command x times, on failure add to failed queue """
attempt = 1
while (attempt <= attempts):
dest = self.get_a_dest(attempts) # Keeps it if not consuming
if not dest or not self.basepath_ok():
self.path_queue.put(path, priority=50) # Keep path in queue
self.path_queue.consume() # But stop locking it
time.sleep(self.TIME_OUT) # Wait before new attempt
return 1, None
port, host, _ = tuple(dest.split(':', 2))
if self.netcat:
code, output = self.run_netcat(path, host, port)
else:
code, output = self.run_rsync(path, host, port)
if code == 0:
self.completed_queue.put(path)
return code, output
attempt += 1
time.sleep(self.WAITTIME) # Wait before new attempt
self.log.error('There were issues with path %s!' % path)
self.failed_queue.put(path)
return 0, output # otherwise client get stuck
def parse_output(self, output):
"""
Parse the rsync output stats, and when verbose, print the files marked for transmission
"""
if self.rsync_verbose:
outp = output.split("%s%s" % (os.linesep, os.linesep))
self.log.info('Verbose file list output is: %s' % outp[0])
del outp[0]
output = os.linesep.join(outp)
lines = output.splitlines()
for line in lines:
keyval = line.split(':')
if len(keyval) < 2 or keyval[1] == ' ':
self.log.debug('output line not parsed: %s' % line)
continue
key = re.sub(' ', '_', keyval[0])
val = keyval[1].split()[0]
val = re.sub(',', '', val)
if key not in self.RSYNC_STATS:
self.log.debug('output metric not recognised: %s' % key)
continue
self.counters[key] += int(val)
def get_flags(self, files, recursive):
"""
Make an array of flags to be used
"""
# Start rsync recursive or non recursive; archive mode (a) is equivalent to -rlptgoD (see man rsync)
flags = ['--stats', '--numeric-ids', '-lptgoD', '--files-from=%s' % files]
if recursive:
flags.append('-r')
if self.rsync_delete:
flags.append('--delete')
if self.rsync_checksum:
flags.append('--checksum')
if self.rsync_dropcache:
flags.append('--drop-cache')
if self.rsync_hardlinks:
flags.append('--hard-links')
if self.rsync_timeout:
flags.extend(['--timeout', str(self.rsync_timeout)])
if self.rsync_verbose:
flags.append('--verbose')
if self.rsync_dry:
flags.append('-n')
# This should always be processed last
if self.rsync_arbitopts:
arbopts = []
for arbopt in self.rsync_arbitopts:
opt = "--%s" % re.sub(':', '=', arbopt, 1)
arbopts.append(opt)
self.log.warning('Adding unchecked flags %s' % ' '.join(arbopts))
flags.extend(arbopts)
return flags
def run_rsync(self, encpath, host, port):
"""
Runs the rsync command with or without recursion, delete or dry-run option.
It uses the destination module linked with this session.
"""
path, recursive = decode_path(encpath)
gfile = self.generate_file(path)
flags = self.get_flags(gfile, recursive)
self.log.info('%s is sending path %s to %s %s' % (self.whoami, path, host, port))
self.log.debug('Used flags: "%s"' % ' '.join(flags))
command = 'rsync %s %s/ rsync://%s:%s/%s' % (' '.join(flags), self.rsyncpath,
host, port, self.module)
code, output = RunAsyncLoopLog.run(command)
os.remove(gfile)
parsed = self.parse_output(output)
return code, parsed
def run_netcat(self, path, host, port):
""" Test run with netcat """
time.sleep(self.SLEEPTIME)
flags = self.get_flags('nofile', 0)
command = 'echo %s is sending %s with flags "%s" | nc %s %s' % (self.whoami, path, ' '.join(flags), host, port)
return RunAsyncLoopLog.run(command)
def rsync_path(self, path):
""" start rsync session for given path and destination, returns true if successful """
if not path:
self.log.raiseException('Empty path given!')
elif not isinstance(path, basestring):
self.log.raiseException('Invalid path: %s !' % path)
else:
code, output = self.attempt_run(path)
if output:
self.output_queue.put(output)
return (code == 0)
def get_a_dest(self, attempts):
""" Try to get a destination x times """
attempt = 1
while (attempt <= attempts):
dest = self.try_a_dest(self.TIME_OUT) # Keeps it if not consuming
if dest: # We locked a rsync daemon
self.log.debug('Got destination %s' % dest)
return dest
attempt += 1
time.sleep(self.WAITTIME)
self.log.warning('Still no destination after %s tries' % attempts)
return None
def dest_is_sane(self, dest):
""" Lock destination state, fetch it and disable if paused """
return self.dest_state(dest, self.STATUS)
def handle_dest_state(self, dest, destpath, current_state):
"""
Disable destination by consuming it from queue when paused and return false
If destination is active, return true
"""
if current_state == self.STATE_PAUSED:
self.set_znode(destpath, self.STATE_DISABLED)
self.dest_queue.consume()
self.log.debug('Destination %s was disabled and removed from queue' % dest)
return False
elif current_state == self.STATE_ACTIVE:
return True
else:
self.log.error('Destination %s is in an unknown state %s' % (dest, current_state))
return False
def try_a_dest(self, timeout):
"""
Try to get a destination.
check if destination is still running, otherwise remove
"""
if len(self.dest_queue) == 0:
self.log.debug('Destinations not yet available')
dest = self.dest_queue.get(timeout)
if dest:
port, whoami = tuple(dest.split(':', 1))
if not self.member_of_party(whoami, 'allsd'):
self.log.debug('destination is not found in party')
self.dest_queue.consume()
return None
elif not self.dest_is_sane(whoami):
self.log.debug('recieved destination was paused')
return None
else:
portmap = '%s/portmap/%s' % (self.session, whoami)
lport, _ = self.get_znode(portmap)
if port != lport:
self.log.error('destination port not valid') # Should not happen
self.dest_queue.consume()
return None
return dest
def rsync(self, timeout=None):
""" Get a destination, a path and call a new rsync iteration """
if self.verifypath:
if not self.basepath_ok():
self.log.warning('Basepath not available, waiting')
time.sleep(self.CHECK_WAIT)
return None
path = self.path_queue.get(timeout)
if path:
if self.rsync_path(path):
self.path_queue.consume()
def __init__(self,
hosts,
session=None,
name=None,
default_acl=None,
auth_data=None,
rsyncpath=None,
rsyncdepth=-1,
rsubpaths=None,
netcat=False,
dryrun=False,
delete=False,
checksum=False,
hardlinks=False,
inplace=False,
verbose=False,
dropcache=False,
timeout=None,
excludere=None,
excl_usr=None,
verifypath=True,
done_file=None,
arbitopts=None):
kwargs = {
'hosts': hosts,
'session': session,
'name': name,
'default_acl': default_acl,
'auth_data': auth_data,
'rsyncpath': rsyncpath,
'verifypath': verifypath,
'netcat': netcat,
'dropcache': dropcache,
}
super(RsyncSource, self).__init__(**kwargs)
self.lockpath = self.znode_path(self.session + '/lock')
self.lock = None
self.path_queue = LockingQueue(
self, self.znode_path(self.session + '/pathQueue'))
self.completed_queue = LockingQueue(
self, self.znode_path(self.session + '/completedQueue'))
self.failed_queue = LockingQueue(
self, self.znode_path(self.session + '/failedQueue'))
self.output_queue = LockingQueue(
self, self.znode_path(self.session + '/outputQueue'))
self.stats_path = '%s/stats' % self.session
self.init_stats()
if rsyncdepth < 0:
self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth)
else:
self.rsyncdepth = rsyncdepth
self.rsync_arbitopts = arbitopts
self.rsync_checksum = checksum
self.rsync_delete = delete
self.rsync_dry = dryrun
self.rsync_hardlinks = hardlinks
self.rsync_inplace = inplace
self.rsync_timeout = timeout
self.rsync_verbose = verbose
self.done_file = done_file
self.excludere = excludere
self.excl_usr = excl_usr
self.rsubpaths = rsubpaths
class RsyncSource(RsyncController):
"""
Class for controlling rsync with Zookeeper.
Builds a tree of paths to devide, and effectively rsyncs the subpaths.
Stops when ready
"""
BASE_PARTIES = RsyncController.BASE_PARTIES + ['sources']
NC_RANGE = 15
SLEEPTIME = 1 # For netcat stub
TIME_OUT = 5 # waiting for destination
WAITTIME = 5 # check interval of closure of other clients
CHECK_WAIT = 20 # wait for path to be available
RSYNC_STATS = [
'Number_of_files', 'Number_of_regular_files_transferred',
'Total_file_size', 'Total_transferred_file_size', 'Literal_data',
'Matched_data', 'File_list_size', 'Total_bytes_sent',
'Total_bytes_received'
]
def __init__(self,
hosts,
session=None,
name=None,
default_acl=None,
auth_data=None,
rsyncpath=None,
rsyncdepth=-1,
rsubpaths=None,
netcat=False,
dryrun=False,
delete=False,
checksum=False,
hardlinks=False,
inplace=False,
verbose=False,
dropcache=False,
timeout=None,
excludere=None,
excl_usr=None,
verifypath=True,
done_file=None,
arbitopts=None):
kwargs = {
'hosts': hosts,
'session': session,
'name': name,
'default_acl': default_acl,
'auth_data': auth_data,
'rsyncpath': rsyncpath,
'verifypath': verifypath,
'netcat': netcat,
'dropcache': dropcache,
}
super(RsyncSource, self).__init__(**kwargs)
self.lockpath = self.znode_path(self.session + '/lock')
self.lock = None
self.path_queue = LockingQueue(
self, self.znode_path(self.session + '/pathQueue'))
self.completed_queue = LockingQueue(
self, self.znode_path(self.session + '/completedQueue'))
self.failed_queue = LockingQueue(
self, self.znode_path(self.session + '/failedQueue'))
self.output_queue = LockingQueue(
self, self.znode_path(self.session + '/outputQueue'))
self.stats_path = '%s/stats' % self.session
self.init_stats()
if rsyncdepth < 0:
self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth)
else:
self.rsyncdepth = rsyncdepth
self.rsync_arbitopts = arbitopts
self.rsync_checksum = checksum
self.rsync_delete = delete
self.rsync_dry = dryrun
self.rsync_hardlinks = hardlinks
self.rsync_inplace = inplace
self.rsync_timeout = timeout
self.rsync_verbose = verbose
self.done_file = done_file
self.excludere = excludere
self.excl_usr = excl_usr
self.rsubpaths = rsubpaths
def init_stats(self):
self.ensure_path(self.znode_path(self.stats_path))
self.counters = {}
for stat in self.RSYNC_STATS:
self.counters[stat] = Counter(
self, self.znode_path('%s/%s' % (self.stats_path, stat)))
def output_stats(self):
self.stats = dict([(k, v.value) for k, v in self.counters.items()])
jstring = json.dumps(self.stats)
self.log.info('progress stats: %s' % jstring)
return jstring
def get_sources(self):
""" Get all zookeeper clients in this session registered as clients """
hosts = []
for host in self.parties['sources']:
hosts.append(host)
return hosts
def acq_lock(self):
""" Try to acquire lock. Returns true if lock is acquired """
self.lock = self.Lock(self.lockpath, "")
return self.lock.acquire(False)
def release_lock(self):
""" Release the acquired lock """
return self.lock.release()
def start_ready_rwatch(self):
""" Start a watch other clients can register to, but release lock and exit on error """
watch = self.start_ready_watch()
if not watch:
if len(self.get_all_hosts()) == 1: # Fix previous unclean shutdown
self.cleanup()
self.release_lock()
self.exit()
return False
else:
return watch
def build_pathqueue(self):
""" Build a queue of paths that needs to be rsynced """
self.log.info('removing old queue and building new queue')
if self.exists(self.path_queue.path):
self.delete(self.path_queue.path, recursive=True)
if self.netcat:
paths = [str(i) for i in range(self.NC_RANGE)]
time.sleep(self.SLEEPTIME)
else:
tuplpaths = get_pathlist(
self.rsyncpath,
self.rsyncdepth,
exclude_re=self.excludere,
# By default don't exclude user files
exclude_usr=self.excl_usr,
rsubpaths=self.rsubpaths)
paths = encode_paths(tuplpaths)
self.paths_total = len(paths)
for path in paths:
self.path_queue.put(path.encode(
)) # Put_all can issue a zookeeper connection error with big lists
self.log.info('pathqueue building finished')
return self.paths_total
def isempty_pathqueue(self):
""" Returns true if all paths in pathqueue are done """
return len(self.path_queue) == 0
def output_progress(self, todo):
self.log.info('Progress: %s of %s paths remaining, %s failed' %
(todo, self.paths_total, len(self.failed_queue)))
self.output_stats()
def output_clients(self, total, sources):
dests = total - sources
sources = sources - 1
self.log.info(
'Connected source (slave) clients: %s, connected destination clients: %s',
sources, dests)
def wait_and_keep_progress(self):
todo_paths = self.paths_total
total_clients = len(self.get_all_hosts())
total_sources = len(self.get_sources())
while not self.isempty_pathqueue():
todo_new = self.len_paths()
if todo_paths != todo_new: # Output progress state
todo_paths = todo_new
self.output_progress(todo_paths)
tot_clients_new = len(self.get_all_hosts())
src_clients_new = len(self.get_sources())
if total_clients != tot_clients_new or total_sources != src_clients_new:
total_clients = tot_clients_new
total_sources = src_clients_new
self.output_clients(total_clients, total_sources)
time.sleep(self.WAITTIME)
def len_paths(self):
""" Returns how many elements still in pathQueue """
return len(self.path_queue)
def shutdown_all(self):
""" Send end signal and release lock
Make sure other clients are disconnected, clean up afterwards."""
self.stop_ready_watch()
self.log.debug('watch set to stop')
while len(self.get_all_hosts()) > 1:
# self.log.debug("clients still connected: %s" % self.get_all_hosts())
time.sleep(self.WAITTIME)
self.cleanup()
def get_state(self):
"""Get the state of a running session"""
remain = self.len_paths()
if remain > 0:
code = 0
self.log.info('Remaining: %s, Failed: %s' %
(remain, len(self.failed_queue)))
else:
code = ZKRS_NO_SUCH_SESSION_EXIT_CODE
self.log.info('No active session')
return code
def write_donefile(self, values):
""" Write a cachefile with some stats about the run when done """
cache_file = FileCache(self.done_file)
cache_file.update('stats', values, 0)
cache_file.close()
def cleanup(self):
""" Remove all session nodes in zookeeper after first logging the output queues """
values = {
'unfinished': len(self.path_queue),
'failed': len(self.failed_queue),
'completed': len(self.completed_queue)
}
while (len(self.path_queue) > 0):
self.log.warning('Unfinished Path %s' %
self.path_queue.get().decode())
self.path_queue.consume()
self.delete(self.dest_queue.path, recursive=True)
self.delete(self.path_queue.path, recursive=True)
self.output_stats()
self.delete(self.znode_path(self.stats_path), recursive=True)
while (len(self.failed_queue) > 0):
self.log.error('Failed Path %s' % self.failed_queue.get().decode())
self.failed_queue.consume()
while (len(self.completed_queue) > 0):
self.log.info('Completed Path %s' %
self.completed_queue.get().decode())
self.completed_queue.consume()
self.log.info('Output:')
while (len(self.output_queue) > 0):
self.log.info(self.output_queue.get().decode())
self.output_queue.consume()
self.delete(self.completed_queue.path, recursive=True)
self.delete(self.failed_queue.path, recursive=True)
self.delete(self.output_queue.path, recursive=True)
self.remove_ready_watch()
self.release_lock()
self.log.info('Cleanup done: Lock, Queues and watch removed')
if self.done_file:
self.write_donefile(values)
def generate_file(self, path):
"""
Writes the relative path used for the rsync of this path,
for use by --files-from.
"""
if not path.startswith(self.rsyncpath):
self.log.raiseException(
'Invalid path! %s is not a subpath of %s!' %
(path, self.rsyncpath))
return None
else:
subpath = path[len(self.rsyncpath):]
subpath = subpath.strip(os.path.sep)
fd, name = tempfile.mkstemp(dir=self.RSDIR, text=True)
wfile = os.fdopen(fd, "w")
wfile.write('%s/' % subpath)
return name
def attempt_run(self, path, attempts=3):
""" Try to run a command x times, on failure add to failed queue """
attempt = 1
while (attempt <= attempts):
dest = self.get_a_dest(attempts) # Keeps it if not consuming
if not dest or not self.basepath_ok():
self.path_queue.put(path.encode(),
priority=50) # Keep path in queue
self.path_queue.consume() # But stop locking it
time.sleep(self.TIME_OUT) # Wait before new attempt
return 1, None
port, host, _ = tuple(dest.split(':', 2))
if self.netcat:
code, output = self.run_netcat(path, host, port)
else:
code, output = self.run_rsync(path, host, port)
if code == 0:
self.completed_queue.put(path.encode())
return code, output
attempt += 1
time.sleep(self.WAITTIME) # Wait before new attempt
self.log.error('There were issues with path %s!' % path)
self.failed_queue.put(path.encode())
return 0, output # otherwise client get stuck
def parse_output(self, output):
"""
Parse the rsync output stats, and when verbose, print the files marked for transmission
"""
if self.rsync_verbose:
outp = output.split("%s%s" % (os.linesep, os.linesep))
self.log.info('Verbose file list output is: %s' % outp[0])
del outp[0]
output = os.linesep.join(outp)
lines = output.splitlines()
for line in lines:
keyval = line.split(':')
if len(keyval) < 2 or keyval[1] == ' ':
self.log.debug('output line not parsed: %s' % line)
continue
key = re.sub(' ', '_', keyval[0])
val = keyval[1].split()[0]
val = re.sub(',', '', val)
if key not in self.RSYNC_STATS:
self.log.debug('output metric not recognised: %s' % key)
continue
self.counters[key] += int(val)
def get_flags(self, files, recursive):
"""
Make an array of flags to be used
"""
# Start rsync recursive or non recursive; archive mode (a) is equivalent to -rlptgoD (see man rsync)
flags = [
'--stats', '--numeric-ids', '-lptgoD',
'--files-from=%s' % files
]
if recursive:
flags.append('-r')
if self.rsync_delete:
flags.append('--delete')
if self.rsync_checksum:
flags.append('--checksum')
if self.rsync_dropcache:
flags.append('--drop-cache')
if self.rsync_hardlinks:
flags.append('--hard-links')
if self.rsync_inplace:
flags.append('--inplace')
if self.rsync_timeout:
flags.extend(['--timeout', str(self.rsync_timeout)])
if self.rsync_verbose:
flags.append('--verbose')
if self.rsync_dry:
flags.append('-n')
# This should always be processed last
if self.rsync_arbitopts:
arbopts = []
for arbopt in self.rsync_arbitopts:
opt = "--%s" % re.sub(':', '=', arbopt, 1)
arbopts.append(opt)
self.log.warning('Adding unchecked flags %s' % ' '.join(arbopts))
flags.extend(arbopts)
return flags
def run_rsync(self, encpath, host, port):
"""
Runs the rsync command with or without recursion, delete or dry-run option.
It uses the destination module linked with this session.
"""
path, recursive = decode_path(encpath)
gfile = self.generate_file(path)
flags = self.get_flags(gfile, recursive)
self.log.info('%s is sending path %s to %s %s' %
(self.whoami, path, host, port))
self.log.debug('Used flags: "%s"' % ' '.join(flags))
command = 'rsync %s %s/ rsync://%s:%s/%s' % (
' '.join(flags), self.rsyncpath, host, port, self.module)
code, output = RunAsyncLoopLog.run(command)
os.remove(gfile)
parsed = self.parse_output(output)
return code, parsed
def run_netcat(self, path, host, port):
""" Test run with netcat """
time.sleep(self.SLEEPTIME)
flags = self.get_flags('nofile', 0)
command = 'echo %s is sending %s with flags "%s" | nc %s %s' % (
self.whoami, path, ' '.join(flags), host, port)
return RunAsyncLoopLog.run(command)
def rsync_path(self, path):
""" start rsync session for given path and destination, returns true if successful """
if not path:
self.log.raiseException('Empty path given!')
return None
elif not is_string(path):
self.log.raiseException('Invalid path: %s !' % path)
return None
else:
code, output = self.attempt_run(path)
if output:
self.output_queue.put(output.encode())
return (code == 0)
def get_a_dest(self, attempts):
""" Try to get a destination x times """
attempt = 1
while (attempt <= attempts):
dest = self.try_a_dest(self.TIME_OUT) # Keeps it if not consuming
if dest: # We locked a rsync daemon
self.log.debug('Got destination %s' % dest)
return dest
attempt += 1
time.sleep(self.WAITTIME)
self.log.warning('Still no destination after %s tries' % attempts)
return None
def dest_is_sane(self, dest):
""" Lock destination state, fetch it and disable if paused """
return self.dest_state(dest, self.STATUS)
def handle_dest_state(self, dest, destpath, current_state):
"""
Disable destination by consuming it from queue when paused and return false
If destination is active, return true
"""
if current_state == self.STATE_PAUSED:
self.set_znode(destpath, self.STATE_DISABLED)
self.dest_queue.consume()
self.log.debug(
'Destination %s was disabled and removed from queue' % dest)
return False
elif current_state == self.STATE_ACTIVE:
return True
else:
self.log.error('Destination %s is in an unknown state %s' %
(dest, current_state))
return False
def try_a_dest(self, timeout):
"""
Try to get a destination.
check if destination is still running, otherwise remove
"""
if len(self.dest_queue) == 0:
self.log.debug('Destinations not yet available')
dest = self.dest_queue.get(timeout).decode()
if dest:
port, whoami = tuple(dest.split(':', 1))
if not self.member_of_party(whoami, 'allsd'):
self.log.debug('destination is not found in party')
self.dest_queue.consume()
return None
elif not self.dest_is_sane(whoami):
self.log.debug('recieved destination was paused')
return None
else:
portmap = '%s/portmap/%s' % (self.session, whoami)
lport, _ = self.get_znode(portmap)
if port != lport:
self.log.error(
'destination port not valid') # Should not happen
self.dest_queue.consume()
return None
return dest
def rsync(self, timeout=None):
""" Get a destination, a path and call a new rsync iteration """
if self.verifypath:
if not self.basepath_ok():
self.log.warning('Basepath not available, waiting')
time.sleep(self.CHECK_WAIT)
return None
path = self.path_queue.get(timeout).decode()
if path:
if self.rsync_path(path):
self.path_queue.consume()
return None
def feeds():
try:
conn = psycopg2.connect(cfg.pgConn)
cur = conn.cursor()
#zkHost = "ZKHOST1234"
zk = KazooClient(hosts=cfg.zkCluster)
zk.start()
# 4 queues
aliasq = LockingQueue(zk, cfg.alias)
symbolq = LockingQueue(zk, cfg.symbol)
issueq = LockingQueue(zk, cfg.issue)
transferq = LockingQueue(zk, cfg.transfer)
# 2 more queues
propIssueq = LockingQueue(zk, cfg.proposeIssue)
propTranq = LockingQueue(zk, cfg.proposeTran)
cur.execute("""select min(id) from alias_redo where progress = 1""")
res = cur.fetchone()
conn.commit()
if res[0]:
minAlias = res[0] - 1
else:
minAlias = -1
cur.execute("""select min(id) from symbol_redo where progress = 1 """)
res = cur.fetchone()
conn.commit()
if res[0]:
minSymbol = res[0] - 1
else:
minSymbol = -1
cur.execute("""select min(id) from issue_redo where progress = 1 """)
res = cur.fetchone()
conn.commit()
if res[0]:
minIssue = res[0] - 1
else:
minIssue = -1
cur.execute("""select min(id) from transfer_redo where progress = 1 """)
res = cur.fetchone()
conn.commit()
if res[0]:
minTransfer = res[0] - 1
else:
minTransfer = -1
cur.execute("""select min(id) from propose_issue where progress = 1""")
res = cur.fetchone()
conn.commit()
if res[0]:
minpropIssue = res[0] - 1
else:
minpropIssue = -1
cur.execute("""select min(id) from propose_transfer where progress = 1""")
res = cur.fetchone()
conn.commit()
if res[0]:
minpropTran = res[0] - 1
else:
minpropTran = -1
while True:
ints = datetime.now()
inload = os.getloadavg()[0]
# process alias
cur.execute("""select alias, "globalId" , id from alias_redo where progress = 1
and id > %s""", [minAlias])
res = cur.fetchall()
conn.commit()
if res:
for en in res:
aliasq.put("{0}||{1}".format(en[0].strip(), en[1].strip()).encode('utf-8'))
minAlias = en[2]
# process symbol
cur.execute("""select symbol, "globalId" , id from symbol_redo where progress = 1
and id > %s""", [minSymbol])
res = cur.fetchall()
conn.commit()
if res:
for en in res:
symbolq.put("{0}||{1}".format(en[0].strip(), en[1].strip()).encode('utf-8'))
minSymbol = en[2]
# process issue
cur.execute("""select proposal ,id from issue_redo where progress = 1
and id > %s""", [minIssue])
res = cur.fetchall()
conn.commit()
if res:
for en in res:
issueq.put("{0}".format(en[0].strip()).encode('utf-8'))
minIssue = en[1]
# process transfer
cur.execute("""select proposal,id from transfer_redo where progress = 1
and id > %s""", [minTransfer])
res = cur.fetchall()
conn.commit()
if res:
for en in res:
transferq.put("{0}".format(en[0].strip()).encode('utf-8'))
minTransfer = en[1]
# process proposeIssue
cur.execute("""select symbol, quantity, "globalId",id from propose_issue where progress = 1 and id > %s
""", [minpropIssue])
res = cur.fetchall()
conn.commit()
if res:
for en in res:
if not en[1] in [1, 2, 8]:
continue
propIssueq.put("{0}||{1}||{2}||{3}".format(en[0].strip(), en[1], en[2].strip(), en[3]).encode('utf-8'))
minpropIssue = en[3]
# process proposeTransfer
cur.execute("""select alias, "rawCode", lastsig , id, "globalId" from propose_transfer where progress = 1
and id > %s""", [minpropTran])
res = cur.fetchall()
conn.commit()
if res:
for en in res:
propTranq.put("{0}&&{1}&&{2}&&{3}&&{4}".format(en[0].strip(), en[1].strip(), en[2].strip(), en[3], en[4].strip()).encode('utf-8'))
minpropTran = en[3]
outts = datetime.now()
outload = os.getloadavg()[0]
# stmt = SimpleStatement("""insert into runstat(id,executable,ints,inload,outts,outload) values (%s, %s, %s, %s, %s, %s)""", consistency_level=ConsistencyLevel.ANY)
# cluster = Cluster(cfg.cassCluster)
# session = cluster.connect(cfg.cassKeyspace)
# session.execute(stmt, (uuid.uuid4(), executable, ints, inload, outts, outload))
time.sleep(1)
except psycopg2.Error as err:
print("SQLError {0}".format(err))
finally:
zk.stop()
zk.close()
cur.close()
conn.close()
