def __init__(self, hosts, session=None, name=None, default_acl=None, auth_data=None, rsyncpath=None, rsyncdepth=-1, netcat=False, dryrun=False, delete=False, excludere=None): kwargs = { 'hosts' : hosts, 'session' : session, 'name' : name, 'default_acl' : default_acl, 'auth_data' : auth_data, 'rsyncpath' : rsyncpath, 'netcat' : netcat } super(RsyncSource, self).__init__(**kwargs) self.lockpath = self.znode_path(self.session + '/lock') self.lock = None self.path_queue = LockingQueue(self, self.znode_path(self.session + '/pathQueue')) self.completed_queue = LockingQueue(self, self.znode_path(self.session + '/completedQueue')) self.failed_queue = LockingQueue(self, self.znode_path(self.session + '/failedQueue')) self.output_queue = LockingQueue(self, self.znode_path(self.session + '/outputQueue')) if rsyncdepth < 0: self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth) else: self.rsyncdepth = rsyncdepth self.rsync_delete = delete self.rsync_dry = dryrun self.excludere = excludere
def processTransfer(): try: conn = psycopg2.connect(dbConnectStr) cur = conn.cursor() zk = KazooClient(hosts=zkHost) zk.start() transferq = LockingQueue(zk, '/transfer/') while True: rawCode = transferq.get() proposal = rawCode.decode().strip() transferq.consume() # print(" proposal = {0} ".format(proposal)) ints = datetime.now() inload = os.getloadavg()[0] pro1 = Popen(['/usr/bin/python36', './processproptran.py', proposal], stdin=None, stdout=None) pro1.wait() outts = datetime.now() outload = os.getloadavg()[0] # insert the runtime info into c* cluster = Cluster(cfg.cassCluster) session = cluster.connect(cfg.cassKeyspace) stmt = SimpleStatement("""insert into runstat(id,executable,ints,inload,outts,outload) values (%s, %s, %s, %s, %s, %s)""", consistency_level=ConsistencyLevel.ANY) session.execute(stmt, (uuid.uuid4(), executable, ints, inload, outts, outload)) except psycopg2.Error as err: print("SQLError {0}".format(err)) finally: zk.stop() zk.close() cur.close() conn.close()
def _work(queue_name, function, resource_name, lock_timeout=10): logger = setup_logger(queue_name) logger.info("Worker Started!") zookeeper_client = get_zookeeper_client() queue = LockingQueue(zookeeper_client, queue_name) while True: try: message = get_next_message(queue) except Exception as e: logger.exception( "Error while retrieving message from queue: {}".format(e) ) try: with get_resource_lock( resource_name, zookeeper_client, message['name'], timeout=lock_timeout ): if queue.consume(): try: function(message, logger) except Exception as e: logger.exception( "Function raised exception: {}".format(e) ) except exceptions.LockTimeout: logger.info("LockTimeout on {}".format(message['name'])) except Exception as e: logger.exception("Exception: {}".format(e))
def processSymbol(): try: conn = psycopg2.connect(dbConnectStr) cur = conn.cursor() zk = KazooClient(hosts=zkHost) zk.start() symbolq = LockingQueue(zk, cfg.symbol) while True: rawCode = symbolq.get() ints = datetime.now() inload = os.getloadavg()[0] symbol = rawCode.decode().split('||')[0] globalId = rawCode.decode().split('||')[1] symbolq.consume() alias = '' while not alias: print("loop for the alias of {0}".format(globalId)) cur.execute(""" select alias from player0 where "globalId" = %s """, [globalId]) res = cur.fetchone() conn.commit() if res: alias = res[0] print("process symbol:{0} alias:{1} globalId:{2}".format(symbol, alias, globalId)) lock0 = zk.Lock(symbol, 'jg') with lock0: # the operation cmd = "cd /{4}/;{3}/openssl-1.0.2o/apps/openssl genrsa 2048| {3}/openssl-1.0.2o/apps/openssl asn1parse|/{4}/parseoutput.pl; /usr/bin/perl /{4}/makeissuer.pl '{0}' '{1}' '{2}'".format(alias, symbol, globalId, baseDir, workshopInstance) f = os.popen(cmd) while True: en = f.readline() if en == '': break cur.execute(""" update symbol_redo set progress= 0, setup=now() where symbol=%s """, [symbol]) conn.commit() outts = datetime.now() outload = os.getloadavg()[0] # insert the runtime info into c* cluster = Cluster(cfg.cassCluster) session = cluster.connect(cfg.cassKeyspace) stmt = SimpleStatement("""insert into runstat(id,executable,ints,inload,outts,outload) values (%s, %s, %s, %s, %s, %s)""", consistency_level=ConsistencyLevel.ANY) session.execute(stmt, (uuid.uuid4(), executable, ints, inload, outts, outload)) except psycopg2.Error as err: print("SQLError {0}".format(err)) finally: zk.stop() zk.close() cur.close() conn.close()
def __init__(self, hosts, session=None, name=None, default_acl=None, auth_data=None, rsyncpath=None, netcat=None, verifypath=True, dropcache=False): kwargs = { 'hosts' : hosts, 'session' : session, 'name' : name, 'default_acl' : default_acl, 'auth_data' : auth_data, } self.netcat = netcat super(RsyncController, self).__init__(**kwargs) self.rsyncpath = rsyncpath.rstrip(os.path.sep) if not netcat: if verifypath and not self.basepath_ok(): self.log.raiseException('Path does not exists in filesystem: %s' % rsyncpath) if not os.path.isdir(self.RSDIR): os.mkdir(self.RSDIR, 0o700) self.module = 'zkrs-%s' % self.session self.dest_queue = LockingQueue(self, self.znode_path(self.session + '/destQueue')) self.verifypath = verifypath self.rsync_dropcache = dropcache
def __init__(self, hosts, session=None, name=None, default_acl=None, auth_data=None, rsyncpath=None, rsyncdepth=-1, rsubpaths=None, netcat=False, dryrun=False, delete=False, checksum=False, hardlinks=False, verbose=False, dropcache=False, timeout=None, excludere=None, excl_usr=None, verifypath=True, done_file=None, arbitopts=None): kwargs = { 'hosts' : hosts, 'session' : session, 'name' : name, 'default_acl' : default_acl, 'auth_data' : auth_data, 'rsyncpath' : rsyncpath, 'verifypath' : verifypath, 'netcat' : netcat, 'dropcache' : dropcache, } super(RsyncSource, self).__init__(**kwargs) self.lockpath = self.znode_path(self.session + '/lock') self.lock = None self.path_queue = LockingQueue(self, self.znode_path(self.session + '/pathQueue')) self.completed_queue = LockingQueue(self, self.znode_path(self.session + '/completedQueue')) self.failed_queue = LockingQueue(self, self.znode_path(self.session + '/failedQueue')) self.output_queue = LockingQueue(self, self.znode_path(self.session + '/outputQueue')) self.stats_path = '%s/stats' % self.session self.init_stats() if rsyncdepth < 0: self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth) else: self.rsyncdepth = rsyncdepth self.rsync_arbitopts = arbitopts self.rsync_checksum = checksum self.rsync_delete = delete self.rsync_dry = dryrun self.rsync_hardlinks = hardlinks self.rsync_timeout = timeout self.rsync_verbose = verbose self.done_file = done_file self.excludere = excludere self.excl_usr = excl_usr self.rsubpaths = rsubpaths
def wrapper(): try: conn = psycopg2.connect(dbConnectStr) cur = conn.cursor() zk = KazooClient(hosts=zkHost) zk.start() propTranq = LockingQueue(zk, cfg.proposeTran) while True: rawC = propTranq.get() ints = datetime.now() inload = os.getloadavg()[0] alias = rawC.decode().split('&&')[0] rawCode = rawC.decode().split('&&')[1] lastsig = rawC.decode().split('&&')[2] entryId = rawC.decode().split('&&')[3] globalId = rawC.decode().split('&&')[4] pro1 = Popen(['/usr/bin/python36','/{0}/payer{1}.py'.format(workshopInstance, alias), rawCode, lastsig, globalId], stdin=None, stdout=None) pro1.wait() cur.execute("""update propose_transfer set progress = 0 , setup = now() where id = %s """, [entryId]) conn.commit() propTranq.consume() outts = datetime.now() outload = os.getloadavg()[0] # insert the runtime info into c* cluster = Cluster(cfg.cassCluster) session = cluster.connect(cfg.cassKeyspace) stmt = SimpleStatement("""insert into runstat(id,executable,ints,inload,outts,outload) values (%s, %s, %s, %s, %s, %s)""", consistency_level=ConsistencyLevel.ANY) session.execute(stmt, (uuid.uuid4(), executable, ints, inload, outts, outload)) except psycopg2.Error as err: print("SQLError {0}".format(err)) finally: zk.stop() zk.close() cur.close() conn.close()
# Start client # read_only mode must be turned on for the servers in the Zookeeper # cluster for the client to utilize it. zk = KazooClient(hosts='127.0.0.1:2181') zk.start() print("Client started!\n") # Clear the nodes at the end or else you migth get a NodeExistsError next time zk.delete("/app", recursive=True) # Ensure a path, create if it doesn't exist zk.ensure_path("/app") # Initialize queue at specified location zk_queue = LockingQueue(zk, "/app/queue") # Thread handler handler = SequentialThreadingHandler() # Watchers --------------------------------------------------------------- @zk.DataWatch("/app/queue") def watch_queue(data, stat): if data == "": pass elif data is not None or stat is not None: global input_done # print("Queue Node:") # print("%s\n" % data)
def processAlias(): try: conn = psycopg2.connect(dbConnectStr) cur = conn.cursor() zk = KazooClient(hosts=zkHost) zk.start() aliasq = LockingQueue(zk, cfg.alias) while True: rawCode = aliasq.get() ints = datetime.now() inload = os.getloadavg()[0] alias = rawCode.decode().split('||')[0] globalId = rawCode.decode().split('||')[1] aliasq.consume() print("process alias:{0} globalId:{1}".format(alias, globalId)) lock0 = zk.Lock(alias, 'jg') with lock0: # the operation cmd = "cd /{3}/; {2}/openssl-1.0.2o/apps/openssl genrsa 2048| {2}/openssl-1.0.2o/apps/openssl asn1parse|/{3}/parseoutput.pl; /usr/bin/perl /{3}/makepayer.pl '{0}' '{1}'".format(alias, globalId, baseDir, workshopInstance) f = os.popen(cmd) while True: en = f.readline() if en == '': break cur.execute(""" update alias_redo set progress= 0, setup=now() where alias=%s """, [alias]) conn.commit() # insert the entry to symbol_redo sha256 = hashlib.sha256() while True: sha256.update('{0}'.format(globalId).encode('utf-8')) sha256.update('{0}'.format(alias).encode('utf-8')) hashCode = sha256.hexdigest() symbol = hashCode[:5] symbol = symbol.upper() cur.execute(""" select symbol from symbol_redo where symbol = %s union select symbol from issuer0 where symbol = %s """, [symbol, symbol]) res = cur.fetchone() conn.commit() if not res: break cur.execute(""" insert into symbol_redo("globalId",symbol,setup,progress) values(%s,%s,now(), 1) """, [globalId, symbol]) conn.commit() # insert the runtime_stat to c* outload = os.getloadavg()[0] outts = datetime.now() stmt = SimpleStatement("""insert into runstat(id,executable,ints,inload,outts,outload) values (%s, %s, %s, %s, %s, %s)""", consistency_level=ConsistencyLevel.ANY) cluster = Cluster(cfg.cassCluster) session = cluster.connect(cfg.cassKeyspace) session.execute(stmt, (uuid.uuid4(), executable, ints, inload, outts, outload)) except psycopg2.Error as err: print("SQLError {0}".format(err)) finally: zk.stop() zk.close() cur.close() conn.close()
class RsyncSource(RsyncController): """ Class for controlling rsync with Zookeeper. Builds a tree of paths to devide, and effectively rsyncs the subpaths. Stops when ready """ BASE_PARTIES = RsyncController.BASE_PARTIES + ['sources'] NC_RANGE = 15 SLEEPTIME = 1 # For netcat stub TIME_OUT = 5 # waiting for destination WAITTIME = 5 # check interval of closure of other clients def __init__(self, hosts, session=None, name=None, default_acl=None, auth_data=None, rsyncpath=None, rsyncdepth=-1, netcat=False, dryrun=False, delete=False, excludere=None): kwargs = { 'hosts' : hosts, 'session' : session, 'name' : name, 'default_acl' : default_acl, 'auth_data' : auth_data, 'rsyncpath' : rsyncpath, 'netcat' : netcat } super(RsyncSource, self).__init__(**kwargs) self.lockpath = self.znode_path(self.session + '/lock') self.lock = None self.path_queue = LockingQueue(self, self.znode_path(self.session + '/pathQueue')) self.completed_queue = LockingQueue(self, self.znode_path(self.session + '/completedQueue')) self.failed_queue = LockingQueue(self, self.znode_path(self.session + '/failedQueue')) self.output_queue = LockingQueue(self, self.znode_path(self.session + '/outputQueue')) if rsyncdepth < 0: self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth) else: self.rsyncdepth = rsyncdepth self.rsync_delete = delete self.rsync_dry = dryrun self.excludere = excludere def get_sources(self): """ Get all zookeeper clients in this session registered as clients """ hosts = [] for host in self.parties['sources']: hosts.append(host) return hosts def acq_lock(self): """ Try to acquire lock. Returns true if lock is acquired """ self.lock = self.Lock(self.lockpath, "") return self.lock.acquire(False) def release_lock(self): """ Release the acquired lock """ return self.lock.release() def start_ready_rwatch(self): """ Start a watch other clients can register to, but release lock and exit on error """ watch = self.start_ready_watch() if not watch: if len(self.get_all_hosts()) == 1: # Fix previous unclean shutdown self.cleanup() self.release_lock() self.exit() return False else: return watch def build_pathqueue(self): """ Build a queue of paths that needs to be rsynced """ self.log.debug('removing old queue and building new queue') if self.exists(self.path_queue.path): self.delete(self.path_queue.path, recursive=True) if self.netcat: paths = [str(i) for i in range(self.NC_RANGE)] time.sleep(self.SLEEPTIME) else: tuplpaths = get_pathlist(self.rsyncpath, self.rsyncdepth, exclude_re=self.excludere, exclude_usr='******') # Don't exclude user files paths = encode_paths(tuplpaths) self.paths_total = len(paths) for path in paths: self.path_queue.put(path) # Put_all can issue a zookeeper connection error with big lists self.log.debug('pathqueue building finished') return self.paths_total def isempty_pathqueue(self): """ Returns true if all paths in pathqueue are done """ return len(self.path_queue) == 0 def len_paths(self): """ Returns how many elements still in pathQueue """ return len(self.path_queue) def shutdown_all(self): """ Send end signal and release lock Make sure other clients are disconnected, clean up afterwards.""" self.stop_ready_watch() self.log.debug('watch set to stop') while len(self.get_all_hosts()) > 1: # self.log.debug("clients still connected: %s" % self.get_all_hosts()) time.sleep(self.WAITTIME) self.cleanup() def cleanup(self): """ Remove all session nodes in zookeeper after first logging the output queues """ self.delete(self.dest_queue.path, recursive=True) self.delete(self.path_queue.path, recursive=True) while (len(self.failed_queue) > 0): self.log.error('Failed Path %s' % self.failed_queue.get()) self.failed_queue.consume() while (len(self.completed_queue) > 0): self.log.info('Completed Path %s' % self.completed_queue.get()) self.completed_queue.consume() self.log.info('Output:') while (len(self.output_queue) > 0): self.log.info(self.output_queue.get()) self.output_queue.consume() self.delete(self.completed_queue.path, recursive=True) self.delete(self.failed_queue.path, recursive=True) self.delete(self.output_queue.path, recursive=True) self.remove_ready_watch() self.release_lock() self.log.debug('Lock, Queues and watch removed') def generate_file(self, path): """ Writes the relative path used for the rsync of this path, for use by --files-from. """ if not path.startswith(self.rsyncpath): self.log.raiseException('Invalid path! %s is not a subpath of %s!' % (path, self.rsyncpath)) else: subpath = path[len(self.rsyncpath):] subpath = subpath.strip(os.path.sep) fd, name = tempfile.mkstemp(dir=self.RSDIR, text=True) file = os.fdopen(fd, "w") file.write('%s/' % subpath) return name def attempt_run(self, path, attempts=3): """ Try to run a command x times, on failure add to failed queue """ attempt = 1 while (attempt <= attempts): dest = self.get_a_dest(self.TIME_OUT) # Keeps it if not consuming if dest: # We locked a rsync daemon self.log.debug('Got destination %s' % dest) port, host, other = tuple(dest.split(':', 2)) if self.netcat: code, output = self.run_netcat(path, host, port) else: code, output = self.run_rsync(path, host, port) if code == 0: self.completed_queue.put(path) return code, output attempt += 1 time.sleep(self.WAITTIME) # Wait before new attempt if dest: self.log.error('There were issues with path %s!' % path) self.failed_queue.put(path) return 0, output # otherwise client get stuck else: self.log.debug('Still no destination after %s tries' % attempts) self.path_queue.put(path, priority=50) # Keep path in queue self.path_queue.consume() # But stop locking it time.sleep(self.TIME_OUT) # Wait before new attempt return 1, None def run_rsync(self, encpath, host, port): """ Runs the rsync command with or without recursion, delete or dry-run option. It uses the destination module linked with this session. """ path, recursive = decode_path(encpath) file = self.generate_file(path) # Start rsync recursive or non recursive; archive mode (a) is equivalent to -rlptgoD (see man rsync) flags = ['--stats', '--numeric-ids', '-lptgoD', '--files-from=%s' % file] if recursive: flags.append('-r') if self.rsync_delete: flags.append('--delete') if self.rsync_dry: flags.append('-n') self.log.debug('echo %s is sending %s to %s %s' % (self.whoami, path, host, port)) command = 'rsync %s %s/ rsync://%s:%s/%s' % (' '.join(flags), self.rsyncpath, host, port, self.module) code, output = RunAsyncLoopLog.run(command) os.remove(file) return code, output def run_netcat(self, path, host, port): """ Test run with netcat """ time.sleep(self.SLEEPTIME) command = 'echo %s is sending %s | nc %s %s' % (self.whoami, path, host, port) return RunAsyncLoopLog.run(command) def rsync_path(self, path): """ start rsync session for given path and destination, returns true if successful """ if not path: self.log.raiseException('Empty path given!') elif not isinstance(path, basestring): self.log.raiseException('Invalid path: %s !' % path) else: code, output = self.attempt_run(path) if output: self.output_queue.put(output) return (code == 0) def get_a_dest(self, timeout): """ Try to get a destination. check if destination is still running, otherwise remove """ if len(self.dest_queue) == 0: self.log.debug('Destinations not yet available') dest = self.dest_queue.get(timeout) if dest: port, whoami = tuple(dest.split(':', 1)) if not self.member_of_party(whoami, 'allsd'): self.log.debug('destination is not found in party') self.dest_queue.consume() return None else: portmap = '%s/portmap/%s' % (self.session, whoami) lport, stat = self.get_znode(portmap) if port != lport: self.log.error('destination port not valid') # Should not happen self.dest_queue.consume() return None return dest def rsync(self, timeout=None): """ Get a destination, a path and call a new rsync iteration """ path = self.path_queue.get(timeout) if path: if self.rsync_path(path): self.path_queue.consume()
class RsyncSource(RsyncController): """ Class for controlling rsync with Zookeeper. Builds a tree of paths to devide, and effectively rsyncs the subpaths. Stops when ready """ BASE_PARTIES = RsyncController.BASE_PARTIES + ['sources'] NC_RANGE = 15 SLEEPTIME = 1 # For netcat stub TIME_OUT = 5 # waiting for destination WAITTIME = 5 # check interval of closure of other clients CHECK_WAIT = 20 # wait for path to be available RSYNC_STATS = ['Number_of_files', 'Number_of_files_transferred', 'Total_file_size', 'Total_transferred_file_size', 'Literal_data', 'Matched_data', 'File_list_size', 'Total_bytes_sent', 'Total_bytes_received']; def __init__(self, hosts, session=None, name=None, default_acl=None, auth_data=None, rsyncpath=None, rsyncdepth=-1, rsubpaths=None, netcat=False, dryrun=False, delete=False, checksum=False, hardlinks=False, verbose=False, dropcache=False, timeout=None, excludere=None, excl_usr=None, verifypath=True, done_file=None, arbitopts=None): kwargs = { 'hosts' : hosts, 'session' : session, 'name' : name, 'default_acl' : default_acl, 'auth_data' : auth_data, 'rsyncpath' : rsyncpath, 'verifypath' : verifypath, 'netcat' : netcat, 'dropcache' : dropcache, } super(RsyncSource, self).__init__(**kwargs) self.lockpath = self.znode_path(self.session + '/lock') self.lock = None self.path_queue = LockingQueue(self, self.znode_path(self.session + '/pathQueue')) self.completed_queue = LockingQueue(self, self.znode_path(self.session + '/completedQueue')) self.failed_queue = LockingQueue(self, self.znode_path(self.session + '/failedQueue')) self.output_queue = LockingQueue(self, self.znode_path(self.session + '/outputQueue')) self.stats_path = '%s/stats' % self.session self.init_stats() if rsyncdepth < 0: self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth) else: self.rsyncdepth = rsyncdepth self.rsync_arbitopts = arbitopts self.rsync_checksum = checksum self.rsync_delete = delete self.rsync_dry = dryrun self.rsync_hardlinks = hardlinks self.rsync_timeout = timeout self.rsync_verbose = verbose self.done_file = done_file self.excludere = excludere self.excl_usr = excl_usr self.rsubpaths = rsubpaths def init_stats(self): self.ensure_path(self.znode_path(self.stats_path)) self.counters = {}; for stat in self.RSYNC_STATS: self.counters[stat] = Counter(self, self.znode_path('%s/%s' % (self.stats_path, stat))) def output_stats(self): self.stats = dict([(k, v.value) for k, v in self.counters.items()]) jstring = json.dumps(self.stats) self.log.info('progress stats: %s' % jstring) return jstring def get_sources(self): """ Get all zookeeper clients in this session registered as clients """ hosts = [] for host in self.parties['sources']: hosts.append(host) return hosts def acq_lock(self): """ Try to acquire lock. Returns true if lock is acquired """ self.lock = self.Lock(self.lockpath, "") return self.lock.acquire(False) def release_lock(self): """ Release the acquired lock """ return self.lock.release() def start_ready_rwatch(self): """ Start a watch other clients can register to, but release lock and exit on error """ watch = self.start_ready_watch() if not watch: if len(self.get_all_hosts()) == 1: # Fix previous unclean shutdown self.cleanup() self.release_lock() self.exit() return False else: return watch def build_pathqueue(self): """ Build a queue of paths that needs to be rsynced """ self.log.info('removing old queue and building new queue') if self.exists(self.path_queue.path): self.delete(self.path_queue.path, recursive=True) if self.netcat: paths = [str(i) for i in range(self.NC_RANGE)] time.sleep(self.SLEEPTIME) else: tuplpaths = get_pathlist(self.rsyncpath, self.rsyncdepth, exclude_re=self.excludere, exclude_usr=self.excl_usr, rsubpaths=self.rsubpaths) # By default don't exclude user files paths = encode_paths(tuplpaths) self.paths_total = len(paths) for path in paths: self.path_queue.put(path) # Put_all can issue a zookeeper connection error with big lists self.log.info('pathqueue building finished') return self.paths_total def isempty_pathqueue(self): """ Returns true if all paths in pathqueue are done """ return len(self.path_queue) == 0 def output_progress(self, todo): self.log.info('Progress: %s of %s paths remaining, %s failed' % (todo, self.paths_total, len(self.failed_queue))) self.output_stats() def output_clients(self, total, sources): dests = total - sources sources = sources - 1 self.log.info('Connected source (slave) clients: %s, connected destination clients: %s', sources, dests) def wait_and_keep_progress(self): todo_paths = self.paths_total total_clients = len(self.get_all_hosts()) total_sources = len(self.get_sources()) while not self.isempty_pathqueue(): todo_new = self.len_paths() if todo_paths != todo_new: # Output progress state todo_paths = todo_new self.output_progress(todo_paths) tot_clients_new = len(self.get_all_hosts()) src_clients_new = len(self.get_sources()) if total_clients != tot_clients_new or total_sources != src_clients_new: total_clients = tot_clients_new total_sources = src_clients_new self.output_clients(total_clients, total_sources); time.sleep(self.WAITTIME) def len_paths(self): """ Returns how many elements still in pathQueue """ return len(self.path_queue) def shutdown_all(self): """ Send end signal and release lock Make sure other clients are disconnected, clean up afterwards.""" self.stop_ready_watch() self.log.debug('watch set to stop') while len(self.get_all_hosts()) > 1: # self.log.debug("clients still connected: %s" % self.get_all_hosts()) time.sleep(self.WAITTIME) self.cleanup() def get_state(self): """Get the state of a running session""" remain = self.len_paths() if remain > 0: code = 0 self.log.info('Remaining: %s, Failed: %s' % (remain, len(self.failed_queue))) else: code = ZKRS_NO_SUCH_SESSION_EXIT_CODE self.log.info('No active session') return code def write_donefile(self, values): """ Write a cachefile with some stats about the run when done """ cache_file = FileCache(self.done_file) cache_file.update('stats', values, 0) cache_file.close() def cleanup(self): """ Remove all session nodes in zookeeper after first logging the output queues """ values = { 'unfinished' : len(self.path_queue), 'failed' : len(self.path_queue), 'completed' : len(self.completed_queue) } while (len(self.path_queue) > 0): self.log.warning('Unfinished Path %s' % self.path_queue.get()) self.path_queue.consume() self.delete(self.dest_queue.path, recursive=True) self.delete(self.path_queue.path, recursive=True) self.output_stats() self.delete(self.znode_path(self.stats_path), recursive=True) while (len(self.failed_queue) > 0): self.log.error('Failed Path %s' % self.failed_queue.get()) self.failed_queue.consume() while (len(self.completed_queue) > 0): self.log.info('Completed Path %s' % self.completed_queue.get()) self.completed_queue.consume() self.log.info('Output:') while (len(self.output_queue) > 0): self.log.info(self.output_queue.get()) self.output_queue.consume() self.delete(self.completed_queue.path, recursive=True) self.delete(self.failed_queue.path, recursive=True) self.delete(self.output_queue.path, recursive=True) self.remove_ready_watch() self.release_lock() self.log.info('Cleanup done: Lock, Queues and watch removed') if self.done_file: self.write_donefile(values) def generate_file(self, path): """ Writes the relative path used for the rsync of this path, for use by --files-from. """ if not path.startswith(self.rsyncpath): self.log.raiseException('Invalid path! %s is not a subpath of %s!' % (path, self.rsyncpath)) else: subpath = path[len(self.rsyncpath):] subpath = subpath.strip(os.path.sep) fd, name = tempfile.mkstemp(dir=self.RSDIR, text=True) wfile = os.fdopen(fd, "w") wfile.write('%s/' % subpath) return name def attempt_run(self, path, attempts=3): """ Try to run a command x times, on failure add to failed queue """ attempt = 1 while (attempt <= attempts): dest = self.get_a_dest(attempts) # Keeps it if not consuming if not dest or not self.basepath_ok(): self.path_queue.put(path, priority=50) # Keep path in queue self.path_queue.consume() # But stop locking it time.sleep(self.TIME_OUT) # Wait before new attempt return 1, None port, host, _ = tuple(dest.split(':', 2)) if self.netcat: code, output = self.run_netcat(path, host, port) else: code, output = self.run_rsync(path, host, port) if code == 0: self.completed_queue.put(path) return code, output attempt += 1 time.sleep(self.WAITTIME) # Wait before new attempt self.log.error('There were issues with path %s!' % path) self.failed_queue.put(path) return 0, output # otherwise client get stuck def parse_output(self, output): """ Parse the rsync output stats, and when verbose, print the files marked for transmission """ if self.rsync_verbose: outp = output.split("%s%s" % (os.linesep, os.linesep)) self.log.info('Verbose file list output is: %s' % outp[0]) del outp[0] output = os.linesep.join(outp) lines = output.splitlines() for line in lines: keyval = line.split(':') if len(keyval) < 2 or keyval[1] == ' ': self.log.debug('output line not parsed: %s' % line) continue key = re.sub(' ', '_', keyval[0]) val = keyval[1].split()[0] val = re.sub(',', '', val) if key not in self.RSYNC_STATS: self.log.debug('output metric not recognised: %s' % key) continue self.counters[key] += int(val) def get_flags(self, files, recursive): """ Make an array of flags to be used """ # Start rsync recursive or non recursive; archive mode (a) is equivalent to -rlptgoD (see man rsync) flags = ['--stats', '--numeric-ids', '-lptgoD', '--files-from=%s' % files] if recursive: flags.append('-r') if self.rsync_delete: flags.append('--delete') if self.rsync_checksum: flags.append('--checksum') if self.rsync_dropcache: flags.append('--drop-cache') if self.rsync_hardlinks: flags.append('--hard-links') if self.rsync_timeout: flags.extend(['--timeout', str(self.rsync_timeout)]) if self.rsync_verbose: flags.append('--verbose') if self.rsync_dry: flags.append('-n') # This should always be processed last if self.rsync_arbitopts: arbopts = [] for arbopt in self.rsync_arbitopts: opt = "--%s" % re.sub(':', '=', arbopt, 1) arbopts.append(opt) self.log.warning('Adding unchecked flags %s' % ' '.join(arbopts)) flags.extend(arbopts) return flags def run_rsync(self, encpath, host, port): """ Runs the rsync command with or without recursion, delete or dry-run option. It uses the destination module linked with this session. """ path, recursive = decode_path(encpath) gfile = self.generate_file(path) flags = self.get_flags(gfile, recursive) self.log.info('%s is sending path %s to %s %s' % (self.whoami, path, host, port)) self.log.debug('Used flags: "%s"' % ' '.join(flags)) command = 'rsync %s %s/ rsync://%s:%s/%s' % (' '.join(flags), self.rsyncpath, host, port, self.module) code, output = RunAsyncLoopLog.run(command) os.remove(gfile) parsed = self.parse_output(output) return code, parsed def run_netcat(self, path, host, port): """ Test run with netcat """ time.sleep(self.SLEEPTIME) flags = self.get_flags('nofile', 0) command = 'echo %s is sending %s with flags "%s" | nc %s %s' % (self.whoami, path, ' '.join(flags), host, port) return RunAsyncLoopLog.run(command) def rsync_path(self, path): """ start rsync session for given path and destination, returns true if successful """ if not path: self.log.raiseException('Empty path given!') elif not isinstance(path, basestring): self.log.raiseException('Invalid path: %s !' % path) else: code, output = self.attempt_run(path) if output: self.output_queue.put(output) return (code == 0) def get_a_dest(self, attempts): """ Try to get a destination x times """ attempt = 1 while (attempt <= attempts): dest = self.try_a_dest(self.TIME_OUT) # Keeps it if not consuming if dest: # We locked a rsync daemon self.log.debug('Got destination %s' % dest) return dest attempt += 1 time.sleep(self.WAITTIME) self.log.warning('Still no destination after %s tries' % attempts) return None def dest_is_sane(self, dest): """ Lock destination state, fetch it and disable if paused """ return self.dest_state(dest, self.STATUS) def handle_dest_state(self, dest, destpath, current_state): """ Disable destination by consuming it from queue when paused and return false If destination is active, return true """ if current_state == self.STATE_PAUSED: self.set_znode(destpath, self.STATE_DISABLED) self.dest_queue.consume() self.log.debug('Destination %s was disabled and removed from queue' % dest) return False elif current_state == self.STATE_ACTIVE: return True else: self.log.error('Destination %s is in an unknown state %s' % (dest, current_state)) return False def try_a_dest(self, timeout): """ Try to get a destination. check if destination is still running, otherwise remove """ if len(self.dest_queue) == 0: self.log.debug('Destinations not yet available') dest = self.dest_queue.get(timeout) if dest: port, whoami = tuple(dest.split(':', 1)) if not self.member_of_party(whoami, 'allsd'): self.log.debug('destination is not found in party') self.dest_queue.consume() return None elif not self.dest_is_sane(whoami): self.log.debug('recieved destination was paused') return None else: portmap = '%s/portmap/%s' % (self.session, whoami) lport, _ = self.get_znode(portmap) if port != lport: self.log.error('destination port not valid') # Should not happen self.dest_queue.consume() return None return dest def rsync(self, timeout=None): """ Get a destination, a path and call a new rsync iteration """ if self.verifypath: if not self.basepath_ok(): self.log.warning('Basepath not available, waiting') time.sleep(self.CHECK_WAIT) return None path = self.path_queue.get(timeout) if path: if self.rsync_path(path): self.path_queue.consume()
def __init__(self, hosts, session=None, name=None, default_acl=None, auth_data=None, rsyncpath=None, rsyncdepth=-1, rsubpaths=None, netcat=False, dryrun=False, delete=False, checksum=False, hardlinks=False, inplace=False, verbose=False, dropcache=False, timeout=None, excludere=None, excl_usr=None, verifypath=True, done_file=None, arbitopts=None): kwargs = { 'hosts': hosts, 'session': session, 'name': name, 'default_acl': default_acl, 'auth_data': auth_data, 'rsyncpath': rsyncpath, 'verifypath': verifypath, 'netcat': netcat, 'dropcache': dropcache, } super(RsyncSource, self).__init__(**kwargs) self.lockpath = self.znode_path(self.session + '/lock') self.lock = None self.path_queue = LockingQueue( self, self.znode_path(self.session + '/pathQueue')) self.completed_queue = LockingQueue( self, self.znode_path(self.session + '/completedQueue')) self.failed_queue = LockingQueue( self, self.znode_path(self.session + '/failedQueue')) self.output_queue = LockingQueue( self, self.znode_path(self.session + '/outputQueue')) self.stats_path = '%s/stats' % self.session self.init_stats() if rsyncdepth < 0: self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth) else: self.rsyncdepth = rsyncdepth self.rsync_arbitopts = arbitopts self.rsync_checksum = checksum self.rsync_delete = delete self.rsync_dry = dryrun self.rsync_hardlinks = hardlinks self.rsync_inplace = inplace self.rsync_timeout = timeout self.rsync_verbose = verbose self.done_file = done_file self.excludere = excludere self.excl_usr = excl_usr self.rsubpaths = rsubpaths
class RsyncSource(RsyncController): """ Class for controlling rsync with Zookeeper. Builds a tree of paths to devide, and effectively rsyncs the subpaths. Stops when ready """ BASE_PARTIES = RsyncController.BASE_PARTIES + ['sources'] NC_RANGE = 15 SLEEPTIME = 1 # For netcat stub TIME_OUT = 5 # waiting for destination WAITTIME = 5 # check interval of closure of other clients CHECK_WAIT = 20 # wait for path to be available RSYNC_STATS = [ 'Number_of_files', 'Number_of_regular_files_transferred', 'Total_file_size', 'Total_transferred_file_size', 'Literal_data', 'Matched_data', 'File_list_size', 'Total_bytes_sent', 'Total_bytes_received' ] def __init__(self, hosts, session=None, name=None, default_acl=None, auth_data=None, rsyncpath=None, rsyncdepth=-1, rsubpaths=None, netcat=False, dryrun=False, delete=False, checksum=False, hardlinks=False, inplace=False, verbose=False, dropcache=False, timeout=None, excludere=None, excl_usr=None, verifypath=True, done_file=None, arbitopts=None): kwargs = { 'hosts': hosts, 'session': session, 'name': name, 'default_acl': default_acl, 'auth_data': auth_data, 'rsyncpath': rsyncpath, 'verifypath': verifypath, 'netcat': netcat, 'dropcache': dropcache, } super(RsyncSource, self).__init__(**kwargs) self.lockpath = self.znode_path(self.session + '/lock') self.lock = None self.path_queue = LockingQueue( self, self.znode_path(self.session + '/pathQueue')) self.completed_queue = LockingQueue( self, self.znode_path(self.session + '/completedQueue')) self.failed_queue = LockingQueue( self, self.znode_path(self.session + '/failedQueue')) self.output_queue = LockingQueue( self, self.znode_path(self.session + '/outputQueue')) self.stats_path = '%s/stats' % self.session self.init_stats() if rsyncdepth < 0: self.log.raiseException('Invalid rsync depth: %i' % rsyncdepth) else: self.rsyncdepth = rsyncdepth self.rsync_arbitopts = arbitopts self.rsync_checksum = checksum self.rsync_delete = delete self.rsync_dry = dryrun self.rsync_hardlinks = hardlinks self.rsync_inplace = inplace self.rsync_timeout = timeout self.rsync_verbose = verbose self.done_file = done_file self.excludere = excludere self.excl_usr = excl_usr self.rsubpaths = rsubpaths def init_stats(self): self.ensure_path(self.znode_path(self.stats_path)) self.counters = {} for stat in self.RSYNC_STATS: self.counters[stat] = Counter( self, self.znode_path('%s/%s' % (self.stats_path, stat))) def output_stats(self): self.stats = dict([(k, v.value) for k, v in self.counters.items()]) jstring = json.dumps(self.stats) self.log.info('progress stats: %s' % jstring) return jstring def get_sources(self): """ Get all zookeeper clients in this session registered as clients """ hosts = [] for host in self.parties['sources']: hosts.append(host) return hosts def acq_lock(self): """ Try to acquire lock. Returns true if lock is acquired """ self.lock = self.Lock(self.lockpath, "") return self.lock.acquire(False) def release_lock(self): """ Release the acquired lock """ return self.lock.release() def start_ready_rwatch(self): """ Start a watch other clients can register to, but release lock and exit on error """ watch = self.start_ready_watch() if not watch: if len(self.get_all_hosts()) == 1: # Fix previous unclean shutdown self.cleanup() self.release_lock() self.exit() return False else: return watch def build_pathqueue(self): """ Build a queue of paths that needs to be rsynced """ self.log.info('removing old queue and building new queue') if self.exists(self.path_queue.path): self.delete(self.path_queue.path, recursive=True) if self.netcat: paths = [str(i) for i in range(self.NC_RANGE)] time.sleep(self.SLEEPTIME) else: tuplpaths = get_pathlist( self.rsyncpath, self.rsyncdepth, exclude_re=self.excludere, # By default don't exclude user files exclude_usr=self.excl_usr, rsubpaths=self.rsubpaths) paths = encode_paths(tuplpaths) self.paths_total = len(paths) for path in paths: self.path_queue.put(path.encode( )) # Put_all can issue a zookeeper connection error with big lists self.log.info('pathqueue building finished') return self.paths_total def isempty_pathqueue(self): """ Returns true if all paths in pathqueue are done """ return len(self.path_queue) == 0 def output_progress(self, todo): self.log.info('Progress: %s of %s paths remaining, %s failed' % (todo, self.paths_total, len(self.failed_queue))) self.output_stats() def output_clients(self, total, sources): dests = total - sources sources = sources - 1 self.log.info( 'Connected source (slave) clients: %s, connected destination clients: %s', sources, dests) def wait_and_keep_progress(self): todo_paths = self.paths_total total_clients = len(self.get_all_hosts()) total_sources = len(self.get_sources()) while not self.isempty_pathqueue(): todo_new = self.len_paths() if todo_paths != todo_new: # Output progress state todo_paths = todo_new self.output_progress(todo_paths) tot_clients_new = len(self.get_all_hosts()) src_clients_new = len(self.get_sources()) if total_clients != tot_clients_new or total_sources != src_clients_new: total_clients = tot_clients_new total_sources = src_clients_new self.output_clients(total_clients, total_sources) time.sleep(self.WAITTIME) def len_paths(self): """ Returns how many elements still in pathQueue """ return len(self.path_queue) def shutdown_all(self): """ Send end signal and release lock Make sure other clients are disconnected, clean up afterwards.""" self.stop_ready_watch() self.log.debug('watch set to stop') while len(self.get_all_hosts()) > 1: # self.log.debug("clients still connected: %s" % self.get_all_hosts()) time.sleep(self.WAITTIME) self.cleanup() def get_state(self): """Get the state of a running session""" remain = self.len_paths() if remain > 0: code = 0 self.log.info('Remaining: %s, Failed: %s' % (remain, len(self.failed_queue))) else: code = ZKRS_NO_SUCH_SESSION_EXIT_CODE self.log.info('No active session') return code def write_donefile(self, values): """ Write a cachefile with some stats about the run when done """ cache_file = FileCache(self.done_file) cache_file.update('stats', values, 0) cache_file.close() def cleanup(self): """ Remove all session nodes in zookeeper after first logging the output queues """ values = { 'unfinished': len(self.path_queue), 'failed': len(self.failed_queue), 'completed': len(self.completed_queue) } while (len(self.path_queue) > 0): self.log.warning('Unfinished Path %s' % self.path_queue.get().decode()) self.path_queue.consume() self.delete(self.dest_queue.path, recursive=True) self.delete(self.path_queue.path, recursive=True) self.output_stats() self.delete(self.znode_path(self.stats_path), recursive=True) while (len(self.failed_queue) > 0): self.log.error('Failed Path %s' % self.failed_queue.get().decode()) self.failed_queue.consume() while (len(self.completed_queue) > 0): self.log.info('Completed Path %s' % self.completed_queue.get().decode()) self.completed_queue.consume() self.log.info('Output:') while (len(self.output_queue) > 0): self.log.info(self.output_queue.get().decode()) self.output_queue.consume() self.delete(self.completed_queue.path, recursive=True) self.delete(self.failed_queue.path, recursive=True) self.delete(self.output_queue.path, recursive=True) self.remove_ready_watch() self.release_lock() self.log.info('Cleanup done: Lock, Queues and watch removed') if self.done_file: self.write_donefile(values) def generate_file(self, path): """ Writes the relative path used for the rsync of this path, for use by --files-from. """ if not path.startswith(self.rsyncpath): self.log.raiseException( 'Invalid path! %s is not a subpath of %s!' % (path, self.rsyncpath)) return None else: subpath = path[len(self.rsyncpath):] subpath = subpath.strip(os.path.sep) fd, name = tempfile.mkstemp(dir=self.RSDIR, text=True) wfile = os.fdopen(fd, "w") wfile.write('%s/' % subpath) return name def attempt_run(self, path, attempts=3): """ Try to run a command x times, on failure add to failed queue """ attempt = 1 while (attempt <= attempts): dest = self.get_a_dest(attempts) # Keeps it if not consuming if not dest or not self.basepath_ok(): self.path_queue.put(path.encode(), priority=50) # Keep path in queue self.path_queue.consume() # But stop locking it time.sleep(self.TIME_OUT) # Wait before new attempt return 1, None port, host, _ = tuple(dest.split(':', 2)) if self.netcat: code, output = self.run_netcat(path, host, port) else: code, output = self.run_rsync(path, host, port) if code == 0: self.completed_queue.put(path.encode()) return code, output attempt += 1 time.sleep(self.WAITTIME) # Wait before new attempt self.log.error('There were issues with path %s!' % path) self.failed_queue.put(path.encode()) return 0, output # otherwise client get stuck def parse_output(self, output): """ Parse the rsync output stats, and when verbose, print the files marked for transmission """ if self.rsync_verbose: outp = output.split("%s%s" % (os.linesep, os.linesep)) self.log.info('Verbose file list output is: %s' % outp[0]) del outp[0] output = os.linesep.join(outp) lines = output.splitlines() for line in lines: keyval = line.split(':') if len(keyval) < 2 or keyval[1] == ' ': self.log.debug('output line not parsed: %s' % line) continue key = re.sub(' ', '_', keyval[0]) val = keyval[1].split()[0] val = re.sub(',', '', val) if key not in self.RSYNC_STATS: self.log.debug('output metric not recognised: %s' % key) continue self.counters[key] += int(val) def get_flags(self, files, recursive): """ Make an array of flags to be used """ # Start rsync recursive or non recursive; archive mode (a) is equivalent to -rlptgoD (see man rsync) flags = [ '--stats', '--numeric-ids', '-lptgoD', '--files-from=%s' % files ] if recursive: flags.append('-r') if self.rsync_delete: flags.append('--delete') if self.rsync_checksum: flags.append('--checksum') if self.rsync_dropcache: flags.append('--drop-cache') if self.rsync_hardlinks: flags.append('--hard-links') if self.rsync_inplace: flags.append('--inplace') if self.rsync_timeout: flags.extend(['--timeout', str(self.rsync_timeout)]) if self.rsync_verbose: flags.append('--verbose') if self.rsync_dry: flags.append('-n') # This should always be processed last if self.rsync_arbitopts: arbopts = [] for arbopt in self.rsync_arbitopts: opt = "--%s" % re.sub(':', '=', arbopt, 1) arbopts.append(opt) self.log.warning('Adding unchecked flags %s' % ' '.join(arbopts)) flags.extend(arbopts) return flags def run_rsync(self, encpath, host, port): """ Runs the rsync command with or without recursion, delete or dry-run option. It uses the destination module linked with this session. """ path, recursive = decode_path(encpath) gfile = self.generate_file(path) flags = self.get_flags(gfile, recursive) self.log.info('%s is sending path %s to %s %s' % (self.whoami, path, host, port)) self.log.debug('Used flags: "%s"' % ' '.join(flags)) command = 'rsync %s %s/ rsync://%s:%s/%s' % ( ' '.join(flags), self.rsyncpath, host, port, self.module) code, output = RunAsyncLoopLog.run(command) os.remove(gfile) parsed = self.parse_output(output) return code, parsed def run_netcat(self, path, host, port): """ Test run with netcat """ time.sleep(self.SLEEPTIME) flags = self.get_flags('nofile', 0) command = 'echo %s is sending %s with flags "%s" | nc %s %s' % ( self.whoami, path, ' '.join(flags), host, port) return RunAsyncLoopLog.run(command) def rsync_path(self, path): """ start rsync session for given path and destination, returns true if successful """ if not path: self.log.raiseException('Empty path given!') return None elif not is_string(path): self.log.raiseException('Invalid path: %s !' % path) return None else: code, output = self.attempt_run(path) if output: self.output_queue.put(output.encode()) return (code == 0) def get_a_dest(self, attempts): """ Try to get a destination x times """ attempt = 1 while (attempt <= attempts): dest = self.try_a_dest(self.TIME_OUT) # Keeps it if not consuming if dest: # We locked a rsync daemon self.log.debug('Got destination %s' % dest) return dest attempt += 1 time.sleep(self.WAITTIME) self.log.warning('Still no destination after %s tries' % attempts) return None def dest_is_sane(self, dest): """ Lock destination state, fetch it and disable if paused """ return self.dest_state(dest, self.STATUS) def handle_dest_state(self, dest, destpath, current_state): """ Disable destination by consuming it from queue when paused and return false If destination is active, return true """ if current_state == self.STATE_PAUSED: self.set_znode(destpath, self.STATE_DISABLED) self.dest_queue.consume() self.log.debug( 'Destination %s was disabled and removed from queue' % dest) return False elif current_state == self.STATE_ACTIVE: return True else: self.log.error('Destination %s is in an unknown state %s' % (dest, current_state)) return False def try_a_dest(self, timeout): """ Try to get a destination. check if destination is still running, otherwise remove """ if len(self.dest_queue) == 0: self.log.debug('Destinations not yet available') dest = self.dest_queue.get(timeout).decode() if dest: port, whoami = tuple(dest.split(':', 1)) if not self.member_of_party(whoami, 'allsd'): self.log.debug('destination is not found in party') self.dest_queue.consume() return None elif not self.dest_is_sane(whoami): self.log.debug('recieved destination was paused') return None else: portmap = '%s/portmap/%s' % (self.session, whoami) lport, _ = self.get_znode(portmap) if port != lport: self.log.error( 'destination port not valid') # Should not happen self.dest_queue.consume() return None return dest def rsync(self, timeout=None): """ Get a destination, a path and call a new rsync iteration """ if self.verifypath: if not self.basepath_ok(): self.log.warning('Basepath not available, waiting') time.sleep(self.CHECK_WAIT) return None path = self.path_queue.get(timeout).decode() if path: if self.rsync_path(path): self.path_queue.consume() return None
def feeds(): try: conn = psycopg2.connect(cfg.pgConn) cur = conn.cursor() #zkHost = "ZKHOST1234" zk = KazooClient(hosts=cfg.zkCluster) zk.start() # 4 queues aliasq = LockingQueue(zk, cfg.alias) symbolq = LockingQueue(zk, cfg.symbol) issueq = LockingQueue(zk, cfg.issue) transferq = LockingQueue(zk, cfg.transfer) # 2 more queues propIssueq = LockingQueue(zk, cfg.proposeIssue) propTranq = LockingQueue(zk, cfg.proposeTran) cur.execute("""select min(id) from alias_redo where progress = 1""") res = cur.fetchone() conn.commit() if res[0]: minAlias = res[0] - 1 else: minAlias = -1 cur.execute("""select min(id) from symbol_redo where progress = 1 """) res = cur.fetchone() conn.commit() if res[0]: minSymbol = res[0] - 1 else: minSymbol = -1 cur.execute("""select min(id) from issue_redo where progress = 1 """) res = cur.fetchone() conn.commit() if res[0]: minIssue = res[0] - 1 else: minIssue = -1 cur.execute("""select min(id) from transfer_redo where progress = 1 """) res = cur.fetchone() conn.commit() if res[0]: minTransfer = res[0] - 1 else: minTransfer = -1 cur.execute("""select min(id) from propose_issue where progress = 1""") res = cur.fetchone() conn.commit() if res[0]: minpropIssue = res[0] - 1 else: minpropIssue = -1 cur.execute("""select min(id) from propose_transfer where progress = 1""") res = cur.fetchone() conn.commit() if res[0]: minpropTran = res[0] - 1 else: minpropTran = -1 while True: ints = datetime.now() inload = os.getloadavg()[0] # process alias cur.execute("""select alias, "globalId" , id from alias_redo where progress = 1 and id > %s""", [minAlias]) res = cur.fetchall() conn.commit() if res: for en in res: aliasq.put("{0}||{1}".format(en[0].strip(), en[1].strip()).encode('utf-8')) minAlias = en[2] # process symbol cur.execute("""select symbol, "globalId" , id from symbol_redo where progress = 1 and id > %s""", [minSymbol]) res = cur.fetchall() conn.commit() if res: for en in res: symbolq.put("{0}||{1}".format(en[0].strip(), en[1].strip()).encode('utf-8')) minSymbol = en[2] # process issue cur.execute("""select proposal ,id from issue_redo where progress = 1 and id > %s""", [minIssue]) res = cur.fetchall() conn.commit() if res: for en in res: issueq.put("{0}".format(en[0].strip()).encode('utf-8')) minIssue = en[1] # process transfer cur.execute("""select proposal,id from transfer_redo where progress = 1 and id > %s""", [minTransfer]) res = cur.fetchall() conn.commit() if res: for en in res: transferq.put("{0}".format(en[0].strip()).encode('utf-8')) minTransfer = en[1] # process proposeIssue cur.execute("""select symbol, quantity, "globalId",id from propose_issue where progress = 1 and id > %s """, [minpropIssue]) res = cur.fetchall() conn.commit() if res: for en in res: if not en[1] in [1, 2, 8]: continue propIssueq.put("{0}||{1}||{2}||{3}".format(en[0].strip(), en[1], en[2].strip(), en[3]).encode('utf-8')) minpropIssue = en[3] # process proposeTransfer cur.execute("""select alias, "rawCode", lastsig , id, "globalId" from propose_transfer where progress = 1 and id > %s""", [minpropTran]) res = cur.fetchall() conn.commit() if res: for en in res: propTranq.put("{0}&&{1}&&{2}&&{3}&&{4}".format(en[0].strip(), en[1].strip(), en[2].strip(), en[3], en[4].strip()).encode('utf-8')) minpropTran = en[3] outts = datetime.now() outload = os.getloadavg()[0] # stmt = SimpleStatement("""insert into runstat(id,executable,ints,inload,outts,outload) values (%s, %s, %s, %s, %s, %s)""", consistency_level=ConsistencyLevel.ANY) # cluster = Cluster(cfg.cassCluster) # session = cluster.connect(cfg.cassKeyspace) # session.execute(stmt, (uuid.uuid4(), executable, ints, inload, outts, outload)) time.sleep(1) except psycopg2.Error as err: print("SQLError {0}".format(err)) finally: zk.stop() zk.close() cur.close() conn.close()