def clear_brokers(customer_id): service_dir = settings.ServiceDir('service_private_groups') brokers_dir = os.path.join(service_dir, 'brokers') customer_dir = os.path.join(brokers_dir, customer_id) known_brokers(customer_id, erase_brokers=True) if os.path.isdir(customer_dir): bpio.rmdir_recursive(customer_dir, ignore_errors=True)
def load_groups(): service_dir = settings.ServiceDir('service_private_groups') groups_dir = os.path.join(service_dir, 'groups') if not os.path.isdir(groups_dir): bpio._dirs_make(groups_dir) brokers_dir = os.path.join(service_dir, 'brokers') if not os.path.isdir(brokers_dir): bpio._dirs_make(brokers_dir) for group_key_id in os.listdir(groups_dir): if group_key_id not in known_groups(): known_groups()[group_key_id] = { 'last_sequence_id': -1, 'active': False, 'archive_folder_path': None, } group_path = os.path.join(groups_dir, group_key_id) group_info = jsn.loads_text(local_fs.ReadTextFile(group_path)) if group_info: known_groups()[group_key_id] = group_info for customer_id in os.listdir(brokers_dir): customer_path = os.path.join(brokers_dir, customer_id) for broker_id in os.listdir(customer_path): if customer_id not in known_brokers(): known_brokers()[customer_id] = [None, ] * REQUIRED_BROKERS_COUNT if broker_id in known_brokers(customer_id): lg.warn('broker %r already exist' % broker_id) continue broker_path = os.path.join(customer_path, broker_id) broker_info = jsn.loads_text(local_fs.ReadTextFile(broker_path)) known_brokers()[customer_id][int(broker_info['position'])] = broker_id
def clear_broker(customer_id, position): service_dir = settings.ServiceDir('service_private_groups') brokers_dir = os.path.join(service_dir, 'brokers') customer_dir = os.path.join(brokers_dir, customer_id) if not os.path.isdir(customer_dir): if _Debug: lg.args(_DebugLevel, customer_id=customer_id, position=position) return False to_be_erased = [] for broker_id in os.listdir(customer_dir): broker_path = os.path.join(customer_dir, broker_id) broker_info = jsn.loads_text(local_fs.ReadTextFile(broker_path)) if not broker_info: to_be_erased.append(broker_id) lg.warn('found empty broker info for customer %r : %r' % (customer_id, broker_id, )) continue if broker_info.get('position') != position: continue to_be_erased.append(broker_id) if not to_be_erased: if _Debug: lg.args(_DebugLevel, customer_id=customer_id, position=position, to_be_erased=to_be_erased) return False removed = [] for broker_id in to_be_erased: broker_path = os.path.join(customer_dir, broker_id) os.remove(broker_path) removed.append(broker_path) if _Debug: lg.args(_DebugLevel, customer_id=customer_id, position=position, removed=removed) return True
def start(self): from twisted.internet.defer import Deferred, succeed from logs import lg from dht import dht_records from dht import dht_service from dht import known_nodes from main import settings from main.config import conf conf().addConfigNotifier('services/entangled-dht/udp-port', self._on_udp_port_modified) known_seeds = known_nodes.nodes() dht_layers = list(dht_records.LAYERS_REGISTRY.keys()) dht_service.init( udp_port=settings.getDHTPort(), dht_dir_path=settings.ServiceDir('service_entangled_dht'), open_layers=dht_layers, ) lg.info('DHT known seed nodes are : %r DHT layers are : %r' % (known_seeds, dht_layers, )) self.starting_deferred = Deferred() d = dht_service.connect( seed_nodes=known_seeds, layer_id=0, attach=True, ) d.addCallback(self._on_connected) d.addErrback(self._on_connect_failed) return self.starting_deferred or succeed(True)
def write_state(customer_id, broker_id, json_value): service_dir = settings.ServiceDir('service_message_broker') keepers_dir = os.path.join(service_dir, 'keepers') broker_dir = os.path.join(keepers_dir, broker_id) keeper_state_file_path = os.path.join(broker_dir, customer_id) if json_value is None: if os.path.isfile(keeper_state_file_path): try: os.remove(keeper_state_file_path) except: lg.exc() if _Debug: lg.args(_DebugLevel, customer_id=customer_id, broker_id=broker_id) return None if not os.path.isdir(broker_dir): try: os.makedirs(broker_dir) except: lg.exc() return None if not local_fs.WriteTextFile(keeper_state_file_path, jsn.dumps(json_value)): lg.err('failed writing queue_keeper state for customer %r of broker %r to %r' % ( customer_id, broker_id, keeper_state_file_path, )) return None if _Debug: lg.args(_DebugLevel, customer_id=customer_id, broker_id=broker_id, json_value=json_value) return json_value
def read_state(customer_id, broker_id): service_dir = settings.ServiceDir('service_message_broker') keepers_dir = os.path.join(service_dir, 'keepers') broker_dir = os.path.join(keepers_dir, broker_id) keeper_state_file_path = os.path.join(broker_dir, customer_id) json_value = None if os.path.isfile(keeper_state_file_path): try: json_value = jsn.loads_text(local_fs.ReadTextFile(keeper_state_file_path)) except: lg.exc() return None if _Debug: lg.args(_DebugLevel, customer_id=customer_id, broker_id=broker_id, json_value=json_value) return json_value broker_idurl = global_id.glob2idurl(broker_id) if id_url.is_cached(broker_idurl): for one_broker_id in os.listdir(keepers_dir): one_broker_idurl = global_id.glob2idurl(one_broker_id) if id_url.is_cached(one_broker_idurl): if one_broker_idurl == broker_idurl: broker_dir = os.path.join(keepers_dir, one_broker_id) keeper_state_file_path = os.path.join(broker_dir, customer_id) json_value = None if os.path.isfile(keeper_state_file_path): try: json_value = jsn.loads_text(local_fs.ReadTextFile(keeper_state_file_path)) except: lg.exc() return None if _Debug: lg.args(_DebugLevel, customer_id=customer_id, broker_id=one_broker_id, json_value=json_value) return json_value return None
def _on_restore_done(self, result, backupID, outfd, tarfilename): try: os.close(outfd) except: lg.exc() if result == 'done': lg.info('archive %r restore success from %r' % ( backupID, tarfilename, )) else: lg.err('archive %r restore failed from %r with : %r' % ( backupID, tarfilename, result, )) if result == 'done': _, pathID, versionName = packetid.SplitBackupID(backupID) service_dir = settings.ServiceDir('service_private_groups') queues_dir = os.path.join(service_dir, 'queues') queue_dir = os.path.join(queues_dir, self.group_key_id) snapshot_dir = os.path.join(queue_dir, pathID, versionName) if not os.path.isdir(snapshot_dir): bpio._dirs_make(snapshot_dir) d = backup_tar.extracttar_thread(tarfilename, snapshot_dir) d.addCallback(self._on_extract_done, backupID, tarfilename, snapshot_dir) d.addErrback(self._on_extract_failed, backupID, tarfilename, snapshot_dir) return d tmpfile.throw_out(tarfilename, 'restore ' + result) return None
def read_group_info(group_key_id): service_dir = settings.ServiceDir('service_private_groups') groups_dir = os.path.join(service_dir, 'groups') group_info_path = os.path.join(groups_dir, group_key_id) if not os.path.isfile(group_info_path): return None group_info = jsn.loads_text(local_fs.ReadTextFile(group_info_path)) return group_info
def erase_group_info(group_key_id): if not is_group_exist(group_key_id): lg.warn('group %r is not known' % group_key_id) return False service_dir = settings.ServiceDir('service_private_groups') groups_dir = os.path.join(service_dir, 'groups') group_info_path = os.path.join(groups_dir, group_key_id) if not os.path.isfile(group_info_path): return False os.remove(group_info_path) if _Debug: lg.args(_DebugLevel, group_key_id=group_key_id, group_info_path=group_info_path) return True
def save_group_info(group_key_id): if not is_group_exist(group_key_id): lg.warn('group %r is not known' % group_key_id) return False group_info = known_groups()[group_key_id] service_dir = settings.ServiceDir('service_private_groups') groups_dir = os.path.join(service_dir, 'groups') group_info_path = os.path.join(groups_dir, group_key_id) if not os.path.isdir(groups_dir): bpio._dirs_make(groups_dir) ret = local_fs.WriteTextFile(group_info_path, jsn.dumps(group_info)) if _Debug: lg.args(_DebugLevel, group_key_id=group_key_id, group_info_path=group_info_path, ret=ret) return ret
def set_broker(customer_id, broker_id, position=0): service_dir = settings.ServiceDir('service_private_groups') brokers_dir = os.path.join(service_dir, 'brokers') customer_dir = os.path.join(brokers_dir, customer_id) broker_path = os.path.join(customer_dir, broker_id) if os.path.isfile(broker_path): lg.warn('broker %r already exist for customer %r, overwriting' % (broker_id, customer_id, )) if not os.path.isdir(customer_dir): bpio._dirs_make(customer_dir) broker_info = { 'position': position, } if not local_fs.WriteTextFile(broker_path, jsn.dumps(broker_info)): lg.err('failed to set broker %r at position %d for customer %r' % (broker_id, position, customer_id, )) return False known_brokers(customer_id)[position] = broker_id if _Debug: lg.args(_DebugLevel, customer_id=customer_id, broker_id=broker_id, broker_info=broker_info) return True
def set_broker(customer_id, broker_id, position=0): customer_id = global_id.latest_glob_id(customer_id) broker_id = global_id.latest_glob_id(broker_id) service_dir = settings.ServiceDir('service_private_groups') brokers_dir = os.path.join(service_dir, 'brokers') customer_dir = os.path.join(brokers_dir, customer_id) broker_path = os.path.join(customer_dir, broker_id) if not os.path.isdir(customer_dir): bpio._dirs_make(customer_dir) if os.path.isfile(broker_path): if _Debug: lg.dbg(_DebugLevel, 'broker %r already exist for customer %r, overwriting' % (broker_id, customer_id, )) broker_info = { 'position': position, } prev_borker_id = known_brokers(customer_id)[position] if prev_borker_id: if prev_borker_id == broker_id: if _Debug: lg.args(_DebugLevel, customer_id=customer_id, position=position, broker_id=broker_id, prev_borker_id=prev_borker_id) return True prev_broker_path = os.path.join(customer_dir, prev_borker_id) if os.path.isfile(prev_broker_path): lg.info('replacing existing broker for customer %r at position %d : %r -> %r' % ( customer_id, position, prev_borker_id, broker_id, )) try: os.remove(prev_broker_path) except: lg.exc() return False if not local_fs.WriteTextFile(broker_path, jsn.dumps(broker_info)): lg.err('failed to set broker %r at position %d for customer %r' % (broker_id, position, customer_id, )) return False known_brokers(customer_id)[position] = broker_id if _Debug: lg.args(_DebugLevel, customer_id=customer_id, broker_id=broker_id, broker_info=broker_info) return True
def on_identity_url_changed(evt): from access import group_member service_dir = settings.ServiceDir('service_private_groups') groups_dir = os.path.join(service_dir, 'groups') brokers_dir = os.path.join(service_dir, 'brokers') old_idurl = id_url.field(evt.data['old_idurl']) new_idurl = id_url.field(evt.data['new_idurl']) active_group_keys = list(active_groups()) to_be_reconnected = [] for group_key_id in active_group_keys: if not group_key_id: continue group_creator_idurl = global_id.glob2idurl(group_key_id) if id_url.is_the_same(group_creator_idurl, old_idurl): old_group_path = os.path.join(groups_dir, group_key_id) latest_group_key_id = my_keys.latest_key_id(group_key_id) latest_group_path = os.path.join(groups_dir, latest_group_key_id) lg.info('going to rename rotated group file: %r -> %r' % (old_group_path, latest_group_path, )) if os.path.isfile(old_group_path): try: os.rename(old_group_path, latest_group_path) except: lg.exc() continue else: lg.warn('key file %r was not found, key was not renamed' % old_group_path) active_groups()[latest_group_key_id] = active_groups().pop(group_key_id) group_member.rotate_active_group_memeber(group_key_id, latest_group_key_id) gm = group_member.get_active_group_member(group_key_id) if gm and gm.connected_brokers and id_url.is_in(old_idurl, gm.connected_brokers.values()): lg.info('connected broker %r IDURL is rotated, going to reconnect %r' % (old_idurl, gm, )) if group_key_id not in to_be_reconnected: to_be_reconnected.append(group_key_id) known_customers = list(known_brokers().keys()) for customer_id in known_customers: latest_customer_id = global_id.idurl2glob(new_idurl) customer_idurl = global_id.glob2idurl(customer_id) if id_url.is_the_same(customer_idurl, old_idurl): latest_customer_dir = os.path.join(brokers_dir, latest_customer_id) lg.info('going to rename rotated customer id: %r -> %r' % (customer_id, latest_customer_id, )) old_customer_dir = os.path.join(brokers_dir, customer_id) if os.path.isdir(old_customer_dir): try: bpio.move_dir_recursive(old_customer_dir, latest_customer_dir) bpio.rmdir_recursive(old_customer_dir) except: lg.exc() continue known_brokers()[latest_customer_id] = known_brokers().pop(customer_id) for broker_pos, broker_id in enumerate(known_brokers(latest_customer_id)): if not broker_id: continue broker_idurl = global_id.glob2idurl(broker_id) if broker_idurl == old_idurl: latest_broker_id = global_id.idurl2glob(new_idurl) latest_broker_path = os.path.join(latest_customer_dir, latest_broker_id) lg.info('going to rename rotated broker id: %r -> %r' % (broker_id, latest_broker_id, )) old_broker_path = os.path.join(latest_customer_dir, broker_id) if os.path.isfile(old_broker_path): try: os.rename(old_broker_path, latest_broker_path) except: lg.exc() continue if latest_broker_id in known_brokers(latest_customer_id): lg.warn('broker %r already exist' % latest_broker_id) continue known_brokers()[latest_customer_id][broker_pos] = latest_broker_id if _Debug: lg.args(_DebugLevel, to_be_reconnected=to_be_reconnected) for group_key_id in to_be_reconnected: gm = group_member.get_active_group_member(group_key_id) if gm: gm.automat('reconnect')
def load_groups(): loaded_brokers = 0 loaded_groups = 0 service_dir = settings.ServiceDir('service_private_groups') groups_dir = os.path.join(service_dir, 'groups') if not os.path.isdir(groups_dir): bpio._dirs_make(groups_dir) brokers_dir = os.path.join(service_dir, 'brokers') if not os.path.isdir(brokers_dir): bpio._dirs_make(brokers_dir) for group_key_id in os.listdir(groups_dir): latest_group_key_id = my_keys.latest_key_id(group_key_id) latest_group_path = os.path.join(groups_dir, latest_group_key_id) if latest_group_key_id != group_key_id: lg.info('going to rename rotated group key: %r -> %r' % (group_key_id, latest_group_key_id, )) old_group_path = os.path.join(groups_dir, group_key_id) try: os.rename(old_group_path, latest_group_path) except: lg.exc() continue latest_group_info = jsn.loads_text(local_fs.ReadTextFile(latest_group_path)) if not latest_group_info: lg.err('was not able to load group info from %r' % latest_group_path) continue active_groups()[latest_group_key_id] = latest_group_info loaded_groups += 1 for customer_id in os.listdir(brokers_dir): latest_customer_id = global_id.latest_glob_id(customer_id) latest_customer_dir = os.path.join(brokers_dir, latest_customer_id) if latest_customer_id != customer_id: lg.info('going to rename rotated customer id: %r -> %r' % (customer_id, latest_customer_id, )) old_customer_dir = os.path.join(brokers_dir, customer_id) try: bpio.move_dir_recursive(old_customer_dir, latest_customer_dir) bpio.rmdir_recursive(old_customer_dir) except: lg.exc() continue for broker_id in os.listdir(latest_customer_dir): if latest_customer_id not in known_brokers(): known_brokers(latest_customer_id) latest_broker_id = global_id.latest_glob_id(broker_id) latest_broker_path = os.path.join(latest_customer_dir, latest_broker_id) if latest_broker_id != broker_id: lg.info('going to rename rotated broker id: %r -> %r' % (broker_id, latest_broker_id, )) old_broker_path = os.path.join(latest_customer_dir, broker_id) try: os.rename(old_broker_path, latest_broker_path) except: lg.exc() continue latest_broker_info = jsn.loads_text(local_fs.ReadTextFile(latest_broker_path)) if not latest_broker_info: lg.err('was not able to load broker info from %r' % latest_broker_path) continue existing_broker_id = known_brokers(latest_customer_id)[int(latest_broker_info['position'])] if existing_broker_id: if os.path.isfile(latest_broker_path): lg.err('found duplicated broker for customer %r on position %d, erasing file %r' % ( latest_customer_id, int(latest_broker_info['position']), latest_broker_path, )) try: os.remove(latest_broker_path) except: lg.exc() continue known_brokers()[latest_customer_id][int(latest_broker_info['position'])] = latest_broker_id loaded_brokers += 1 if _Debug: lg.args(_DebugLevel, loaded_groups=loaded_groups, loaded_brokers=loaded_brokers)
def is_group_stored(group_key_id): service_dir = settings.ServiceDir('service_private_groups') groups_dir = os.path.join(service_dir, 'groups') group_info_path = os.path.join(groups_dir, group_key_id) return os.path.isfile(group_info_path)
def data_dir_path(self): from main import settings return settings.ServiceDir(self.service_name)