def transfer_list_v1(node, wait_all_finish=False, attempts=90, delay=2, verbose=False): if verbose: dbg('transfer/list/v1 [%s]\n' % node) for _ in range(attempts): response = request_get(node, 'transfer/list/v1', timeout=20, verbose=verbose) assert response.status_code == 200 if verbose: dbg('transfer/list/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() if not wait_all_finish: break some_incoming = False some_outgoing = False for r in response.json()['result']: if r.get('incoming', []): some_incoming = True break if r.get('outgoing', []): some_outgoing = True break if not some_incoming and not some_outgoing: break time.sleep(delay) else: assert False, 'some transfers are still running on [%s]' % node return response.json()
def supplier_switch_v1(customer: str, supplier_idurl: str, position: int, validate_retries=30, delay=3): response = request_put(customer, 'supplier/switch/v1', json={ 'index': position, 'new_idurl': supplier_idurl, }, timeout=20) assert response.status_code == 200 dbg('supplier/switch/v1 [%s] with new supplier %s at position %r : %s\n' % ( customer, supplier_idurl, position, pprint.pformat(response.json()))) assert response.json()['status'] == 'OK', response.json() if not validate_retries: return response.json() count = 0 while True: if count >= validate_retries: break current_suppliers_idurls = supplier_list_v1(customer, expected_min_suppliers=None, expected_max_suppliers=None, attempts=1) if supplier_idurl in current_suppliers_idurls: _pos = current_suppliers_idurls.index(supplier_idurl) assert position == _pos dbg('found supplier %r at position %d for customer %r' % (supplier_idurl, position, customer)) return current_suppliers_idurls count += 1 time.sleep(delay) assert False, 'failed to switch supplier at position %r to %r after %d retries' % ( position, supplier_idurl, count, ) return None
def group_info_v1(node: str, group_key_id, wait_state=None, validate_retries=90, delay=2, stop_state=None): response = request_get(node, 'group/info/v1?group_key_id=%s' % group_key_id, timeout=20) assert response.status_code == 200 dbg('group/info/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()))) assert response.json()['status'] == 'OK', response.json() if wait_state is None: return response.json() count = 0 while True: if count >= validate_retries: dbg('group/info/v1 [%s] attempt %d : %s\n' % (node, count, pprint.pformat(response.json()))) break response = request_get(node, 'group/info/v1?group_key_id=%s' % group_key_id, timeout=20) assert response.status_code == 200 assert response.json()['status'] == 'OK', response.json() # print('group/info/v1 [%s] attempt %d : %s\n' % (node, count, pprint.pformat(response.json()))) dbg(' group/info/v1 [%s] attempt %d : state=%s' % (node, count, response.json()['result']['state'], )) if response.json()['result']['state'] == wait_state: dbg('group/info/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()))) return response.json() if stop_state and response.json()['result']['state'] == stop_state: dbg('group/info/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()))) return response.json() count += 1 time.sleep(delay) assert False, 'state %r was not detected for %r after %d retries' % (wait_state, group_key_id, count, )
def dht_db_dump_v1(node): try: response = request_get(node, 'dht/db/dump/v1', timeout=20) except: return None dbg('dht/db/dump/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()), )) return response.json()
def dht_value_get_v1(node, key, expected_data, record_type='skip_validation', retries=2, fallback_observer=None): response = None for i in range(retries + 1): if i == retries - 1 and fallback_observer: node = fallback_observer response = request_get(node, 'dht/value/get/v1?record_type=%s&key=%s' % (record_type, key, ), timeout=20) try: assert response.status_code == 200 dbg('dht/value/get/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() assert len(response.json()['result']) > 0, response.json() assert response.json()['result']['key'] == key, response.json() if expected_data == 'not_exist': assert response.json()['result']['read'] == 'failed', response.json() assert 'value' not in response.json()['result'], response.json() assert len(response.json()['result']['closest_nodes']) > 0, response.json() else: if response.json()['result']['read'] == 'failed': warn('first request failed, retry one more time') response = request_get(node, 'dht/value/get/v1?record_type=%s&key=%s' % (record_type, key, ), timeout=20) assert response.status_code == 200 assert response.json()['status'] == 'OK', response.json() assert response.json()['result']['read'] == 'success', response.json() assert 'value' in response.json()['result'], response.json() assert response.json()['result']['value']['data'] in expected_data, response.json() assert response.json()['result']['value']['key'] == key, response.json() assert response.json()['result']['value']['type'] == record_type, response.json() except: time.sleep(2) if i == retries - 1: assert False, f'DHT value read validation failed: {node} {key} {expected_data} : {response.json()}' return response.json()
def packet_list_v1(node, wait_all_finish=False, attempts=90, delay=2, verbose=False): if verbose: dbg('packet/list/v1 [%s]\n' % node) response = None for _ in range(attempts): response = request_get(node, 'packet/list/v1', timeout=20, verbose=verbose) assert response.status_code == 200 if verbose: dbg('packet/list/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() found_packet = False for r in response.json()['result']: if r.get('packet_id', '').count('idle_ping:'): continue # if r.get('command') == 'CancelService' and r.get('direction') == 'outgoing': # continue if r.get('command') == 'Retrieve' and r.get('direction') == 'outgoing' and r.get('label', '').count('-rotated'): continue if r.get('command') == 'Data' and r.get('direction') == 'outgoing' and r.get('label', '').count('-rotated'): continue if r.get('command') == 'Identity' and r.get('direction') == 'outgoing' and r.get('label', '').count('-rotated'): continue found_packet = True if not found_packet or not wait_all_finish: break time.sleep(delay) else: warn('packet/list/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()), )) assert False, 'some packets are still have in/out progress on [%s]' % node return response.json()
def automat_info_v1(node, automat_index, expected_state=None): response = request_get(node, 'automat/info/v1?index=%d' % int(automat_index), timeout=20) assert response.status_code == 200 dbg('automat/info/v1?index=%d [%s] : %s\n' % (int(automat_index), node, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() if expected_state is not None: assert response.json()['result']['state'] == expected_state, response.json() return response.json()
def friend_list_v1(node, extract_idurls=False): response = request_get(node, 'friend/list/v1', timeout=20) assert response.status_code == 200 dbg('friend/list/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() if not extract_idurls: return response.json() return [f['idurl'] for f in response.json()['result']]
def queue_peddler_list_v1(node, extract_ids=False): response = request_get(node, 'queue/peddler/list/v1', timeout=20) assert response.status_code == 200 dbg('queue/peddler/list/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() if not extract_ids: return response.json() return [f['queue_id'] for f in response.json()['result']]
def group_share_v1(customer: str, group_key_id, trusted_id, attempts=1, timeout=60): response = request_put(customer, 'group/share/v1', json={ 'group_key_id': group_key_id, 'trusted_id': trusted_id, 'timeout': timeout, }, timeout=timeout+1, attempts=attempts) assert response.status_code == 200 dbg('group/share/v1 [%s] group_key_id=%r trusted_id=%r : %s\n' % (customer, group_key_id, trusted_id, pprint.pformat(response.json()))) assert response.json()['status'] == 'OK', response.json() return response.json()
def _validate(obs): response = None num_suppliers = 0 count = 0 while True: mistakes = 0 if count >= retries: dbg('DHT info still wrong after %d retries, currently see %d suppliers, but expected %d' % ( count, num_suppliers, expected_suppliers_number)) return False try: response = request_get(obs, 'supplier/list/dht/v1?id=%s' % customer_id, timeout=20) except requests.exceptions.ConnectionError as exc: warn('connection error: %r' % exc) return False if response.status_code != 200: count += 1 time.sleep(delay) continue dbg('supplier/list/dht/v1?id=%s from %s\n%s\n' % (customer_id, obs, pprint.pformat(response.json()))) if not response.json()['status'] == 'OK': count += 1 time.sleep(delay) continue if not response.json()['result']: count += 1 time.sleep(delay) continue if not response.json()['result']['customer_idurl'].count('%s.xml' % customer_node): warn('currently see customer_idurl=%r, but expect family owner to be %r\n' % ( response.json()['result']['customer_idurl'], customer_node)) count += 1 time.sleep(delay) continue ss = response.json()['result']['suppliers'] num_suppliers = len(ss) if num_suppliers != expected_suppliers_number: warn('currently see %d suppliers but expected number is %d\n' % (num_suppliers, expected_suppliers_number)) count += 1 time.sleep(delay) continue if len(list(filter(None, ss))) != expected_suppliers_number: mistakes += abs(expected_suppliers_number - len(list(filter(None, ss)))) warn('found missing suppliers\n') if not response.json()['result']['ecc_map'] == expected_ecc_map: mistakes += 1 warn('currently see ecc_map=%r, but expect to be %r\n' % ( response.json()['result']['ecc_map'], expected_ecc_map)) if mistakes > accepted_mistakes: warn('currently see %d mistakes\n' % mistakes) count += 1 time.sleep(delay) continue break return True
def wait_event(nodes, event, expected_count=1, attempts=10, delay=5, verbose=False): dbg('wait event "%s" to occur %d times on %d nodes' % (event, expected_count, len(nodes), )) for node in nodes: for _ in range(attempts): event_log = run_ssh_command_and_wait(node, 'cat /root/.bitdust/logs/event.log', verbose=verbose)[0].strip() if event_log.count(event) == expected_count: break time.sleep(delay) else: assert False, f'event "{event}" did not occurred {expected_count} times on [{node}] after many attempts' dbg('')
def config_set_v1(node, key, value): response = request_post(node, 'config/set/v1', json={ 'key': key, 'value': value, }, timeout=20, ) assert response.status_code == 200 dbg('config/set/v1 [%s] key=%r value=%r : %s\n' % ( node, key, value, pprint.pformat(response.json()))) assert response.json()['status'] == 'OK', response.json() return response.json()
def friend_add_v1(node, friend_idurl, friend_alias=''): response = request_post(node, 'friend/add/v1', json={ 'idurl': friend_idurl, 'alias': friend_alias, }, timeout=20, ) assert response.status_code == 200 dbg('friend/add/v1 [%s] idurl=%r alias=%r : %s\n' % ( node, friend_idurl, friend_alias, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() return response.json()
def message_send_group_v1(node, group_key_id, data, timeout=120): dbg('message/send/group/v1 [%s] group_key_id=%r data=%r' % (node, group_key_id, data, )) response = request_post(node, 'message/send/group/v1', json={ 'group_key_id': group_key_id, 'data': data, }, timeout=timeout, ) assert response.status_code == 200 dbg(f'message/send/group/v1 [%s] : %s\n' % ( node, pprint.pformat(response.json()))) assert response.json()['status'] == 'OK', response.json() return response.json()
def message_send_v1(node, recipient, data, timeout=30, expect_consumed=True): response = request_post(node, 'message/send/v1', json={ 'id': recipient, 'data': data, 'timeout': timeout, }, timeout=timeout+1, ) assert response.status_code == 200 dbg(f'message/send/v1 [%s] : %s\n' % ( node, pprint.pformat(response.json()))) assert response.json()['status'] == 'OK', response.json() if expect_consumed is not None: assert response.json()['result']['consumed'] is expect_consumed return response.json()
def message_receive_v1(node, expected_data=None, consumer='test_consumer', get_result=None, timeout=15, polling_timeout=10, attempts=1): response = request_get(node, f'message/receive/{consumer}/v1?polling_timeout=%d' % polling_timeout, timeout=timeout, attempts=attempts) assert response.status_code == 200 dbg(f'message/receive/{consumer}/v1 [%s] : %s\n' % ( node, pprint.pformat(response.json()))) if get_result is not None: if response.json()['status'] == 'OK': get_result[0] = response.json() return get_result assert response.json()['status'] == 'OK', response.json() if expected_data is not None: received_result = response.json()['result'] received_data = {} if not received_result else received_result[0]['data'] received_data.pop('action', None) received_data.pop('msg_type', None) assert received_data == expected_data, response.json() return response.json()
def wait_specific_event(nodes, event, match_regex, expected_count=1, attempts=20, delay=5, verbose=False): dbg('wait event "%s" with "%s" to occur %d times on %d nodes' % (event, match_regex, expected_count, len(nodes), )) for node in nodes: for _ in range(attempts): event_log = run_ssh_command_and_wait(node, 'cat /root/.bitdust/logs/event.log', verbose=verbose)[0].strip() captured = False for ln in event_log.split('\n'): if ln.count(event): if re.match(match_regex, ln): captured = True break if captured: break time.sleep(delay) else: assert False, f'event "{event}" did not occurred {expected_count} times on [{node}] after many attempts' dbg('')
def event_listen_v1(node, expected_event_id, consumer_id='regression_tests_wait_event', attempts=3, timeout=10,): found = None count = 0 while not found: response = request_get(node, f'event/listen/{consumer_id}/v1', timeout=timeout) assert response.status_code == 200 assert response.json()['status'] == 'OK', response.json() dbg(f'event/listen/{consumer_id}/v1 : %s\n' % pprint.pformat(response.json())) for e in response.json()['result']: if e['id'] == expected_event_id: found = e break if found: break count += 1 if count >= attempts: assert False, f'event "{expected_event_id}" was not raised on node [{node}]' return found
def file_download_start_v1(customer: str, remote_path: str, destination: str, open_share=True, wait_result=True, download_attempts=1, wait_finish_attempts=20, delay=5, wait_tasks_finish=True): for _ in range(download_attempts): response = request_post(customer, 'file/download/start/v1', json={ 'remote_path': remote_path, 'destination_folder': destination, 'wait_result': '1' if wait_result else '0', 'open_share': '1' if open_share else '0', }, timeout=30, ) assert response.status_code == 200 dbg('file/download/start/v1 [%s] remote_path=%s destination_folder=%s : %s\n' % ( customer, remote_path, destination, pprint.pformat(response.json()), )) if response.json()['status'] == 'OK': # print('file/download/start/v1 [%s] remote_path=%s destination_folder=%s : %s\n' % ( # customer, remote_path, destination, pprint.pformat(response.json()), )) break if response.json()['errors'][0].count('downloading') and response.json()['errors'][0].count('already scheduled'): # print('file/download/start/v1 [%s] remote_path=%s destination_folder=%s : %s\n' % ( # customer, remote_path, destination, 'ALREADY STARTED', )) break if response.json()['errors'][0].count('failed') and response.json()['errors'][0].count('downloading'): time.sleep(delay) else: assert False, response.json() else: assert False, 'failed to start downloading uploaded file on [%r]: %r' % (customer, response.json(), ) if wait_tasks_finish: for _ in range(wait_finish_attempts): response = request_get(customer, 'file/download/v1', timeout=20) assert response.status_code == 200 dbg('file/download/v1 [%s] : %s\n' % (customer, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() if len(response.json()['result']) == 0: break time.sleep(delay) else: assert False, 'some downloading tasks are still running on [%s]' % customer return response.json()
def file_list_all_v1(node, expected_reliable=100, reliable_shares=True, attempts=20, delay=5, verbose=False): if expected_reliable is None: response = request_get(node, 'file/list/all/v1', timeout=20) assert response.status_code == 200 if verbose: dbg('file/list/all/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() return response.json() response = None latest_reliable = None latest_reliable_fileinfo = None count = 0 while latest_reliable is None or latest_reliable < expected_reliable: response = request_get(node, 'file/list/all/v1', timeout=20) assert response.status_code == 200 if verbose: dbg('file/list/all/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() lowest = 100 lowest_file = None for fil in response.json()['result']: if fil['remote_path'].startswith('share_') and not reliable_shares: continue for ver in fil['versions']: reliable = int(ver['reliable'].replace('%', '')) if reliable < lowest: lowest = reliable lowest_file = fil latest_reliable = lowest latest_reliable_fileinfo = fil if latest_reliable >= expected_reliable: break count += 1 if count >= attempts: warn(f' latest reliable item info: {latest_reliable_fileinfo}') assert False, f"file {lowest_file} is not {expected_reliable} % reliable after {attempts} attempts" return time.sleep(delay) return response.json()
def file_upload_start_v1(customer: str, remote_path: str, local_path: str, open_share=True, wait_result=True, wait_finish_attempts=20, delay=5, wait_job_finish=True, wait_packets_finish=True, wait_transfers_finish=True, ): response = request_post(customer, 'file/upload/start/v1', json={ 'remote_path': remote_path, 'local_path': local_path, 'wait_result': '1' if wait_result else '0', 'open_share': '1' if open_share else '0', }, timeout=30, ) assert response.status_code == 200 dbg('file/upload/start/v1 [%r] remote_path=%s local_path=%s : %s\n' % ( customer, remote_path, local_path, pprint.pformat(response.json()),)) assert response.json()['status'] == 'OK', response.json() if wait_job_finish: for _ in range(wait_finish_attempts): response = request_get(customer, 'file/upload/v1', timeout=20) assert response.status_code == 200 dbg('file/upload/v1 [%s] : %s\n' % (customer, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() if len(response.json()['result']['pending']) == 0 and len(response.json()['result']['running']) == 0: break time.sleep(delay) else: assert False, 'some uploading tasks are still running on [%s]' % customer if wait_packets_finish: packet_list_v1(customer, wait_all_finish=True) if wait_transfers_finish: transfer_list_v1(customer, wait_all_finish=True) return response.json()
def dht_value_set_v1(node, key, new_data, record_type='skip_validation', ): response = request_post(node, 'dht/value/set/v1', json={ 'key': key, 'record_type': record_type, 'value': { 'data': new_data, 'type': record_type, 'key': key, }, }, timeout=20, ) assert response.status_code == 200 dbg('dht/value/set/v1 [%s] key=%s : %s\n' % (node, key, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() assert len(response.json()['result']) > 0, response.json() assert response.json()['result']['write'] == 'success', response.json() assert response.json()['result']['key'] == key, response.json() assert response.json()['result']['value']['data'] == new_data, response.json() assert response.json()['result']['value']['key'] == key, response.json() assert response.json()['result']['value']['type'] == record_type, response.json() assert len(response.json()['result']['closest_nodes']) > 0, response.json() return response.json()
def group_info_dht_v1(node: str, customer_id, timeout=20, wait_records=None, retries=10, delay=3): response = request_get(node, 'group/info/dht/v1?group_creator_id=%s' % customer_id, timeout=timeout) assert response.status_code == 200 dbg('group/info/dht/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()))) assert response.json()['status'] == 'OK', response.json() if wait_records is None: return response.json() count = 0 while True: if len(response.json()['result']) == wait_records: dbg('group/info/dht/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()))) return response.json() if count >= retries: dbg('group/info/dht/v1 [%s] attempt %d : %s\n' % (node, count, pprint.pformat(response.json()))) break count += 1 time.sleep(delay) dbg('retrying group/info/dht/v1?group_creator_id=%s' % customer_id) response = request_get(node, 'group/info/dht/v1?group_creator_id=%s' % customer_id, timeout=timeout) assert response.status_code == 200 assert response.json()['status'] == 'OK', response.json() assert False, 'exactly %d dht records not detected for %r after %d retries' % (wait_records, customer_id, count, )
def supplier_list_v1(customer: str, expected_min_suppliers=None, expected_max_suppliers=None, attempts=20, delay=5, extract_suppliers=True, verbose=True): count = 0 num_connected = 0 while True: if count > attempts: assert False, f'{customer} failed to hire correct number of suppliers after many attempts. currently %d, expected min %d and max %d' % ( num_connected, expected_min_suppliers, expected_max_suppliers, ) response = request_get(customer, 'supplier/list/v1', timeout=20, verbose=verbose) assert response.status_code == 200 if verbose: dbg('supplier/list/v1 : %s\n' % pprint.pformat(response.json())) assert response.json()['status'] == 'OK', response.json() if expected_min_suppliers is None and expected_max_suppliers is None: break num_connected = 0 num_total = len(response.json()['result']) for s in response.json()['result']: if s['supplier_state'] == 'CONNECTED' and s['contact_state'] == 'CONNECTED': num_connected += 1 if expected_min_suppliers is not None and (num_connected < expected_min_suppliers or num_total < expected_min_suppliers): if verbose: dbg('found %d connected suppliers at the moment, but expect at least %d\n' % (num_connected, expected_min_suppliers)) count += 1 time.sleep(delay) continue if expected_max_suppliers is not None and (num_connected > expected_max_suppliers or num_total > expected_max_suppliers): if verbose: dbg('found %d connected suppliers at the moment, but expect no more than %d\n' % (num_connected, expected_max_suppliers)) count += 1 time.sleep(delay) continue if verbose: dbg('found %d connected suppliers at the moment\n' % num_connected) break if not extract_suppliers: return response.json() return [s['idurl'] for s in response.json()['result']]
def service_info_v1(node, service_name, expected_state, attempts=90, delay=2, verbose=True): current_state = None count = 0 while current_state is None or current_state != expected_state: response = request_get(node, f'service/info/{service_name}/v1', timeout=20, verbose=verbose) assert response.status_code == 200 assert response.json()['status'] == 'OK', response.json() current_state = response.json()['result']['state'] if current_state == expected_state: if verbose: dbg(f'service/info/{service_name}/v1 [{node}] : %s' % pprint.pformat(response.json())) break if verbose: dbg(f' service/info/{service_name}/v1 [{node}] : %s' % current_state) count += 1 if count >= attempts: assert False, f"service {service_name} is not {expected_state} after {attempts} attempts" return time.sleep(delay) if verbose: dbg(f'service/info/{service_name}/v1 [{node}] : OK\n')
def wait_suppliers_connected(nodes, expected_min_suppliers=2, expected_max_suppliers=2, verbose=False): dbg('wait min %d and max %d suppliers to be connected on %d nodes' % (expected_min_suppliers, expected_max_suppliers, len(nodes), )) for node in nodes: supplier_list_v1(node, expected_min_suppliers=expected_min_suppliers, expected_max_suppliers=expected_max_suppliers, verbose=verbose) dbg('')
def wait_service_state(nodes, service_name, state, verbose=False): dbg('wait service %r state to become %r on %d nodes' % (service_name, state, len(nodes), )) for node in nodes: service_info_v1(node, service_name, state, verbose=verbose) dbg('')
def wait_packets_finished(nodes, verbose=False): dbg('wait %d nodes to stop sending packets' % len(nodes)) for node in nodes: packet_list_v1(node, wait_all_finish=True, verbose=verbose) dbg('')
def automat_list_v1(node): response = request_get(node, 'automat/list/v1', timeout=20) assert response.status_code == 200 dbg('automat/list/v1 [%s] : %s\n' % (node, pprint.pformat(response.json()), )) assert response.json()['status'] == 'OK', response.json() return response.json()