def get_zonefile(domain): resp = rest_to_api("/v1/names/{}/zonefile".format(domain)) if resp.status_code != 200: log.error("Error fetch zonefile for {} : {} {}".format( domain, resp.status_code, resp.text)) raise Exception("Failed to fetch zonefile") zf_raw = resp.json()["zonefile"] if zf_raw: return ysi_zones.parse_zone_file(str(zf_raw)) raise Exception("No zonefile returned")
def get_cached_zonefile( zonefile_hash, zonefile_dir=None ): """ Get a cached zonefile dict from local disk Return None if not found """ data = get_cached_zonefile_data( zonefile_hash, zonefile_dir=zonefile_dir ) if data is None: return None try: zonefile_dict = ysi_zones.parse_zone_file( data ) assert ysi_client.is_user_zonefile( zonefile_dict ), "Not a user zonefile: %s" % zonefile_hash return zonefile_dict except Exception, e: log.error("Failed to parse zonefile") return None
def lookup_index_manifest_url(blockchain_id, driver_name, index_stem, config_path): """ Given a blockchain ID, go and get the index manifest url. This is only applicable for certain drivers--i.e. the ones that need a name-to-URL index since the storage system generates URLs to data on-the-fly. This includes Dropbox, Google Drive, Onedrive, etc. The storage index URL will be located as an 'account', where * 'service' will be set to the driver name * 'identifier' will be set to 'storage' * 'contentUrl' will be set to the index url Return the index manifest URL on success. Return None if there is no URL Raise on error TODO: this method needs to be rewritten to use the token file format, and to use the proper public key to verify it. """ import ysi_client import ysi_client.proxy as proxy import ysi_client.user import ysi_client.storage import ysi_client.schemas if blockchain_id is None: # try getting it directly (we should have it) return index_settings_get_index_manifest_url(driver_name, config_path) name_record = proxy.get_name_blockchain_record(blockchain_id) if 'error' in name_record: raise Exception( "Failed to load name record for {}".format(blockchain_id)) zonefile_txt = get_zonefile_from_atlas(blockchain_id, config_path, name_record=name_record) zonefile_pubkey = None try: zonefile = ysi_zones.parse_zone_file(zonefile_txt) zonefile = dict(zonefile) zonefile_pubkey = ysi_client.user.user_zonefile_data_pubkey(zonefile) except: raise Exception("Non-standard zonefile for {}".format(blockchain_id)) # get the profile... # we're assuming here that some of the profile URLs are at least HTTP-accessible # (i.e. we can get them without having to go through the indexing system) # TODO: let drivers report their 'safety' profile_txt = None urls = ysi_client.user.user_zonefile_urls(zonefile) for url in urls: profile_txt = None try: profile_txt = get_chunk_via_http(url, blockchain_id=blockchain_id) except Exception as e: if DEBUG: log.exception(e) log.debug("Failed to load profile from {}".format(url)) continue if profile_txt is None: log.debug("Failed to load profile from {}".format(url)) continue profile = ysi_client.storage.parse_mutable_data( profile_txt, zonefile_pubkey, public_key_hash=name_record['address']) if not profile: log.debug("Failed to load profile from {}".format(url)) continue # TODO: load this from the tokens file # got profile! the storage information will be listed as an account, where the 'service' is the driver name and the 'identifier' is the manifest url if 'account' not in profile: log.error( "No 'account' key in profile for {}".format(blockchain_id)) return None accounts = profile['account'] if not isinstance(accounts, list): log.error("Invalid 'account' key in profile for {}".format( blockchain_id)) return None for account in accounts: try: jsonschema.validate(account, ysi_client.schemas.PROFILE_ACCOUNT_SCHEMA) except jsonschema.ValidationError: continue if account['service'] != driver_name: log.debug("Skipping account for '{}'".format( account['service'])) continue if account['identifier'] != 'storage': log.debug("Skipping non-storage account for '{}'".format( account['service'])) continue if not account.has_key('contentUrl'): continue url = account['contentUrl'] parsed_url = urlparse.urlparse(url) # must be valid http(s) URL, or a test:// URL if (not parsed_url.scheme or not parsed_url.netloc) and not url.startswith('test://'): log.warning("Skip invalid '{}' driver URL".format(driver_name)) continue log.debug("Index manifest URL for {} is {}".format( blockchain_id, url)) return url return None
def check(state_engine): global wallet_keys, datasets, zonefile_hash # not revealed, but ready ns = state_engine.get_namespace_reveal("test") if ns is not None: print "namespace not ready" return False ns = state_engine.get_namespace("test") if ns is None: print "no namespace" return False if ns['namespace_id'] != 'test': print "wrong namespace" return False # not preordered preorder = state_engine.get_name_preorder( "foo.test", virtualchain.make_payment_script(wallets[2].addr), wallets[3].addr) if preorder is not None: print "still have preorder" return False # registered name_rec = state_engine.get_name("foo.test") if name_rec is None: print "name does not exist" return False # owned if name_rec['address'] != wallets[3].addr or name_rec[ 'sender'] != virtualchain.make_payment_script(wallets[3].addr): print "name has wrong owner" return False srv = xmlrpclib.ServerProxy("http://localhost:%s" % ysi.RPC_SERVER_PORT) # zonefile and profile replicated to ysi server try: zonefile_by_hash_str = srv.get_zonefiles([name_rec['value_hash']]) zonefile_by_hash = json.loads(zonefile_by_hash_str) assert 'error' not in zonefile_by_hash, json.dumps(zonefile_by_hash, indent=4, sort_keys=True) zf1 = None try: zf1 = base64.b64decode( zonefile_by_hash['zonefiles'][name_rec['value_hash']]) except: print zonefile_by_hash raise zonefile = ysi_zones.parse_zone_file(zf1) user_pubkey = ysi_client.user.user_zonefile_data_pubkey(zonefile) assert user_pubkey is not None, "no zonefile public key" profile_resp_txt = srv.get_profile("foo.test") profile_resp = json.loads(profile_resp_txt) assert 'error' not in profile_resp, "error:\n%s" % json.dumps( profile_resp, indent=4, sort_keys=True) assert 'profile' in profile_resp, "missing profile:\n%s" % json.dumps( profile_resp, indent=4, sort_keys=True) # profile will be in 'raw' form raw_profile = profile_resp['profile'] profile = ysi_client.storage.parse_mutable_data( raw_profile, user_pubkey) except Exception, e: traceback.print_exc() print "Invalid profile" return False
def scenario( wallets, **kw ): global wallet_keys, wallet_keys_2, error, index_file_data, resource_data wallet_keys = testlib.ysi_client_initialize_wallet( "0123456789abcdef", wallets[5].privkey, wallets[3].privkey, wallets[3].privkey ) test_proxy = testlib.TestAPIProxy() ysi_client.set_default_proxy( test_proxy ) testlib.ysi_namespace_preorder( "test", wallets[1].addr, wallets[0].privkey ) testlib.next_block( **kw ) testlib.ysi_namespace_reveal( "test", wallets[1].addr, 52595, 250, 4, [6,5,4,3,2,1,0,0,0,0,0,0,0,0,0,0], 10, 10, wallets[0].privkey ) testlib.next_block( **kw ) testlib.ysi_namespace_ready( "test", wallets[1].privkey ) testlib.next_block( **kw ) testlib.ysi_name_preorder( "foo.test", wallets[2].privkey, wallets[3].addr ) testlib.next_block( **kw ) testlib.ysi_name_register( "foo.test", wallets[2].privkey, wallets[3].addr ) testlib.next_block( **kw ) # migrate profiles, but no data key in the zone file res = testlib.migrate_profile( "foo.test", zonefile_has_data_key=False, proxy=test_proxy, wallet_keys=wallet_keys ) if 'error' in res: res['test'] = 'Failed to initialize foo.test profile' print json.dumps(res, indent=4, sort_keys=True) error = True return # tell serialization-checker that value_hash can be ignored here print "BLOCKSTACK_SERIALIZATION_CHECK_IGNORE value_hash" sys.stdout.flush() testlib.next_block( **kw ) config_path = os.environ.get("BLOCKSTACK_CLIENT_CONFIG", None) # make a session datastore_pk = keylib.ECPrivateKey(wallets[-1].privkey).to_hex() res = testlib.ysi_cli_app_signin("foo.test", datastore_pk, 'register.app', ['names', 'register', 'prices', 'zonefiles', 'blockchain', 'node_read', 'user_read']) if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) error = True return ses = res['token'] # register the name bar.test. autogenerate the rest old_user_zonefile = ysi_client.zonefile.make_empty_zonefile('bar.test', None) old_user_zonefile_txt = ysi_zones.make_zone_file(old_user_zonefile) res = testlib.ysi_REST_call('POST', '/v1/names', ses, data={'name': 'bar.test', 'zonefile': old_user_zonefile_txt, 'make_profile': True} ) if 'error' in res: res['test'] = 'Failed to register user' print json.dumps(res) error = True return False print res tx_hash = res['response']['transaction_hash'] # wait for preorder to get confirmed... for i in xrange(0, 6): testlib.next_block( **kw ) res = testlib.verify_in_queue(ses, 'bar.test', 'preorder', tx_hash ) if not res: return False # wait for the preorder to get confirmed for i in xrange(0, 4): testlib.next_block( **kw ) # wait for register to go through print 'Wait for register to be submitted' time.sleep(10) # wait for the register/update to get confirmed for i in xrange(0, 6): testlib.next_block( **kw ) res = testlib.verify_in_queue(ses, 'bar.test', 'register', None ) if not res: return False for i in xrange(0, 4): testlib.next_block( **kw ) # wait for register to go through print 'Wait for zonefile to replicate' time.sleep(10) res = testlib.ysi_REST_call("GET", "/v1/names/bar.test", ses) if 'error' in res or res['http_status'] != 200: res['test'] = 'Failed to get name bar.test' print json.dumps(res) return False old_expire_block = res['response']['expire_block'] # get the zonefile res = testlib.ysi_REST_call("GET", "/v1/names/bar.test/zonefile", ses ) if 'error' in res or res['http_status'] != 200: res['test'] = 'Failed to get name zonefile' print json.dumps(res) return False # zonefile must not have a public key listed zonefile_txt = res['response']['zonefile'] print zonefile_txt parsed_zonefile = ysi_zones.parse_zone_file(zonefile_txt) if parsed_zonefile.has_key('txt'): print 'have txt records' print parsed_zonefile return False # renew it, but put the *current* owner key as the zonefile's *new* public key new_user_zonefile = ysi_client.zonefile.make_empty_zonefile('bar.test', wallets[3].pubkey_hex ) new_user_zonefile_txt = ysi_zones.make_zone_file(new_user_zonefile) res = testlib.ysi_REST_call("POST", "/v1/names", ses, data={'name': 'bar.test', 'zonefile': new_user_zonefile_txt} ) if 'error' in res or res['http_status'] != 202: res['test'] = 'Failed to renew name' print json.dumps(res) return False # verify in renew queue for i in xrange(0, 6): testlib.next_block( **kw ) res = testlib.verify_in_queue(ses, 'bar.test', 'renew', None ) if not res: return False for i in xrange(0, 4): testlib.next_block( **kw ) # new expire block res = testlib.ysi_REST_call("GET", "/v1/names/bar.test", ses) if 'error' in res or res['http_status'] != 200: res['test'] = 'Failed to get name bar.test' print json.dumps(res) return False new_expire_block = res['response']['expire_block'] # do we have the history for the name? res = testlib.ysi_REST_call("GET", "/v1/names/bar.test/history", ses ) if 'error' in res or res['http_status'] != 200: res['test'] = "Failed to get name history for bar.test" print json.dumps(res) return False # valid history? hist = res['response'] if len(hist.keys()) != 3: res['test'] = 'Failed to get update history' res['history'] = hist print json.dumps(res, indent=4, sort_keys=True) return False # get the zonefile res = testlib.ysi_REST_call("GET", "/v1/names/bar.test/zonefile", ses ) if 'error' in res or res['http_status'] != 200: res['test'] = 'Failed to get name zonefile' print json.dumps(res) return False # zonefile must have old owner key zonefile_txt = res['response']['zonefile'] parsed_zonefile = ysi_zones.parse_zone_file(zonefile_txt) if not parsed_zonefile.has_key('txt'): print 'missing txt' print parsed_zonefile return False found = False for txtrec in parsed_zonefile['txt']: if txtrec['name'] == 'pubkey' and txtrec['txt'] == 'pubkey:data:{}'.format(wallets[3].pubkey_hex): found = True if not found: print 'missing public key {}'.format(wallets[3].pubkey_hex) return False # profile lookup must work res = testlib.ysi_REST_call("GET", "/v1/users/bar.test", ses) if 'error' in res or res['http_status'] != 200: res['text'] = 'failed to get profile for bar.test' print json.dumps(res) return False print '' print json.dumps(res['response'], indent=4, sort_keys=True) print '' # verify pushed back if old_expire_block + 12 > new_expire_block: # didn't go through print >> sys.stderr, "Renewal didn't work: %s --> %s" % (old_expire_block, new_expire_block) return False
def decode_name_zonefile(name, zonefile_txt, allow_legacy=False): """ Decode a serialized zonefile into a JSON dict. If allow_legacy is True, then support legacy zone file formats (including Onename profiles) Otherwise, the data must actually be a Blockstack zone file. * If the zonefile does not have $ORIGIN, or if $ORIGIN does not match the name, then this fails. Return None on error """ user_zonefile = None try: # by default, it's a zonefile-formatted text file user_zonefile_defaultdict = ysi_zones.parse_zone_file(zonefile_txt) assert user_db.is_user_zonefile( user_zonefile_defaultdict), 'Not a user zonefile' # force dict user_zonefile = dict(user_zonefile_defaultdict) except (IndexError, ValueError, ysi_zones.InvalidLineException): if not allow_legacy: return {'error': 'Legacy zone file'} # might be legacy profile log.debug( 'WARN: failed to parse user zonefile; trying to import as legacy') try: user_zonefile = json.loads(zonefile_txt) if not isinstance(user_zonefile, dict): log.debug('Not a legacy user zonefile') return None except Exception as e: if BLOCKSTACK_DEBUG: log.exception(e) log.error('Failed to parse non-standard zonefile') return None except Exception as e: if BLOCKSTACK_DEBUG: log.exception(e) log.error('Failed to parse zonefile') return None if user_zonefile is None: return None if not allow_legacy: # additional checks if not user_zonefile.has_key('$origin'): log.debug("Zonefile has no $ORIGIN") return None if user_zonefile['$origin'] != name: log.debug("Name/zonefile mismatch: $ORIGIN = {}, name = {}".format( user_zonefile['$origin'], name)) return None return user_zonefile
def scenario( wallets, **kw ): global wallet_keys, wallet_keys_2, error, index_file_data, resource_data wallet_keys = testlib.ysi_client_initialize_wallet( "0123456789abcdef", wallets[5].privkey, wallets[3].privkey, wallets[3].privkey ) test_proxy = testlib.TestAPIProxy() ysi_client.set_default_proxy( test_proxy ) testlib.ysi_namespace_preorder( "test", wallets[1].addr, wallets[0].privkey ) testlib.next_block( **kw ) testlib.ysi_namespace_reveal( "test", wallets[1].addr, 52595, 250, 4, [6,5,4,3,2,1,0,0,0,0,0,0,0,0,0,0], 10, 10, wallets[0].privkey ) testlib.next_block( **kw ) testlib.ysi_namespace_ready( "test", wallets[1].privkey ) testlib.next_block( **kw ) testlib.ysi_name_preorder( "foo.test", wallets[2].privkey, wallets[3].addr ) testlib.next_block( **kw ) testlib.ysi_name_register( "foo.test", wallets[2].privkey, wallets[3].addr ) testlib.next_block( **kw ) # migrate profiles, but no data key in the zone file res = testlib.migrate_profile( "foo.test", zonefile_has_data_key=False, proxy=test_proxy, wallet_keys=wallet_keys ) if 'error' in res: res['test'] = 'Failed to initialize foo.test profile' print json.dumps(res, indent=4, sort_keys=True) error = True return # tell serialization-checker that value_hash can be ignored here print "BLOCKSTACK_SERIALIZATION_CHECK_IGNORE value_hash" sys.stdout.flush() testlib.next_block( **kw ) config_path = os.environ.get("BLOCKSTACK_CLIENT_CONFIG", None) # make a session datastore_pk = keylib.ECPrivateKey(wallets[-1].privkey).to_hex() res = testlib.ysi_cli_app_signin("foo.test", datastore_pk, 'register.app', ['names', 'register', 'prices', 'zonefiles', 'blockchain', 'node_read', 'user_read']) if 'error' in res: print json.dumps(res, indent=4, sort_keys=True) error = True return ses = res['token'] # register the name bar.test. autogenerate the rest old_user_zonefile = ysi_client.zonefile.make_empty_zonefile('bar.test', None) old_user_zonefile_txt = ysi_zones.make_zone_file(old_user_zonefile) res = testlib.ysi_REST_call('POST', '/v1/names', ses, data={'name': 'bar.test', 'zonefile': old_user_zonefile_txt, 'make_profile': True} ) if 'error' in res: res['test'] = 'Failed to register user' print json.dumps(res) error = True return False print res tx_hash = res['response']['transaction_hash'] # wait for preorder to get confirmed... for i in xrange(0, 6): testlib.next_block( **kw ) res = testlib.verify_in_queue(ses, 'bar.test', 'preorder', tx_hash ) if not res: return False # wait for the preorder to get confirmed for i in xrange(0, 4): testlib.next_block( **kw ) # wait for register to go through print 'Wait for register to be submitted' time.sleep(10) # wait for the register/update to get confirmed for i in xrange(0, 6): testlib.next_block( **kw ) res = testlib.verify_in_queue(ses, 'bar.test', 'register', None ) if not res: return False for i in xrange(0, 3): testlib.next_block( **kw ) # should have nine confirmations now res = testlib.get_queue(ses, 'register') if 'error' in res: print res return False if len(res) != 1: print res return False reg = res[0] confs = ysi_client.get_tx_confirmations(reg['tx_hash']) if confs != 9: print 'wrong number of confs for {} (expected 9): {}'.format(reg['tx_hash'], confs) return False # stop the API server testlib.stop_api() # advance blockchain testlib.next_block(**kw) testlib.next_block(**kw) confs = ysi_client.get_tx_confirmations(reg['tx_hash']) if confs != 11: print 'wrong number of confs for {} (expected 11): {}'.format(reg['tx_hash'], confs) return False # make sure the registrar does not process reg/up zonefile replication # (i.e. we want to make sure that the zonefile gets processed even if the blockchain goes too fast) os.environ['BLOCKSTACK_TEST_REGISTRAR_FAULT_INJECTION_SKIP_REGUP_REPLICATION'] = '1' testlib.start_api("0123456789abcdef") print 'Wait to verify that we do not remove the zone file just because the tx is confirmed' time.sleep(10) # verify that this is still in the queue res = testlib.get_queue(ses, 'register') if 'error' in res: print res return False if len(res) != 1: print res return False # clear the fault print 'Clearing regup replication fault' testlib.ysi_test_setenv("BLOCKSTACK_TEST_REGISTRAR_FAULT_INJECTION_SKIP_REGUP_REPLICATION", "0") # wait for register to go through print 'Wait for zonefile to replicate' time.sleep(10) res = testlib.ysi_REST_call("GET", "/v1/names/bar.test", ses) if 'error' in res or res['http_status'] != 200: res['test'] = 'Failed to get name bar.test' print json.dumps(res) return False old_expire_block = res['response']['expire_block'] # get the zonefile res = testlib.ysi_REST_call("GET", "/v1/names/bar.test/zonefile", ses ) if 'error' in res or res['http_status'] != 200: res['test'] = 'Failed to get name zonefile' print json.dumps(res) return False # zonefile must not have a public key listed zonefile_txt = res['response']['zonefile'] print zonefile_txt parsed_zonefile = ysi_zones.parse_zone_file(zonefile_txt) if parsed_zonefile.has_key('txt'): print 'have txt records' print parsed_zonefile return False # renew it, but put the *current* owner key as the zonefile's *new* public key new_user_zonefile = ysi_client.zonefile.make_empty_zonefile('bar.test', wallets[3].pubkey_hex ) new_user_zonefile_txt = ysi_zones.make_zone_file(new_user_zonefile) res = testlib.ysi_REST_call("POST", "/v1/names", ses, data={'name': 'bar.test', 'zonefile': new_user_zonefile_txt} ) if 'error' in res or res['http_status'] != 202: res['test'] = 'Failed to renew name' print json.dumps(res) return False # verify in renew queue for i in xrange(0, 6): testlib.next_block( **kw ) res = testlib.verify_in_queue(ses, 'bar.test', 'renew', None ) if not res: return False for i in xrange(0, 3): testlib.next_block( **kw ) # should have nine confirmations now res = testlib.get_queue(ses, 'renew') if 'error' in res: print res return False if len(res) != 1: print res return False reg = res[0] confs = ysi_client.get_tx_confirmations(reg['tx_hash']) if confs != 9: print 'wrong number of confs for {} (expected 9): {}'.format(reg['tx_hash'], confs) return False # stop the API server testlib.stop_api() # advance blockchain testlib.next_block(**kw) testlib.next_block(**kw) confs = ysi_client.get_tx_confirmations(reg['tx_hash']) if confs != 11: print 'wrong number of confs for {} (expected 11): {}'.format(reg['tx_hash'], confs) return False # make the registrar skip the first few steps, so the only thing it does is clear out confirmed updates # (i.e. we want to make sure that the renewal's zonefile gets processed even if the blockchain goes too fast) os.environ['BLOCKSTACK_TEST_REGISTRAR_FAULT_INJECTION_SKIP_RENEWAL_REPLICATION'] = '1' testlib.start_api("0123456789abcdef") # wait a while print 'Wait to verify that clearing out confirmed transactions does NOT remove zonefiles' time.sleep(10) # verify that this is still in the queue res = testlib.get_queue(ses, 'renew') if 'error' in res: print res return False if len(res) != 1: print res return False # clear the fault print 'Clearing renewal replication fault' testlib.ysi_test_setenv("BLOCKSTACK_TEST_REGISTRAR_FAULT_INJECTION_SKIP_RENEWAL_REPLICATION", "0") # now the renewal zonefile should replicate print 'Wait for renewal zonefile to replicate' time.sleep(10) # new expire block res = testlib.ysi_REST_call("GET", "/v1/names/bar.test", ses) if 'error' in res or res['http_status'] != 200: res['test'] = 'Failed to get name bar.test' print json.dumps(res) return False new_expire_block = res['response']['expire_block'] # do we have the history for the name? res = testlib.ysi_REST_call("GET", "/v1/names/bar.test/history", ses ) if 'error' in res or res['http_status'] != 200: res['test'] = "Failed to get name history for bar.test" print json.dumps(res) return False # valid history? hist = res['response'] if len(hist.keys()) != 3: res['test'] = 'Failed to get update history' res['history'] = hist print json.dumps(res, indent=4, sort_keys=True) return False # get the zonefile res = testlib.ysi_REST_call("GET", "/v1/names/bar.test/zonefile", ses ) if 'error' in res or res['http_status'] != 200: res['test'] = 'Failed to get name zonefile' print json.dumps(res) return False # zonefile must have old owner key zonefile_txt = res['response']['zonefile'] parsed_zonefile = ysi_zones.parse_zone_file(zonefile_txt) if not parsed_zonefile.has_key('txt'): print 'missing txt' print parsed_zonefile return False found = False for txtrec in parsed_zonefile['txt']: if txtrec['name'] == 'pubkey' and txtrec['txt'] == 'pubkey:data:{}'.format(wallets[3].pubkey_hex): found = True if not found: print 'missing public key {}'.format(wallets[3].pubkey_hex) return False # profile lookup must work res = testlib.ysi_REST_call("GET", "/v1/users/bar.test", ses) if 'error' in res or res['http_status'] != 200: res['text'] = 'failed to get profile for bar.test' print json.dumps(res) return False print '' print json.dumps(res['response'], indent=4, sort_keys=True) print '' # verify pushed back if old_expire_block + 10 > new_expire_block: # didn't go through print >> sys.stderr, "Renewal didn't work: %s --> %s" % (old_expire_block, new_expire_block) return False