f = open(cache_inst.temp_path+"temp", "w") f.write("some_text") f.close() cache_inst2 = RedisNetInfCache(storage_root, logger) cache_inst2.set_redis_conn(redis_conn) if os.path.isfile(cache_inst2.temp_path+"temp"): print"Temporaries not cleared" #---------------------------------------------------------------------------# ni_url = "ni://mumble.org/sha-256-32;uzFqGA" ni_name = NIname(ni_url) ni_name.validate_ni_url(has_params=True) ni_name_uv = NIname(ni_url) ni_name_np = NIname("ni:///sha-256") ni_name_np.validate_ni_url(has_params=False) print(str(ni_name.get_url())) md = NetInfMetaData(ni_name.get_canonical_ni_url(), "now", loc1="http://www.example.com", extrameta={ "something" : "else" }) print(md) md1 = NetInfMetaData("ni://abr.org/sha_256_64;fjhaie8978", "now", loc2="https://zzz.mumble.org", extrameta={ "something" : "else" }) print md1 try: m, f, n, i = cache_inst.cache_put(ni_name_uv, md, None) print "Fault: cache_put accepted unvalidated ni_name" except Exception, e: print "Error correctly detected: %s" % str(e) try:
def py_niget(): """ @brief Command line program to perform a NetInf 'get' operation using http @brief convergence layer. Uses NIproc global instance of NI operations class Run: > niget.py --help to see usage and options. Exit code is 0 for success, 1 if HTTP returned something except 200, and negative for local errors. """ # Options parsing and verification stuff usage = "%prog [-q] [-l] [-d] [-m|-v] [-f <pathname of content file>] <ni name>\n" + \ "<ni name> must include location (netloc) from which to retrieve object." parser = OptionParser(usage) parser.add_option("-f", "--file", dest="file_name", type="string", help="File to hold retrieved content. Defaults to hash code in current directory if not present") parser.add_option("-q", "--quiet", default=False, action="store_true", dest="quiet", help="Suppress textual output") parser.add_option("-s", "--server", dest="server", type="string", help="hostname:port of server to send the NetInf GET to") parser.add_option("-l", "--lax", default=False, action="store_true", dest="lax", help="Store returned content even if digest doesn't validate") parser.add_option("-m", "--metadata", default=False, action="store_true", dest="metadata", help="Output returned metadata as JSON string") parser.add_option("-v", "--view", default=False, action="store_true", dest="view", help="Pretty print returned metadata.") parser.add_option("-d", "--dump", default=False, action="store_true", dest="dump", help="Dump raw HTTP response to stdout.") (options, args) = parser.parse_args() # Check command line options - -q, -f, -l, -m, -v and -d are optional, <ni name> is mandatory if len(args) != 1: parser.error("URL <ni name> not specified.") sys.exit(-1) verbose = not options.quiet # Create NIname instance for supplied URL and validate it ni_url = NIname(args[0]) # Must be a complete ni: URL with non-empty params field rv = ni_url.validate_ni_url(has_params = True) if (rv != ni_errs.niSUCCESS): if verbose: print("Error: %s is not a complete, valid ni scheme URL: %s" % (ni_url.get_url(), ni_errs_txt[rv])) sys.exit(-2) # Generate file name for output if not specified if (options.file_name == None): options.file_name = ni_url.get_digest() # Generate NetInf form access URL if (options.server != None): server = options.server else: server = ni_url.get_netloc() http_url = "http://%s/netinfproto/get" % server """ if (http_url == None): if verbose: print("Error: Unable to generate http: transformed URL for ni URL %s" % ni_urlparse.get_url()) sys.exit(-3) """ # Set up HTTP form data for get request form_data = urllib.urlencode({ "URI": ni_url.get_url(), "msgid": random.randint(1, 32000), "ext": "" }) # Send POST request to destination server try: http_object = urllib2.urlopen(http_url, form_data) except Exception, e: if verbose: print("Error: Unable to access http URL %s: %s" % (http_url, str(e))) sys.exit(-4)
def py_nigetalt(): """ @brief Command line program to perform a NetInf 'get' operation using http @brief convergence layer. Uses NIproc global instance of NI operations class Run: > nigetalt.py --help to see usage and options. Exit code is 0 for success, 1 if HTTP returned something except 200, and negative for local errors. """ # Options parsing and verification stuff usage = "%prog [-q] [-l] [-d] [-m|-v] [-f <pathname of content file>] [-w <locator>] <ni name>\n" + \ "Either <ni name> must include location (netloc) from which to retrieve object, or\n" + \ "a locator must be given with the -w/--whence option.\n" + \ "The locator may be prefixed with an HTTP ('http://')or DTN ('dtn://') URI scheme identifier.\n" + \ "If no scheme identifier is given then HTTP is assumed. The DTN scheme does not accept ports." parser = OptionParser(usage) parser.add_option("-f", "--file", dest="file_name", type="string", help="File to hold retrieved content. Defaults to hash code in current directory if not present") parser.add_option("-w", "--whence", dest="loc", type="string", default=None, help="Locator to which to send NetInf GET request. May be prefixed with http:// or dtn://") parser.add_option("-q", "--quiet", default=False, action="store_true", dest="quiet", help="Suppress textual output") parser.add_option("-l", "--lax", default=False, action="store_true", dest="lax", help="Store returned content even if digest doesn't validate") parser.add_option("-m", "--metadata", default=False, action="store_true", dest="metadata", help="Output returned metadata as JSON string") parser.add_option("-v", "--view", default=False, action="store_true", dest="view", help="Pretty print returned metadata.") (options, args) = parser.parse_args() # Check command line options - -q, -f, -l, -m, and -v are optional, # <ni name> is mandatory # -w is optional if <ni name> contains a netloc if len(args) != 1: parser.error("URL <ni name> not specified.") sys.exit(-1) verbose = not options.quiet # Create NIname instance for supplied URL and validate it ni_url = NIname(args[0]) # Must be a complete ni: URL with non-empty params field rv = ni_url.validate_ni_url(has_params = True) if (rv != ni_errs.niSUCCESS): if verbose: print("Error: %s is not a complete, valid ni scheme URL: %s" % (ni_url.get_url(), ni_errs_txt[rv])) sys.exit(-2) # Generate file name for output if not specified if (options.file_name == None): options.file_name = ni_url.get_digest() # Decide Convergence Layer to use and locator to access netloc = ni_url.get_netloc() cl = HTTP_SCHEME if netloc == "": # Must have -w option if options.loc is None: if verbose: print("Error: Must provide a locator either in ni URI or via -w/--whence") sys.exit(-3) loc = options.loc.lower() elif options.loc is not None: if verbose: print("Warning: -w/--whence locator option overrides netloc in ni URI") loc = options.loc.lower() else: loc = netloc.lower() # See if URI scheme was specified if loc.startswith(HTTP_SCHEME): loc = loc[len(HTTP_SCHEME):] elif loc.startswith(DTN_SCHEME): loc = loc[len(DTN_SCHEME):] cl = DTN_SCHEME else: ssep = loc.find("://") if ssep != -1: if verbose: print("Error: Convergence Layer for scheme %s is not supported - use dtn or http" % loc[:ssep]) sys.exit(-4) # Default assume HTTP # Action the GET according to CL selected if cl == HTTP_SCHEME: json_report, got_content, faulty = get_via_http(ni_url, loc, options.file_name, verbose, options.lax) else: json_report, got_content, faulty = get_via_dtn(ni_url, loc, options.file_name, verbose, options.lax) if options.view: print("Returned metadata for %s:" % args[0]) print json.dumps(json_report, indent = 4) elif options.metadata: print json.dumps(json_report, separators=(",", ":")) if not got_content: rv = 1 elif faulty: rv = 2 else: rv = 0 if verbose and got_content: if not faulty: print("Content successfully retrieved and placed in file %s." % options.file_name) else: print("Content retrieved and placed in file %s but digest didn't verify." % options.file_name) elif verbose: print("Only metadata retrieved") sys.exit(rv)
def py_nicl(): """ @brief Command line program to generate and validate digests in ni: URLs. Uses NIproc global instance of NI operations class Run: > nicl.py --help to see usage and options. """ # Options parsing and verification stuff usage = "%prog [-g|-w|-v] -n <name> -f <pathname of content file> [-V]\n" usage = usage + " %prog -m -n <name> [-V]\n" usage = usage + " %prog -b -s <suite_number> -f <pathname of content file> [-V]\n" usage = usage + " The name can be either an ni: or nih: scheme URI\n" usage = usage + " Return code: success 0, failure non-zero (-V for more info)\n" usage = usage + " Available hashalg (suite number) options:\n" usage = usage + " %s" % NIname.list_algs() parser = OptionParser(usage) parser.add_option("-g", "--generate", default=False, action="store_true", dest="generate", help="Generate hash based on content file, " + \ "and output name with encoded hash after the hashalg string") parser.add_option("-w", "--well-known", default=False, action="store_true", dest="well_known", help="Generate hash based on content file, " + \ "and output name with encoded hash in the .well_known URL " + \ "after the hashalg string. Applies to ni: scheme only.") parser.add_option("-v", "--verify", default=False, action="store_true", dest="verify", help="Verify hash in name is correct for content file") parser.add_option("-m", "--map", default=False, action="store_true", dest="map_wkn", help="Maps from an ni: name to a .well-known URL") parser.add_option("-b", "--binary", default=False, action="store_true", dest="bin", help="Outputs the name in binary format for a given suite number") parser.add_option("-V", "--verbose", default=False, action="store_true", dest="verbose", help="Be more long winded.") parser.add_option("-n", "--ni-name", dest="ni_name", type="string", help="The ni name template for (-g) or ni name matching (-v) content file.") parser.add_option("-f", "--file", dest="file_name", type="string", help="File with content data named by ni name.") parser.add_option("-s", "--suite-no", dest="suite_no", type="int", help="Suite number for hash algorithm to use.") (opts, args) = parser.parse_args() if not (opts.generate or opts.well_known or opts.verify or opts.map_wkn or opts.bin ): parser.error( "Must specify one of -g/--generate, -w/--well-known, -v/--verify, -m/--map or -b/--binary.") if opts.generate or opts.well_known or opts.verify: if (opts.ni_name == None) or (opts.file_name == None): parser.error("Must specify both name and content file name for -g, -w or -v.") if opts.map_wkn: if (opts.ni_name == None): parser.error("Must specify ni name for -m.") if opts.bin: if (opts.suite_no == None) or (opts.file_name == None): parser.error("Must specify both suite number and content file name for -b.") if len(args) != 0: parser.error("Too many or unrecognised arguments specified") # Execute requested action if opts.generate: n = NIname(opts.ni_name) ret = NIproc.makenif(n, opts.file_name) if ret == ni_errs.niSUCCESS: if opts.verbose: print("Name generated successfully.") print "%s" % n.get_url() sys.exit(0) if opts.verbose: print "Name could not be successfully generated." elif opts.well_known: n = NIname(opts.ni_name) if n.get_scheme() == "nih": if opts.verbose: print "Only applicable to ni: scheme names." sys.exit(1) ret = NIproc.makenif(n, opts.file_name) if ret == ni_errs.niSUCCESS: if opts.verbose: print("Name generated successfully.") print "%s" % n.get_wku_transform() sys.exit(0) if opts.verbose: print "Name could not be successfully generated" elif opts.verify: n = NIname(opts.ni_name) ret = NIproc.checknif(n, opts.file_name) if ret == ni_errs.niSUCCESS: if opts.verbose: print("Name matches content file.") print "%s" % n.get_url() sys.exit(0) if opts.verbose: print "Check of name against content failed." elif opts.map_wkn: n = NIname(opts.ni_name) ret = n.validate_ni_url(has_params = True) if ret == ni_errs.niSUCCESS: if n.get_scheme() == "nih": if opts.verbose: print "Only applicable to ni: scheme names." sys.exit(1) if opts.verbose: print("Name validated successfully.") print "%s" % n.get_wku_transform() sys.exit(0) else: if opts.verbose: print "Name could not be successfully validated." elif opts.bin: (ret, bin_name) = NIproc.makebnf(opts.suite_no, opts.file_name) if ret == ni_errs.niSUCCESS: if opts.verbose: print("Name generated successfully.") print base64.b16encode(str(bin_name)) sys.exit(0) else: if opts.verbose: print "Name could not be successfully generated." else: print"Should not have happened" sys.exit(2) # Print appropriate error message if opts.verbose: print "Error: %s" % ni_errs_txt[ret] sys.exit(1) sys.exit(0)
def publish_with_dtn(ni_name, destination, authority, hash_alg, ext_json, locs, scheme, full_put, file_name, rform, verbose): """ @brief Action a NetInf publish request using the HTTP convergence layer @param ni_name NIname object instance or None - ni URI to publish if given on comand line - otherwise will be constructed @param destination string netloc (FQDN or IP address with optional port) indicating where to send publish request @param authority string netloc component to insert into ni name (may be "") @param hash_alg string name of hash algorithm used for ni URI @param ext_json dictionary additional information to send with request if any in the form of a JSON dictionary or None @param locs list of strings with locators to publish - maybe None @param scheme URI scheme used for ni URI @param full_put boolean True if the file_name with the content was given @param file_name string name of file to publish or None if only doing metadata @param rform string request format of response @param verbose bolean indicates how much error message output is produced @return 2-tuple - target - string the actual ni name published payload - string - the response received on publication """ debug("Publishing via: %s" % destination) # Handle full_put = True cases - we have a file with the octets in it if full_put: if ni_name is None: # Make a ni_name template from specified components ni_name = NIname((scheme, authority, "/%s" % hash_alg)) # Construct the digest from the file name and the template rv = NIproc.makenif(ni_name, file_name) if rv != ni_errs.niSUCCESS: raise PublishFailure("Unable to construct digest of file %s: %s" % (file_name, ni_errs_txt[rv]), -20) else: # Check the ni_name and the file match rv = Niproc.checknif(ni_name, file_name) if rv != ni_errs.niSUCCESS: raise PublishFailure("Digest of file %s does not match ni_name %s: %s" % (file_name, ni_name.get_url(), ni_errs_txt[rv]), -21) # Guess the mimetype of the file m = magic.Magic(mime=True) ctype = m.from_file(file_name) debug("Content-Type: %s" % ctype) if ctype is None: # Guessing didn't work - default ctype = "application/octet-stream" else: ctype = None target = ni_name.get_canonical_ni_url() debug("Using URI string: %s" % target) # Add extra items to ext_json to pass across as metadata ext_json["ni"] = target if ctype is not None: ext_json["ct"] = ctype if authority != "": ext_json["http_auth"] = authority # Send at most two locators as a list if (locs is not None): ext_json["loclist"] = locs[:2] ext_json["fullPut"] = full_put ext_json["rform"] = rform # Create a connection to the DTN daemon dtn_handle = dtnapi.dtn_open() if dtn_handle == -1: raise PublishFailure("Error: unable to open connection with DTN daemon", -22) # Generate EID + service tag for service to be accessed via DTN if destination is None: remote_service_eid = \ dtnapi.dtn_build_local_eid(dtn_handle, "netinfproto/service/publish") i = remote_service_eid.find("/netinfproto") destination = remote_service_eid[:i] else: remote_service_eid = destination + "/netinfproto/service/publish" # Add destination to locs if it isn't there already if locs is None: locs = [] if destination not in locs: locs.append(destination) # Generate the EID and service tag for this service local_service_eid = dtnapi.dtn_build_local_eid(dtn_handle, "netinfproto/app/response") debug("Local Service EID: %s" % local_service_eid) debug("Remote Service EID: %s" % remote_service_eid) # Check if service_eid registration exists and register if not # Otherwise bind to the existing registration regid = dtnapi.dtn_find_registration(dtn_handle, local_service_eid) if (regid == -1): # Need to register the EID.. make it permanent with 'DEFER' # characteristics so that bundles are saved if they arrive # while the handler is inactive # Expire the registration an hour in the future exp = 60 * 60 # The registration is immediately active passive = False # We don't want to execute a script script = "" regid = dtnapi.dtn_register(dtn_handle, local_service_eid, dtnapi.DTN_REG_DEFER, exp, passive, script) else: dtnapi.dtn_bind(dtn_handle, regid) # Build the bundle to send # First a suitable BPQ block bpq = BPQ() bpq.set_bpq_kind(BPQ.BPQ_BLOCK_KIND_PUBLISH) bpq.set_matching_rule(BPQ.BPQ_MATCHING_RULE_EXACT) bpq.set_src_eid(local_service_eid) sent_msgid = str(random.randint(1, 32000)) bpq.set_bpq_id(sent_msgid) bpq.set_bpq_val(target) bpq.clear_frag_desc() # Build an extension blocks structure to hold the block ext_blocks = dtnapi.dtn_extension_block_list(1) # Construct the extension block bpq_block = dtnapi.dtn_extension_block() bpq_block.type = QUERY_EXTENSION_BLOCK bpq_block.flags = 0 bpq_block.data = bpq.build_for_net() ext_blocks.blocks.append(bpq_block) # Build an extension blocks structure to hold the block meta_blocks = dtnapi.dtn_extension_block_list(2) # Build a metadata block for JSON data md = Metadata() md.set_ontology(Metadata.ONTOLOGY_JSON) md.set_ontology_data(json.dumps(ext_json)) json_block = dtnapi.dtn_extension_block() json_block.type = METADATA_BLOCK json_block.flags = 0 json_block.data = md.build_for_net() meta_blocks.blocks.append(json_block) # Set up payload and placeholder if needed if full_put: # No placeholder required (obviously!) pt = dtnapi.DTN_PAYLOAD_FILE pv = file_name else: # DTN bundle always has a payload - distinguish # zero length file form no content available # Payload is the empty string sent via memory pt = dtnapi.DTN_PAYLOAD_MEM pv = "" # Add a payload placeholder metablock md = Metadata() md.set_ontology(Metadata.ONTOLOGY_PAYLOAD_PLACEHOLDER) md.set_ontology_data("No content supplied") pp_block = dtnapi.dtn_extension_block() pp_block.type = METADATA_BLOCK pp_block.flags = 0 pp_block.data = md.build_for_net() meta_blocks.blocks.append(pp_block) # We want delivery reports and publication reports # (and maybe deletion reports?) dopts = dtnapi.DOPTS_DELIVERY_RCPT | dtnapi.DOPTS_PUBLICATION_RCPT # - Send with normal priority. pri = dtnapi.COS_NORMAL # NetInf bundles should last a while.. exp = (24 *60 * 60) # Send the bundle bundle_id = dtnapi.dtn_send(dtn_handle, regid, local_service_eid, remote_service_eid, local_service_eid, pri, dopts, exp, pt, pv, ext_blocks, meta_blocks, "", "") if bundle_id == None: raise PublishFailure("dtn_send failed - %s" % dtnapi.dtn_strerror(dtnapi.dtn_errno(dtn_handle)), -23) # Wait for a reponse - maybe also some reports while(True): # NOTE: BUG in dtnapi - timeout is in msecs recv_timeout = 2000 * 60 bpq_bundle = dtnapi.dtn_recv(dtn_handle, dtnapi.DTN_PAYLOAD_FILE, recv_timeout) # If bpq_bundle is None then either the dtn_recv timed out or # there was some other error. if bpq_bundle != None: # Filter out report bundles if bpq_bundle.status_report != None: debug("Received status report") if bpq_bundle.status_report.flags == dtnapi.STATUS_DELIVERED: if verbose: print("Received delivery report re from %s sent %d seq %d" % (bpq_bundle.status_report.bundle_id.source, bpq_bundle.status_report.bundle_id.creation_secs, bpq_bundle.status_report.bundle_id.creation_seqno)) elif bpq_bundle.status_report.flags == dtnapi.STATUS_DELETED: if verbose: print("Received deletion report re from %s sent %d seq %d" % (bpq_bundle.status_report.bundle_id.source, bpq_bundle.status_report.bundle_id.creation_secs, bpq_bundle.status_report.bundle_id.creation_seqno)) elif bpq_bundle.status_report.flags == dtnapi.STATUS_PUBLISHED: if verbose: print("Received publication report re from %s sent %d seq %d" % (bpq_bundle.status_report.bundle_id.source, bpq_bundle.status_report.bundle_id.creation_secs, bpq_bundle.status_report.bundle_id.creation_seqno)) else: if verbose: print("Received unexpected report: Flags: %d" % bpq_bundle.status_report.flags) # Wait for more status reports and incoming response continue # Check the payload really is in a file if not bpq_bundle.payload_file: raise PublishFailure("Received bundle payload not in file - " "ignoring bundle", -24) # Have to delete this file before an error exit or if empty pfn = bpq_bundle.payload l = len(pfn) if pfn[l-1] == "\x00": pfn = pfn[:-1] debug("Got incoming bundle with response in file %s" % pfn) # Does the bundle have a BPQ block bpq_data = None if bpq_bundle.extension_cnt == 0: os.remove(pfn) raise PublishFailure("Error: Received bundle with no " "extension block.", -25) for blk in bpq_bundle.extension_blks: if blk.type == QUERY_EXTENSION_BLOCK: bpq_data = BPQ() if not bpq_data.init_from_net(blk.data): os.remove(pfn) raise PublishFailure("Error: Bad BPQ block received", -26) if bpq_data is None: os.remove(pfn) raise PublishFailure("Error: Received bundle with no BPQ block " "in extension blocks", -27) debug(bpq_data) # OK.. got the response - finish with daemon break elif dtnapi.dtn_errno(dtn_handle) != dtnapi.DTN_ETIMEOUT: raise PublishFailure(dtnapi.dtn_strerror(dtnapi.dtn_errno(dtn_handle)), -28) else: raise PublishFailure("dtn_recv timed out without receiving " "response bundle", 1) dtnapi.dtn_close(dtn_handle) # Check the BPQ data is right bpq.set_bpq_kind(BPQ.BPQ_BLOCK_KIND_PUBLISH) bpq.set_matching_rule(BPQ.BPQ_MATCHING_RULE_EXACT) if bpq_data.bpq_kind != BPQ.BPQ_BLOCK_KIND_PUBLISH: raise PublishFailure("Returned BPQ block is not PUBLISH kind: %d" % bpq_data.bpq_kind, -29) if bpq_data.matching_rule != BPQ.BPQ_MATCHING_RULE_NEVER: raise PublishFailure("Returned BPQ block does not have NEVER matching rule: %d" % bpq_data.matching_rule, -30) if bpq_data.bpq_id != sent_msgid: raise PublishFailure("Returned BPQ block has unmatched msgis %s vs %s" % (bpq_data.bpq_id, sent_msgid), -31) # Verify the format of the response (a bit) try: pfd = open(pfn, "rb") payload = pfd.read() pfd.close() os.remove(pfn) except Exception, e: raise PublishFailure("Failed to read response from payload file %s" % pfn, -32)