Esempio n. 1
0
class Connection:
    """
    Holds connection and authentication data for the Plexus server and
    provides a method for uploading files.
    
    Mandatory constructor arguments are <user> and <password> for
    authentication with Plexus. Optional argument are
    
        <server>      - the base URL for the Plexus web application
        <manager>     - the manager for newly created projects,
        <interactive> - if True, asks for missing login and/or password
    """
    
    def __init__(self,
                 user,
                 password,
                 server      = "https://plexus.anu.edu.au",
                 manager     = None,
                 interactive = False):
        # -- if interactive, ask for authentication data with the service
        if interactive:
            import getpass
            while user in (None, ""):
                user = raw_input("Server login: "******""):
                password = getpass.getpass("Server password: "******""
            
        # -- set default retry parameters
        self.retry_limit = 10  # number of upload attempts
        self.retry_wait  = 300 # waiting period between upload attempts (sec)
        
        self.log = Logger()

    def post_form(self, selector, fields, files):
        """
        Composes an multipart/form-data message, posts it to the URL on
        the server given by the relative path <selector> and returns a
        tuple consisting of the HTTP status, reason and response text
        as received from the server.
        
        Normal fields are specified as a sequence <fields> of
        (name, value) pairs. Attached file data is specified as a
        sequence <files> of (name, filename, value) elements.
        """
        
        content_type, body = encode_formdata(fields, files)
        headers = {
            'User-Agent': 'python',
            'Content-Type': content_type,
            'Accept': 'application/json'
            }

        count = 0
        while True:
            try:
                if self.server.startswith("https://"):
                    host = re.sub('^https:\/\/', '', self.server)
                    self.log.writeln("Connecting to %s using https" % host)
                    h = httplib.HTTPSConnection(host)
                else:
                    host = re.sub('^http:\/\/', '', self.server)
                    self.log.writeln("Connecting to %s using http" % host)
                    h = httplib.HTTPConnection(host)
                        
                h.request('POST', selector, body, headers)
                res = h.getresponse()
                return res.status, res.reason, res.read()
            except KeyboardInterrupt, ex:
                raise ex
            except Exception, ex:
                self.log.writeln("> %s: %s <" % (ex.__class__.__name__, ex),
                                 LOGGER_ERROR)
                count += 1
                if count < self.retry_limit:
                    self.log.writeln("(will retry in %d seconds)"
                                     % self.retry_wait)
                    time.sleep(self.retry_wait)
                    self.log.writeln("retrying...")
                else:
                    self.log.writeln("too many retries - giving up",
                                     LOGGER_ERROR)
                    raise ex
Esempio n. 2
0
def slices(path,
           existing = [],
           replace = False,
           dry_run = False,
           sizes = (None,),
           info = {}):
    """
    A generator which extracts slice images from a Mango volume data set
    stored in a collection of NetCDF files.

    The parameter <path> specifies a single NetCDF file or a directory
    containing a single volume split into several NetCDF files.

    Basic usage:
        for (data, name, action) in slices(path):
            fp = file(name, 'wb')
            fp.write(data)
            fp.close()
    """

    path      = re.sub("/$", "", path)
    name      = re.sub("[._]nc$", "", os.path.basename(path))
    basename  = re.sub("^tomo", "tom", re.sub("^segmented", "seg", name))
    img_mode  = None
    log       = Logger()

    # -- collect the list of files to be processed
    entries = datafiles(path)
    if not entries:
        return

    # -- find a useable volume variable and analyse it
    filename = entries[0]
    log.writeln("slices(): looking for a volume variable in %s..." %
                os.path.basename(filename))
    var = find_variable(filename)
    if var is None :
        log.writeln("No appropriate volume data found.")
        return

    # -- determine the appropriate mask value
    mask_value = { numpy.uint8:  0xff,
                   numpy.uint16: 0xffff,
                   numpy.int32:  0x7fffffff,
                   numpy.float32: 1.0e30 }[var['dtype']]

    # -- initialize the slice set to be created
    slices = default_slice_set(var, 10, basename)
    r_or_s = 'REPLACE' if replace else 'SKIP'
    actions = list((r_or_s if n in existing else 'ADD')
                   for (s, n) in slices)
    slices = list(slices[i] + (actions[i],)
                  for i in range(len(slices))
                  if actions[i] != 'SKIP')

    if len(slices) == 0:
        log.writeln("No slices are to be made.")
        return

    if not dry_run:
        # -- initialize the histogram
        if var['dtype'] == numpy.float32:
            log.writeln("Determining the data range...")
            (minval, maxval) = data_range(var, entries)
            hist = Histogram(mask_value, minval, maxval)
        else:
            hist = Histogram(mask_value)

        # -- loop through files and copy data into slice arrays
        for filename in entries:
            log.writeln("Processing %s..." % os.path.basename(filename))
            for tmp in z_slices(var, filename):
                z, data = tmp[:2]
                if data is None:
                    log.writeln(tmp[2] + " at z = %d" % z, LOGGER_WARNING)
                else:
                    hist.update(data)
                    for (s, n, a) in slices:
                        s.update(data, z)

        # -- analyse histogram to determine 'lo' and 'hi' values
        log.writeln("Analysing the histogram...")
        if name.startswith("tom"):
            # -- determine 0.1 and 99.9 percentile for contrast stretching
            lo = bottom_percentile(hist, 0.1)
            hi = top_percentile(hist, 0.1)
        else:
            lo = 0
            hi = hist.counts.size - 1

    # -- encode slices as PNG images
    log.writeln("Making the images...")
    for (s, n, a) in slices:
        for sz in sizes:
            if dry_run:
                data = make_image.make_dummy(n, 256, 256, sz)
            else:
                data = image_data(s, lo, hi, mask_value, info, sz)
            prefix = ("__%sx%s__" % sz) if sz else ""
            yield (data, prefix + n, a)

    # -- report success
    log.writeln("Slice image generation finished.")