def _extract_meta(minfo, timestamp):
    """ Extract some meta data from the response header, including

        uri - from request path
        date - from timestamp
        ETag or last-modified - from http header
        content-type - from http header (for its charset)

        The dictionary is case-sensitive and keys are in lowercase.
    """

    meta = {}
    uri = httputil.canonicalize(minfo.req_path)
    if len(uri) > g_maxuri:
        uri = uri[:g_maxuri] + '...'
    meta = {'uri': uri}                                                 # uri
    meta['date'] = timestamp                                            # date

    # Include either etag or last-modified
    # Otherwise use content-length as a weak ETag
    etag = minfo.rsp_headers.get('etag','')
    if etag:
        meta['etag'] = etag                                             # ETag
    else:
        last_modified = minfo.rsp_headers.get('last-modified','')
        if last_modified:
            meta['last-modified'] = last_modified                       # last-modified
        else:
            meta['etag'] = "W/%s" % minfo.clen                          # use clen as a weak etag

    meta['content-type'] = minfo.rsp_headers.get('content-type','')     # content-type

    meta['referer'] = minfo.req_headers.get('referer','')               # referer (from request)

    return meta
 def _append(self, parent, baseuri, uri, ctype, tag):
     abs_uri = urlparse.urljoin(baseuri, uri)
     abs_uri = httputil.canonicalize(abs_uri)    # TODO: test
     if abs_uri in self.uri_set:
         #log.warn('Skip repeated resource - %s %s' % (tag, uri))
         return
     res = Resource(parent, uri=abs_uri, ctype=ctype, tag=tag)
     if parent:
         parent.children.append(res)
     self.resource_list.append(res)
     self.uri_set.add(abs_uri)
     self.fetcher.queue(res)