def __init__(self, ref, callbacks, fetch_client): self.ref = ref self.urls = get_fetch_urls_for_ref(self.ref) self.callbacks = callbacks self.failures = 0 self.cancelled = False self.curl_fetch = None self.fetch_client = fetch_client
def simple_retrieve_object_for_ref(ref, decoder, jobid, master_uri): if isinstance(ref, SWErrorReference): raise Exception("Can't decode %s" % ref) if isinstance(ref, SW2_FutureReference) or isinstance(ref, SW2_StreamReference) or isinstance(ref, SW2_SocketStreamReference): ref = external_get_real_ref(ref, jobid, master_uri) if isinstance(ref, SWDataValue): return retrieve_object_for_ref(ref, decoder, None) elif isinstance(ref, SW2_ConcreteReference): urls = get_fetch_urls_for_ref(ref) _, content = httplib2.Http().request(urls[0]) return decoders[decoder](StringIO(content)) else: raise Exception("Don't know how to retrieve a %s" % ref)
def __init__(self, ref, callbacks, fetch_client): self.url = get_fetch_urls_for_ref(ref)[0] parsed_url = urlparse.urlparse(self.url) self.worker_netloc = parsed_url.netloc self.ref = ref self.fetch_client = fetch_client open(callbacks.bs_ctx.filename, "w").close() self.callbacks = callbacks self.current_data_fetch = None self.previous_fetches_bytes_downloaded = 0 self.remote_done = False self.remote_failed = False self.latest_advertisment = 0 self.cancelled = False self.local_done = False self.current_chunk_size = None self.subscribed_to_remote_adverts = True