def get_chunk(url): """ Get a chunk from onedrive, given its URL. Decompress and return it. """ res = None data = None urltype, urlres = get_url_type(url) if urltype is None and urlres is None: log.error("Invalid URL {}".format(url)) return None if urltype == 'onedrive': # request via OneDrive drive = get_onedrive_handle() log.debug("Fetch {} via onedrive ({})".format(url, urlres)) data = get_chunk_via_onedrive(drive, urlres) else: # request via HTTP log.debug("Fetch {} via HTTP".format(url)) data = get_chunk_via_http(url) if data is None: return None # decompress if ONEDRIVE_COMPRESS: try: res = decompress_chunk(data) except: res = data else: res = data return res
def read_chunk(chunk_path): """ Get a chunk of data from S3. Return the data on success Return None on error, and log an exception. """ global AWS_BUCKET bucket = get_bucket(AWS_BUCKET) if bucket == None: log.error("Failed to get bucket '%s'" % AWS_BUCKET) return None # replace / with \x2f chunk_path = chunk_path.replace("/", r"\x2f") k = Key(bucket) k.key = chunk_path data = None begin = None end = None size = None try: begin = time.time() compressed_data = k.get_contents_as_string() end = time.time() size = len(compressed_data) try: data = decompress_chunk(compressed_data) except: data = compressed_data except Exception, e: log.error("Failed to read '%s'" % chunk_path) log.exception(e)
def read_chunk( chunk_path ): """ Get a chunk of data from S3. Return the data on success Return None on error, and log an exception. """ global AWS_BUCKET bucket = get_bucket( AWS_BUCKET ) if bucket == None: log.error("Failed to get bucket '%s'" % AWS_BUCKET) return None # replace / with \x2f chunk_path = chunk_path.replace( "/", r"\x2f" ) k = Key(bucket) k.key = chunk_path data = None begin = None end = None size = None try: begin = time.time() compressed_data = k.get_contents_as_string() end = time.time() size = len(compressed_data) try: data = decompress_chunk( compressed_data ) except: data = compressed_data except Exception, e: log.error("Failed to read '%s'" % chunk_path) log.exception(e)
url=url, error=r.status_code)) else: compressed_data = r.text else: try: compressed_data = dvconf['driver_info']['api'].files_read( chunk_path) except: try: compressed_data = dvconf['driver_info']['api'].cat(chunk_path) except Exception, e: log.error("Failed to read file '{}'".format(chunk_path)) log.exception(e) try: data = decompress_chunk(compressed_data) except: data = compressed_data return data def ipfs_delete_chunk(dvconf, chunk_path): """ Delete a chunk of data from IPFS. """ try: dvconf['driver_info']['api'].files_rm(chunk_path) except Exception, e: try: dvconf['driver_info']['api'].pin_rm(chunk_path)