def get_chunk(url):
"""
Get a chunk from onedrive, given its URL.
Decompress and return it.
"""
res = None
data = None
urltype, urlres = get_url_type(url)
if urltype is None and urlres is None:
log.error("Invalid URL {}".format(url))
return None
if urltype == 'onedrive':
# request via OneDrive
drive = get_onedrive_handle()
log.debug("Fetch {} via onedrive ({})".format(url, urlres))
data = get_chunk_via_onedrive(drive, urlres)
else:
# request via HTTP
log.debug("Fetch {} via HTTP".format(url))
data = get_chunk_via_http(url)
if data is None:
return None
# decompress
if ONEDRIVE_COMPRESS:
try:
res = decompress_chunk(data)
except:
res = data
else:
res = data
return res
def get_chunk(url):
"""
Get a chunk from onedrive, given its URL.
Decompress and return it.
"""
res = None
data = None
urltype, urlres = get_url_type(url)
if urltype is None and urlres is None:
log.error("Invalid URL {}".format(url))
return None
if urltype == 'onedrive':
# request via OneDrive
drive = get_onedrive_handle()
log.debug("Fetch {} via onedrive ({})".format(url, urlres))
data = get_chunk_via_onedrive(drive, urlres)
else:
# request via HTTP
log.debug("Fetch {} via HTTP".format(url))
data = get_chunk_via_http(url)
if data is None:
return None
# decompress
if ONEDRIVE_COMPRESS:
try:
res = decompress_chunk(data)
except:
res = data
else:
res = data
return res
def read_chunk(chunk_path):
"""
Get a chunk of data from S3.
Return the data on success
Return None on error, and log an exception.
"""
global AWS_BUCKET
bucket = get_bucket(AWS_BUCKET)
if bucket == None:
log.error("Failed to get bucket '%s'" % AWS_BUCKET)
return None
# replace / with \x2f
chunk_path = chunk_path.replace("/", r"\x2f")
k = Key(bucket)
k.key = chunk_path
data = None
begin = None
end = None
size = None
try:
begin = time.time()
compressed_data = k.get_contents_as_string()
end = time.time()
size = len(compressed_data)
try:
data = decompress_chunk(compressed_data)
except:
data = compressed_data
except Exception, e:
log.error("Failed to read '%s'" % chunk_path)
log.exception(e)
def read_chunk( chunk_path ):
"""
Get a chunk of data from S3.
Return the data on success
Return None on error, and log an exception.
"""
global AWS_BUCKET
bucket = get_bucket( AWS_BUCKET )
if bucket == None:
log.error("Failed to get bucket '%s'" % AWS_BUCKET)
return None
# replace / with \x2f
chunk_path = chunk_path.replace( "/", r"\x2f" )
k = Key(bucket)
k.key = chunk_path
data = None
begin = None
end = None
size = None
try:
begin = time.time()
compressed_data = k.get_contents_as_string()
end = time.time()
size = len(compressed_data)
try:
data = decompress_chunk( compressed_data )
except:
data = compressed_data
except Exception, e:
log.error("Failed to read '%s'" % chunk_path)
log.exception(e)
url=url, error=r.status_code))
else:
compressed_data = r.text
else:
try:
compressed_data = dvconf['driver_info']['api'].files_read(
chunk_path)
except:
try:
compressed_data = dvconf['driver_info']['api'].cat(chunk_path)
except Exception, e:
log.error("Failed to read file '{}'".format(chunk_path))
log.exception(e)
try:
data = decompress_chunk(compressed_data)
except:
data = compressed_data
return data
def ipfs_delete_chunk(dvconf, chunk_path):
"""
Delete a chunk of data from IPFS.
"""
try:
dvconf['driver_info']['api'].files_rm(chunk_path)
except Exception, e:
try:
dvconf['driver_info']['api'].pin_rm(chunk_path)