def upload(self, local_file_path, rest_resource_path):
"""
Add log message
@type local_file_path: string
@param local_file_path: Local file path
@type rest_resource_path: string
@param rest_resource_path: Resource file path (without leading slash!)
@rtype: string
@return: Relative HDFS path
"""
file_resource_uri = self.rest_endpoint.get_resource_uri(
rest_resource_path)
if local_file_path is not None:
# with open(local_file_path, 'r') as f:
# strip path and extension from absolute file path to get filename
filename = local_file_path.rpartition('/')[2]
chunks = FileBinaryDataChunks(local_file_path, 65536,
self.progress_reporter).chunks()
file_size = os.path.getsize(local_file_path)
streamer = StreamingIterator(file_size, chunks)
content_type = 'multipart/form-data'
s = requests.Session()
s.auth = ('admin', 'admin')
r = s.put(file_resource_uri.format(filename),
data=streamer,
headers={'Content-Type': content_type})
return ResponseWrapper(success=True, response=r)
else:
return ResponseWrapper(success=False)
def upload_to_hdfs(self, local_file_path, rest_resource_path):
"""
Add log message
@type local_file_path: string
@param local_file_path: Local file path
@type rest_resource_path: string
@param rest_resource_path: Resource file path (without leading slash!)
@rtype: string
@return: Relative HDFS path
"""
file_resource_uri = self.rest_endpoint.get_resource_uri(
rest_resource_path)
if local_file_path is not None:
# with open(local_file_path, 'r') as f:
# strip path and extension from absolute file path to get filename
filename = local_file_path.rpartition('/')[2]
chunks = FileBinaryDataChunks(local_file_path, 65536,
self.progress_reporter).chunks()
file_resource_uri_formatted = file_resource_uri.format(filename)
logger.debug("Uploading file: %s" % local_file_path)
logger.debug("Upload resource uri: %s" %
file_resource_uri_formatted)
r = requests.put(file_resource_uri_formatted, data=chunks)
return ResponseWrapper(success=True, response=r)
else:
return ResponseWrapper(success=False)
def upload(self, local_file_path, ipuuid=None, chunk_size=1048576 * 10):
"""
Add log message
@type local_file_path: string
@param local_file_path: Local file path
@rtype: string
@return: Relative HDFS path
"""
if local_file_path is not None:
# with open(local_file_path, 'r') as f:
# strip path and extension from absolute file path to get filename
hash = hashlib.md5()
filename = local_file_path.rpartition('/')[2]
file_size = os.path.getsize(local_file_path)
if chunk_size > file_size:
chunk_size = file_size
chunks = FileBinaryDataChunks(local_file_path, chunk_size,
self.progress_reporter).chunks()
num = 0
offset_start = 0
offset_stop = chunk_size - 1
upload_id = ''
for chunk in chunks:
hash.update(chunk)
if num == 0:
HTTP_CONTENT_RANGE = 'bytes %s-%s/%s' % (
offset_start, offset_stop, file_size)
m = MultipartEncoder(
fields={
'the_file': (filename, chunk,
'application/octet-stream'),
})
else:
offset_start = offset_stop + 1
offset_stop = offset_stop + chunk_size
if offset_stop > file_size:
offset_stop = file_size - 1
#print 'last offset_stop: %s file_size: %s' % (offset_stop, file_size)
HTTP_CONTENT_RANGE = 'bytes %s-%s/%s' % (
offset_start, offset_stop, file_size)
m = MultipartEncoder(
fields={
'upload_id':
upload_id,
'the_file': (filename, chunk,
'application/octet-stream'),
})
headers = {
'Content-Type': m.content_type,
'Content-Range': HTTP_CONTENT_RANGE
}
try:
#r = self.requests_session.post(self.rest_endpoint, data=m, headers=headers)
r = self.requests_post(self.rest_endpoint,
data=m,
headers=headers)
except UploadPostWarning as e:
raise UploadError(e)
r_json = r.json()
upload_id = r_json['upload_id']
offset = r_json['offset']
expires = r_json['expires']
#print 'upload_id: %s, c_num: %s, offset: %s' % (upload_id, num, offset)
num += 1
m = MultipartEncoder(fields={
'upload_id': upload_id,
'md5': hash.hexdigest(),
'ipuuid': ipuuid
})
headers = {'Content-Type': m.content_type}
try:
#r = self.requests_session.post(self.rest_endpoint+'_complete', data=m, headers=headers)
r = self.requests_post(self.rest_endpoint + '_complete',
data=m,
headers=headers)
except UploadPostWarning as e:
raise UploadError(e)
return 'Success to upload %s' % local_file_path