def multi_upload_fake(self, headers=None, local_file_path=None, part_size=commonconstants.DEFAULT_PART_SIZE): import sys bucket_name_check(self.bucket.name) object_name_check(self.name) pre_upload = self.init_multi_upload() #fp = open(local_file_path, 'rb') m = md5.new() file_size = sys.maxint * sys.maxint num_part = file_size / part_size if file_size % part_size != 0: num_part = num_part + 1 i = 0 while True: i += 1 retry_times=0 data_content=os.urandom(part_size / 1024) * 1024 m.update(data_content) print('part%d (%dM)' % (i, i * part_size / 1024 / 1024)) while retry_times<commonconstants.DEFAULT_RETRY_COUNT: response = self.upload_part(headers, data_content, i + 1, pre_upload['UploadId']) if response.status/100==2: break else: retry_times=retry_times+1 time.sleep(1) if retry_times>=commonconstants.DEFAULT_RETRY_COUNT: raise Exception("-2, after retry %s, failed, multi upload part failed!" % retry_times) result = self.get_uploaded_parts(None, pre_upload['UploadId']) part_submit_json = self._generate_part_json(result) data=self.complete_multi_upload(None, pre_upload['UploadId'], json.dumps(part_submit_json)) #fp.close() return data,m.hexdigest()
def multi_upload_fp_with_size(self,headers=None,fp=None,size=None,part_size=commonconstants.DEFAULT_PART_SIZE): bucket_name_check(self.bucket.name) object_name_check(self.name) m = md5.new() pre_upload = self.init_multi_upload() num_part = size / part_size if size % part_size != 0: num_part = num_part + 1 for i in range(num_part): retry_times=0 data_content=fp.read(part_size) m.update(data_content) while retry_times<commonconstants.DEFAULT_RETRY_COUNT: response = self.upload_part(headers, data_content, i + 1, pre_upload['UploadId']) if response.status/100==2: break else: retry_times=retry_times+1 if retry_times>=commonconstants.DEFAULT_RETRY_COUNT: raise Exception("-2, after retry %s, failed, multi upload part failed!" % retry_times) result = self.get_uploaded_parts(None, pre_upload['UploadId']) part_submit_json = self._generate_part_json(result) data=self.complete_multi_upload(None, pre_upload['UploadId'], json.dumps(part_submit_json)) fp.close() return data,m.hexdigest()
def upload_flow(self,headers=None,data=None,compute_MD5=True): bucket_name_check(self.bucket.name) object_name_check(self.name) if not headers: headers = {} headers['Content-Length'] = len(data) if compute_MD5: m = md5.new() m.update(data) headers['Content-MD5'] = m.hexdigest() headers['Content-Type'] = commonconstants.DEFAULT_CONTENT_TYPE # httplib.HTTPConnection("%s:%d") self.bucket.jss_client.open_connection_to_put('PUT', self.bucket.name, self.name, headers) offset=0 total_size = len(data) while offset < total_size: read_bytes = data[offset:offset + commonconstants.DEFAULT_SEND_SIZE] offset += commonconstants.DEFAULT_SEND_SIZE self.bucket.jss_client.send(read_bytes) response = self.bucket.jss_client.pool.getresponse() #print response.status if response.status / 100 > 2: error_handler(response) else: return response
def multi_upload(self, headers=None, local_file_path=None, part_size=commonconstants.DEFAULT_PART_SIZE, callback=None, prt = None): if not prt: prt = lambda *args, **kwargs:None bucket_name_check(self.bucket.name) object_name_check(self.name) pre_upload = self.init_multi_upload() fp = open(local_file_path, 'rb') m = md5.new() file_size = os.path.getsize(local_file_path) num_part = file_size / part_size if file_size % part_size != 0: num_part = num_part + 1 check_part_trytime = commonconstants.DEFAULT_RETRY_COUNT leak_parts = [i for i in range(1, num_part+1)]#init while check_part_trytime > 0: for i in range(num_part): retry_times=0 if len(leak_parts) == num_part or (i+1) in leak_parts:#firsr time or check leak data_content=fp.read(part_size) if len(leak_parts) == num_part:#first time m.update(data_content) else: fp.seek(part_size) continue while retry_times<commonconstants.DEFAULT_RETRY_COUNT: response = self.upload_part(headers, data_content, i + 1, pre_upload['UploadId']) if response.status/100==2: break else: retry_times=retry_times+1 time.sleep(1) if callback: callback(i,num_part,part_size) if retry_times>=commonconstants.DEFAULT_RETRY_COUNT: raise Exception("-2, after retry %s, failed, multi upload part failed!" % retry_times) result = self.get_uploaded_parts(None, pre_upload['UploadId']) #check #prt(result['Part']) if len(result['Part']) == num_part:#no leak break all_part_finnished = [int(p['PartNumber']) for p in result['Part']] leak_parts = [i for i in range(1, num_part+1) if i not in all_part_finnished] prt('%d parts leaked:%s' % (len(leak_parts), ','.join(map(str, leak_parts)))) check_part_trytime -= 1 time.sleep(3) part_submit_json = self._generate_part_json(result) data=self.complete_multi_upload(None, pre_upload['UploadId'], json.dumps(part_submit_json)) fp.close() return data,m.hexdigest()
def upload(self, headers=None, local_file_path=None, compute_MD5=True): """ :type :bucket_name:string :param :bucket_name:the bucket you file want to put :type :object_name:string :param :object_name:the key of the file you want to upload :type: headers:dict :param: headers:Additional headers to pass along with the request to JSS. :type local_file_path:string :param local_file_path:the path of the file which you want to upload. """ bucket_name_check(self.bucket.name) object_name_check(self.name) fq = open(local_file_path, 'rb') fq.seek(os.SEEK_SET, os.SEEK_END) filesize = fq.tell() fq.seek(os.SEEK_SET) suffix = local_file_path.split('.')[-1] if not headers: headers = {} headers['Content-Length'] = filesize if compute_MD5: fp = open(local_file_path) md5_value = file_MD5(fp) headers['Content-MD5'] = md5_value fp.close() #new add this time if '.' + suffix in self.bucket.jss_client.map: headers['Content-Type'] = self.bucket.jss_client.map['.' + suffix] else: headers['Content-Type'] = commonconstants.DEFAULT_CONTENT_TYPE # httplib.HTTPConnection("%s:%d") self.bucket.jss_client.open_connection_to_put('PUT', self.bucket.name, self.name, headers) fq.seek(os.SEEK_SET) l = fq.read(commonconstants.DEFAULT_SEND_SIZE) while len(l) > 0: self.bucket.jss_client.send(l) l = fq.read(commonconstants.DEFAULT_SEND_SIZE) response = self.bucket.jss_client.pool.getresponse() #print response.status if response.status / 100 > 2: error_handler(response) else: fq.close() # new add this time return response
def multi_thread_upload(self, headers=None, local_file_path=None, part_size=commonconstants.DEFAULT_PART_SIZE, num_thread=10, max_part=10000,timeout=None): bucket_name_check(self.bucket.name) object_name_check(self.name) pre_upload = self.init_multi_upload() task_queue = generate_all_task_slices(file_name=local_file_path, part_size=part_size, max_part=max_part) thread_list = [] for i in range(num_thread): jss_client = jss.connection.JssClient(access_key=self.bucket.jss_client.access_key, secret_key=self.bucket.jss_client.secret_key, host=self.bucket.jss_client.host, port=self.bucket.jss_client.port,timeout=timeout) thread_curr = Thread_Upload_Part(jss_client=jss_client, bucket_name=self.bucket.name, object_name=self.name, upload_id=pre_upload['UploadId'], all_task_slices=task_queue) thread_list.append(thread_curr) thread_curr.start() for item in thread_list: item.join() result = self.get_uploaded_parts(None, pre_upload['UploadId']) part_submit_json = self._generate_part_json(result) data=self.complete_multi_upload(None, pre_upload['UploadId'], json.dumps(part_submit_json)) return data
def create_bucket(self, bucket_name, headers=None): """ creates a new bucket type bucket_name: string :param bucket_name :The name of a new bucket type headers: dict :param headers: Additional headers to pass along with the request to JSS """ bucket_name_check(bucket_name) if not headers: headers={} headers['Content-Length']=0 response = self.make_request('PUT', bucket_name=bucket_name, headers=headers) if response.status / 100 > 2: error_handler(response) else: return response
def multi_thread_upload(self, headers=None, local_file_path=None, part_size=commonconstants.DEFAULT_PART_SIZE, num_thread=10, max_part=10000,timeout=None,prt=None): if not prt: prt = lambda *args, **kwargs:None bucket_name_check(self.bucket.name) object_name_check(self.name) pre_upload = self.init_multi_upload() check_part_trytime = commonconstants.DEFAULT_RETRY_COUNT file_size = os.stat(local_file_path).st_size num_part = file_size / part_size if file_size % part_size != 0: num_part = num_part + 1 leak_parts = [i for i in range(1, num_part+1)]#init while check_part_trytime > 0: task_queue = generate_all_task_slices(file_name=local_file_path, part_size=part_size, max_part=max_part, leak_parts = leak_parts) thread_list = [] for i in range(num_thread): jss_client = jss.connection.JssClient(access_key=self.bucket.jss_client.access_key, secret_key=self.bucket.jss_client.secret_key, host=self.bucket.jss_client.host, port=self.bucket.jss_client.port,timeout=timeout) thread_curr = Thread_Upload_Part(jss_client=jss_client, bucket_name=self.bucket.name, object_name=self.name, upload_id=pre_upload['UploadId'], all_task_slices=task_queue) thread_list.append(thread_curr) thread_curr.start() for item in thread_list: item.join() result = self.get_uploaded_parts(None, pre_upload['UploadId']) #check prt(result['Part']) if len(result['Part']) == num_part:#no leak break all_part_finnished = [int(p['PartNumber']) for p in result['Part']] leak_parts = [i for i in range(1, num_part+1) if i not in all_part_finnished] if len(leak_parts) == 0: break prt('%d parts leaked:%s' % (len(leak_parts), ','.join(map(str, leak_parts)))) check_part_trytime -= 1 time.sleep(3) part_submit_json = self._generate_part_json(result) data=self.complete_multi_upload(None, pre_upload['UploadId'], json.dumps(part_submit_json)) return data
def upload_fp(self, headers=None, fp=None): bucket_name_check(self.bucket.name) object_name_check(self.name) fp.seek(os.SEEK_SET, os.SEEK_END) filesize = fp.tell() fp.seek(os.SEEK_SET) if not headers: headers = {} headers['Content-Length'] = filesize headers['Content-Type'] = commonconstants.DEFAULT_CONTENT_TYPE # httplib.HTTPConnection("%s:%d") self.bucket.jss_client.open_connection_to_put('PUT', self.bucket.name, self.name, headers) l = fp.read(commonconstants.DEFAULT_SEND_SIZE) while len(l) > 0: self.bucket.jss_client.send(l) l = fp.read(commonconstants.DEFAULT_SEND_SIZE) response = self.bucket.jss_client.pool.getresponse() #print response.status if response.status / 100 > 2: error_handler(response) else: return response