Ejemplo n.º 1
0
 def run(self):
     for part in self.part_msg_list:
         part_number = str(part[0])
         if len(part) == 5:
             bucket = self.bucket
             object = self.object
             partsize = part[3]
             offset = part[4]
             retry_times = self.retry_times
             while True:
                 try:
                     if self.uploaded_part_map.has_key(part_number):
                         md5 = part[2]
                         if self.uploaded_part_map[part_number].replace(
                                 '"', "").upper() == md5.upper():
                             break
                     if retry_times <= 0:
                         break
                     res = self.oss.upload_part_from_file_given_pos(
                         bucket, object, self.file_path, offset, partsize,
                         self.upload_id, part_number)
                     if res.status != 200:
                         retry_times = retry_times - 1
                         time.sleep(1)
                     else:
                         etag = res.getheader("etag")
                         if etag:
                             self.uploaded_part_map[part_number] = etag
                         break
                 except:
                     retry_times = retry_times - 1
                     time.sleep(1)
         else:
             print "not expected part for multiupload", part
             pass
Ejemplo n.º 2
0
    def run(self):
        for part in self.part_msg_list:
            if len(part) == 5:
                bucket = self.bucket
                file_name = part[1]
                if isinstance(file_name, unicode):
                    filename = file_name.encode('utf-8')
                object_name = file_name
                retry_times = self.retry_times
                is_skip = False
                while True:
                    try:
                        if retry_times <= 0:
                            break
                        res = self.oss.head_object(bucket, object_name)
                        if res.status == 200:
                            header_map = convert_header2map(res.getheaders())
                            etag = safe_get_element("etag", header_map)
                            md5 = part[2]
                            if etag.replace('"', "").upper() == md5.upper():
                                is_skip = True
                        break
                    except:
                        retry_times = retry_times - 1
                        time.sleep(1)

                if is_skip:
                    continue

                partsize = part[3]
                offset = part[4]
                retry_times = self.retry_times
                while True:
                    try:
                        if retry_times <= 0:
                            break
                        res = self.oss.put_object_from_file_given_pos(
                            bucket, object_name, self.file_path, offset,
                            partsize)
                        if res.status != 200:
                            print "upload ", file_name, "failed!", " ret is:", res.status
                            print "headers", res.getheaders()
                            retry_times = retry_times - 1
                            time.sleep(1)
                        else:
                            break
                    except:
                        retry_times = retry_times - 1
                        time.sleep(1)

            else:
                print "ERROR! part", part, " is not as expected!"
Ejemplo n.º 3
0
    def run(self):
        for part in self.part_msg_list:
            if len(part) == 5:
                bucket = self.bucket
                file_name = part[1]
                if isinstance(file_name, unicode):
                    filename = file_name.encode('utf-8')
                object_name = file_name
                retry_times = self.retry_times
                is_skip = False
                while True:
                    try:
                        if retry_times <= 0:
                            break
                        res = self.oss.head_object(bucket, object_name)
                        if res.status == 200:
                            header_map = convert_header2map(res.getheaders())
                            etag = safe_get_element("etag", header_map)
                            md5 = part[2]
                            if etag.replace('"', "").upper() == md5.upper():
                                is_skip = True
                        break
                    except:
                        retry_times = retry_times - 1
                        time.sleep(1)

                if is_skip:
                    continue

                partsize = part[3]
                offset = part[4]
                retry_times = self.retry_times
                while True:
                    try:
                        if retry_times <= 0:
                            break
                        res = self.oss.put_object_from_file_given_pos(bucket, object_name, self.file_path, offset, partsize)
                        if res.status != 200:
                            print "upload ", file_name, "failed!"," ret is:", res.status
                            print "headers", res.getheaders()
                            retry_times = retry_times - 1
                            time.sleep(1)
                        else:
                            break
                    except:
                        retry_times = retry_times - 1
                        time.sleep(1)

            else:
                print "ERROR! part", part , " is not as expected!"
Ejemplo n.º 4
0
def multi_upload_file2(oss, bucket, object, filename, upload_id, thread_num=10, max_part_num=10000, retry_times=5, headers=None, params=None):
    if not upload_id:
        print "empty upload_id"
        return False
    filename = convert_utf8(filename)
    part_msg_list = []
    part_msg_list = split_large_file(filename, object, max_part_num)
    queue = Queue.Queue(0)
    uploaded_part_map = {}
    part_msg_xml = create_part_xml(part_msg_list)
    each_part_retry_times = 1
    for i in range(retry_times):
        tmp_uploaded_part_map = get_part_map(oss, bucket, object, upload_id)
        if tmp_uploaded_part_map:
            for k, v in tmp_uploaded_part_map.items():
                uploaded_part_map[k] = v
        thread_pool = []
        for part in part_msg_list:
            if len(part) == 5:
                part_number = str(part[0])
                md5 = part[2]
                is_need_upload = True
                if uploaded_part_map.has_key(part_number):
                    md5 = part[2]
                    if uploaded_part_map[part_number].replace('"', "").upper() == md5.upper():
                        is_need_upload = False
                        continue
                if is_need_upload:
                    queue.put((upload_part, oss, bucket, object, upload_id, filename, part))
            else:
                print "not expected part", part
                
        for i in xrange(thread_num):
            current = UploadPartWorker2(each_part_retry_times, queue)
            thread_pool.append(current)
            current.start()
        queue.join()
        for item in thread_pool:
            item.join()

        res = oss.complete_upload(bucket, object, upload_id, part_msg_xml, headers, params)
        if res.status == 200:
            return res
    raise Exception("-3, after retry %s, failed, multi upload file failed! upload_id:%s" % (retry_times, upload_id))
Ejemplo n.º 5
0
def multi_upload_file2(oss, bucket, object, filename, upload_id, thread_num=10, max_part_num=10000, retry_times=5, headers=None, params=None):
    if not upload_id:
        print "empty upload_id"
        return False
    filename = convert_utf8(filename)
    part_msg_list = []
    part_msg_list = split_large_file(filename, object, max_part_num)
    queue = Queue.Queue(0)
    uploaded_part_map = {}
    part_msg_xml = create_part_xml(part_msg_list)
    each_part_retry_times = 1
    for i in range(retry_times):
        tmp_uploaded_part_map = get_part_map(oss, bucket, object, upload_id)
        if tmp_uploaded_part_map:
            for k, v in tmp_uploaded_part_map.items():
                uploaded_part_map[k] = v
        thread_pool = []
        for part in part_msg_list:
            if len(part) == 5:
                part_number = str(part[0])
                md5 = part[2]
                is_need_upload = True
                if uploaded_part_map.has_key(part_number):
                    md5 = part[2]
                    if uploaded_part_map[part_number].replace('"', "").upper() == md5.upper():
                        is_need_upload = False
                        continue
                if is_need_upload:
                    queue.put((upload_part, oss, bucket, object, upload_id, filename, part))
            else:
                print "not expected part", part
                
        for i in xrange(thread_num):
            current = UploadPartWorker2(each_part_retry_times, queue)
            thread_pool.append(current)
            current.start()
        queue.join()
        for item in thread_pool:
            item.join()

        res = oss.complete_upload(bucket, object, upload_id, part_msg_xml, headers, params)
        if res.status == 200:
            return res
    raise Exception("-3, after retry %s, failed, multi upload file failed! upload_id:%s" % (retry_times, upload_id))
Ejemplo n.º 6
0
    def run(self):
        for part in self.part_msg_list:
            part_number = str(part[0])
            if len(part) == 5:
                bucket = self.bucket
                object = self.object
                if self.uploaded_part_map.has_key(part_number):
                    md5 = part[2]
                    if self.uploaded_part_map[part_number].replace(
                            '"', "").upper() == md5.upper():
                        continue

                partsize = part[3]
                offset = part[4]
                retry_times = self.retry_times
                while True:
                    try:
                        if retry_times <= 0:
                            break
                        res = self.oss.upload_part_from_file_given_pos(
                            bucket, object, self.file_path, offset, partsize,
                            self.upload_id, part_number)
                        if res.status != 200:
                            log.warn(
                                "Upload %s/%s from %s, failed! ret is:%s." %
                                (bucket, object, self.file_path, res.status))
                            log.warn("headers:%s" % res.getheaders())
                            retry_times = retry_times - 1
                            time.sleep(1)
                        else:
                            log.info(
                                "Upload %s/%s from %s, OK! ret is:%s." %
                                (bucket, object, self.file_path, res.status))
                            break
                    except:
                        retry_times = retry_times - 1
                        time.sleep(1)
            else:
                log.error("ERROR! part %s is not as expected!" % part)
Ejemplo n.º 7
0
    def run(self):
        for part in self.part_msg_list:
            if len(part) == 5:
                bucket = self.bucket
                file_name = part[1]
                object_name = file_name
                res = self.oss.head_object(bucket, object_name)
                if res.status == 200:
                    header_map = convert_header2map(res.getheaders())
                    etag = safe_get_element("etag", header_map)
                    md5 = part[2] 
                    if etag.replace('"', "").upper() == md5.upper():
                        continue

                partsize = part[3]
                offset = part[4]
                res = self.oss.put_object_from_file_given_pos(bucket, object_name, self.file_path, offset, partsize)
                if res.status != 200:
                    print "upload ", file_name, "failed!"," ret is:", res.status
                    print "headers", res.getheaders()      
            else:
                print "ERROR! part", part , " is not as expected!"
Ejemplo n.º 8
0
    def run(self):
        for part in self.part_msg_list:
            part_number = str(part[0])
            if len(part) == 5:
                bucket = self.bucket
                object = self.object
                if self.uploaded_part_map.has_key(part_number):
                    md5 = part[2]
                    if self.uploaded_part_map[part_number].replace(
                            '"', "").upper() == md5.upper():
                        continue

                partsize = part[3]
                offset = part[4]
                retry_times = 5
                while True:
                    try:
                        if retry_times <= 0:
                            break
                        l.acquire()
                        res = self.oss.upload_part_from_file_given_pos(
                            bucket, object, self.file_path, offset, partsize,
                            self.upload_id, part_number)
                        l.release()
                        if res.status != 200:
                            print "upload ", bucket, object, self.file_path, "failed!", " ret is:", res.status
                            print "headers", res.getheaders()
                            retry_times = retry_times - 1
                            time.sleep(1)
                        else:
                            break
                    except:
                        retry_times = retry_times - 1
                        time.sleep(1)
            else:
                print "ERROR! part", part, " is not as expected!"
Ejemplo n.º 9
0
 def run(self):
     for part in self.part_msg_list:
         part_number = str(part[0])
         if len(part) == 5:
             bucket = self.bucket
             object = self.object
             partsize = part[3]
             offset = part[4]
             retry_times = self.retry_times
             while True:
                 try:
                     if self.uploaded_part_map.has_key(part_number):
                         md5 = part[2]
                         if self.uploaded_part_map[part_number].replace('"', "").upper() == md5.upper():
                             break
                     if retry_times <= 0:
                         break
                     res = self.oss.upload_part_from_file_given_pos(bucket, object, self.file_path, offset, partsize, self.upload_id, part_number)
                     if res.status != 200:
                         retry_times = retry_times - 1
                         time.sleep(1)
                     else:
                         etag = res.getheader("etag")
                         if etag:
                             self.uploaded_part_map[part_number] = etag
                         break
                 except:
                     retry_times = retry_times - 1
                     time.sleep(1)
         else:
             print "not expected part for multiupload", part
             pass
Ejemplo n.º 10
0
    def run(self):
        for part in self.part_msg_list:
            part_number = str(part[0])
            if len(part) == 5:
                bucket = self.bucket
                object = self.object
                if self.uploaded_part_map.has_key(part_number):
                    md5 = part[2]
                    if self.uploaded_part_map[part_number].replace('"', "").upper() == md5.upper():
                        continue

                partsize = part[3]
                offset = part[4]
                retry_times = self.retry_times
                while True:
                    try:
                        if retry_times <= 0:
                            break
                        res = self.oss.upload_part_from_file_given_pos(bucket, object, self.file_path, offset, partsize, self.upload_id, part_number)
                        if res.status != 200:
                            log.warn("Upload %s/%s from %s, failed! ret is:%s." %(bucket, object, self.file_path, res.status))
                            log.warn("headers:%s" % res.getheaders())
                            retry_times = retry_times - 1
                            time.sleep(1)
                        else:
                            log.info("Upload %s/%s from %s, OK! ret is:%s." % (bucket, object, self.file_path, res.status))
                            break
                    except:
                        retry_times = retry_times - 1
                        time.sleep(1)
            else:
                log.error("ERROR! part %s is not as expected!" % part)
Ejemplo n.º 11
0
    def run(self):
        for part in self.part_msg_list:
            part_number = str(part[0]) 
            if len(part) == 5:
                bucket = self.bucket
                object = self.object
                if self.uploaded_part_map.has_key(part_number):
                    md5 = part[2] 
                    if self.uploaded_part_map[part_number].replace('"', "").upper() == md5.upper():
                        continue

                partsize = part[3]
                offset = part[4]
                retry_times = 5
                while True:
                    try:
                        if retry_times <= 0:
                            break
                        l.acquire()
                        res = self.oss.upload_part_from_file_given_pos(bucket, object, self.file_path, offset, partsize, self.upload_id, part_number)
                        l.release()
                        if res.status != 200:
                            print "upload ", bucket, object, self.file_path, "failed!"," ret is:", res.status
                            print "headers", res.getheaders()      
                            retry_times = retry_times - 1
                            time.sleep(1)
                        else:
                            break
                    except:
                        retry_times = retry_times - 1
                        time.sleep(1)
            else:
                print "ERROR! part", part , " is not as expected!"