def continue_upload(token, upload_key, fp, remote_filename, filesize, part_number, split_size=DEFAULT_SPLITSIZE, dir_id=0, md5sum=None): # get the md5 of the entire file (ignore if calculated) if not isinstance(fp, RangeFile): raise Exception, 'fp is not RangeFile' if not md5sum: fp.limitrange(0, filesize - 1) md5sum = filehash(fp) rpc = vdiskrpc() timedata = None md5list = {} ranges = rangesplit(filesize, split_size) parts = len(ranges) for i in range(parts): save_resumedata(token, upload_key, i, md5sum) range_left, range_right = ranges[i][0], ranges[i][1] fp.limitrange(range_left, range_right) current_part = i+1 if i < part_number: # already finished, just add its md5 to the list fp.seek(0) md5sum_part = filehash(fp) md5list[current_part] = md5sum_part continue # now begin to upload for rpc_tries in range(RPC_RETRIES): uripart = rpc.big_file_upload_part(token=token, upload_key=upload_key, part_number=current_part, field='URI') if not isinstance(uripart, dict): break if isinstance(uripart, dict): return {'errcode': 9999, 'err_msg': 'upload RPC failed, please retry'} # time tick here timedata = timetick(timedata, range_left - 1) curtime_str = time.strftime("[%H:%M:%S]") print '%s Uploading Part %d / %d (%.2f %%), Speed %s' % (curtime_str, current_part, parts, float(range_left) / filesize * 100.0, speed_humanreadable(getspeed(timedata))) uri = 'http://' + VDISK_S3HOST + uripart for upload_tries in range(UPLOAD_RETRIES): resultmd5 = None try: resultmd5 = vdisk_uploads3_put(fp, range_left, range_right, uri) # headers? if resultmd5: break except KeyboardInterrupt: print 'user cancelled' raise except Exception, e: print str(e) print 'part upload temporarily failed ... (tried: %d)' % (upload_tries + 1) if not resultmd5: return {'errcode': 9999, 'err_msg': 'part upload failed, please retry'} # time tick here timedata = timetick(timedata, range_right) # part upload ok :) md5list[current_part] = resultmd5
def vdisk_mkdir(token, path): if not path: return {'errcode': 900, 'err_msg': 'bad path'} # should it be 0? path = path.rstrip('/') if not path: path = '/' if path[0] != '/': path = '/' + path id = vdisk_dirid(token, path) # find directly if not isinstance(id, dict): return id # FIXME: errno handling? # not found parentdir = os.path.dirname(path) basename = os.path.basename(path) id = vdisk_mkdir(token, parentdir) if isinstance(id, dict): return {'errcode': 900, 'err_msg': 'create failed'} rpc = vdiskrpc() return rpc.create_dir(token=token, parent_id=id, create_name=basename, field='dir_id')
def vdisk_ls_dirid(token, dir_id, traverse=False): rpc = vdiskrpc() l = rpc.get_list(token=token, dir_id=dir_id) # FIXME: page q = [] if isinstance(l.get('data', None), list): l = l.get('data', []) for i in l: try: if 'sha1' in i: print i['sha1'], int(i['id']), int(i['length']), to_console(i['name']) else: q.append(int(i['id'])) print 'DIR', int(i['id']), 0, to_console(i['name']) except: pass if traverse: for i in q: print 'SUB_DIR', i vdisk_ls_dirid(token, i, True)
def vdisk_ls_dirid(token, dir_id, traverse=False): rpc = vdiskrpc() l = rpc.get_list(token=token, dir_id=dir_id) # FIXME: page q = [] if isinstance(l.get('data', None), list): l = l.get('data', []) for i in l: try: if 'sha1' in i: print i['sha1'], int(i['id']), int( i['length']), to_console(i['name']) else: q.append(int(i['id'])) print 'DIR', int(i['id']), 0, to_console(i['name']) except: pass if traverse: for i in q: print 'SUB_DIR', i vdisk_ls_dirid(token, i, True)
def get_token(user, passwd, vdisk_user=False): rpc = vdiskrpc() return rpc.get_token(user, passwd, vdisk_user=vdisk_user).get('data', {}).get('token', None)
def continue_upload(token, upload_key, fp, remote_filename, filesize, part_number, split_size=DEFAULT_SPLITSIZE, dir_id=0, md5sum=None): # get the md5 of the entire file (ignore if calculated) if not isinstance(fp, RangeFile): raise Exception, 'fp is not RangeFile' if not md5sum: fp.limitrange(0, filesize - 1) md5sum = filehash(fp) rpc = vdiskrpc() timedata = None md5list = {} ranges = rangesplit(filesize, split_size) parts = len(ranges) for i in range(parts): save_resumedata(token, upload_key, i, md5sum) range_left, range_right = ranges[i][0], ranges[i][1] fp.limitrange(range_left, range_right) current_part = i + 1 if i < part_number: # already finished, just add its md5 to the list fp.seek(0) md5sum_part = filehash(fp) md5list[current_part] = md5sum_part continue # now begin to upload for rpc_tries in range(RPC_RETRIES): uripart = rpc.big_file_upload_part(token=token, upload_key=upload_key, part_number=current_part, field='URI') if not isinstance(uripart, dict): break if isinstance(uripart, dict): return { 'errcode': 9999, 'err_msg': 'upload RPC failed, please retry' } # time tick here timedata = timetick(timedata, range_left - 1) curtime_str = time.strftime("[%H:%M:%S]") print '%s Uploading Part %d / %d (%.2f %%), Speed %s' % ( curtime_str, current_part, parts, float(range_left) / filesize * 100.0, speed_humanreadable(getspeed(timedata))) uri = 'http://' + VDISK_S3HOST + uripart for upload_tries in range(UPLOAD_RETRIES): resultmd5 = None try: resultmd5 = vdisk_uploads3_put(fp, range_left, range_right, uri) # headers? if resultmd5: break except KeyboardInterrupt: print 'user cancelled' raise except Exception, e: print str(e) print 'part upload temporarily failed ... (tried: %d)' % ( upload_tries + 1) if not resultmd5: return { 'errcode': 9999, 'err_msg': 'part upload failed, please retry' } # time tick here timedata = timetick(timedata, range_right) # part upload ok :) md5list[current_part] = resultmd5
def vdisk_dirid(token, path='/'): if path[0] != '/': path = '/' + path if path in ['/', '']: return 0 # FIXME: error handling rpc = vdiskrpc() return rpc.get_dirid_with_path(token=token, path=path, field='id')
print 'found dir_id for path %s: %s' % (path, dir_id) remote_filename = get_remote_filename(filename, remote_filename) try: # get filesize, 0 & return filesize = os.path.getsize(filename) if filesize == 0: return {'errcode': 9999, 'err_msg': 'filesize is 0'} # open the file, calculate md5 / sha1 fp = RangeFile(filename, 'rb') md5sum, sha1sum = filemd5sha1(fp) except OSError, e: return {'errcode': 9999, 'err_msg': 'file error: ' + str(e)} print 'File: %s\nMD5 : %s\nSHA1: %s\nSize: %d' % ( to_console(remote_filename), md5sum, sha1sum, filesize) rpc = vdiskrpc() # upload by sha1. for rpc_tries in range(RPC_RETRIES): match_result = rpc.upload_with_sha1(token=token, sha1=sha1sum, dir_id=dir_id, file_name=remote_filename, field='fid') if not isinstance(match_result, dict): # successed! print 'Matched by SHA1, fid: ', match_result return {'errcode': 0, 'fid': match_result} elif match_result.get('errcode') == 1: # Not found break
def get_token(user, passwd, vdisk_user=False): rpc = vdiskrpc() return rpc.get_token(user, passwd, vdisk_user=vdisk_user).get('data',{}).get('token', None)
dir_id = dir_id_new print 'found dir_id for path %s: %s' % (path, dir_id) remote_filename = get_remote_filename(filename, remote_filename) try: # get filesize, 0 & return filesize = os.path.getsize(filename) if filesize == 0: return {'errcode': 9999, 'err_msg': 'filesize is 0'} # open the file, calculate md5 / sha1 fp = RangeFile(filename, 'rb') md5sum, sha1sum = filemd5sha1(fp) except OSError, e: return {'errcode': 9999, 'err_msg': 'file error: ' + str(e)} print 'File: %s\nMD5 : %s\nSHA1: %s\nSize: %d' % (to_console(remote_filename), md5sum, sha1sum, filesize) rpc = vdiskrpc() # upload by sha1. for rpc_tries in range(RPC_RETRIES): match_result = rpc.upload_with_sha1(token=token, sha1=sha1sum, dir_id=dir_id, file_name=remote_filename, field='fid') if not isinstance(match_result, dict): # successed! print 'Matched by SHA1, fid: ', match_result return {'errcode': 0, 'fid': match_result} elif match_result.get('errcode') == 1: # Not found break # TODO: read the resume data by md5sum, remove upload_resume function resumedata = load_resumedata(md5sum) # (token, upload_key, current_part) if resumedata: