def continue_upload(token, upload_key, fp, remote_filename, filesize, part_number, split_size=DEFAULT_SPLITSIZE, dir_id=0, md5sum=None):
# get the md5 of the entire file (ignore if calculated)
if not isinstance(fp, RangeFile): raise Exception, 'fp is not RangeFile'
if not md5sum:
fp.limitrange(0, filesize - 1)
md5sum = filehash(fp)
rpc = vdiskrpc()
timedata = None
md5list = {}
ranges = rangesplit(filesize, split_size)
parts = len(ranges)
for i in range(parts):
save_resumedata(token, upload_key, i, md5sum)
range_left, range_right = ranges[i][0], ranges[i][1]
fp.limitrange(range_left, range_right)
current_part = i+1
if i < part_number: # already finished, just add its md5 to the list
fp.seek(0)
md5sum_part = filehash(fp)
md5list[current_part] = md5sum_part
continue
# now begin to upload
for rpc_tries in range(RPC_RETRIES):
uripart = rpc.big_file_upload_part(token=token, upload_key=upload_key, part_number=current_part, field='URI')
if not isinstance(uripart, dict): break
if isinstance(uripart, dict):
return {'errcode': 9999, 'err_msg': 'upload RPC failed, please retry'}
# time tick here
timedata = timetick(timedata, range_left - 1)
curtime_str = time.strftime("[%H:%M:%S]")
print '%s Uploading Part %d / %d (%.2f %%), Speed %s' % (curtime_str, current_part, parts, float(range_left) / filesize * 100.0, speed_humanreadable(getspeed(timedata)))
uri = 'http://' + VDISK_S3HOST + uripart
for upload_tries in range(UPLOAD_RETRIES):
resultmd5 = None
try:
resultmd5 = vdisk_uploads3_put(fp, range_left, range_right, uri) # headers?
if resultmd5: break
except KeyboardInterrupt:
print 'user cancelled'
raise
except Exception, e:
print str(e)
print 'part upload temporarily failed ... (tried: %d)' % (upload_tries + 1)
if not resultmd5:
return {'errcode': 9999, 'err_msg': 'part upload failed, please retry'}
# time tick here
timedata = timetick(timedata, range_right)
# part upload ok :)
md5list[current_part] = resultmd5
def vdisk_mkdir(token, path):
if not path: return {'errcode': 900, 'err_msg': 'bad path'} # should it be 0?
path = path.rstrip('/')
if not path: path = '/'
if path[0] != '/': path = '/' + path
id = vdisk_dirid(token, path) # find directly
if not isinstance(id, dict): return id # FIXME: errno handling?
# not found
parentdir = os.path.dirname(path)
basename = os.path.basename(path)
id = vdisk_mkdir(token, parentdir)
if isinstance(id, dict):
return {'errcode': 900, 'err_msg': 'create failed'}
rpc = vdiskrpc()
return rpc.create_dir(token=token, parent_id=id, create_name=basename, field='dir_id')
def vdisk_ls_dirid(token, dir_id, traverse=False):
rpc = vdiskrpc()
l = rpc.get_list(token=token, dir_id=dir_id) # FIXME: page
q = []
if isinstance(l.get('data', None), list):
l = l.get('data', [])
for i in l:
try:
if 'sha1' in i:
print i['sha1'], int(i['id']), int(i['length']), to_console(i['name'])
else:
q.append(int(i['id']))
print 'DIR', int(i['id']), 0, to_console(i['name'])
except: pass
if traverse:
for i in q:
print 'SUB_DIR', i
vdisk_ls_dirid(token, i, True)
def vdisk_mkdir(token, path):
if not path:
return {'errcode': 900, 'err_msg': 'bad path'} # should it be 0?
path = path.rstrip('/')
if not path: path = '/'
if path[0] != '/': path = '/' + path
id = vdisk_dirid(token, path) # find directly
if not isinstance(id, dict): return id # FIXME: errno handling?
# not found
parentdir = os.path.dirname(path)
basename = os.path.basename(path)
id = vdisk_mkdir(token, parentdir)
if isinstance(id, dict):
return {'errcode': 900, 'err_msg': 'create failed'}
rpc = vdiskrpc()
return rpc.create_dir(token=token,
parent_id=id,
create_name=basename,
field='dir_id')
def vdisk_ls_dirid(token, dir_id, traverse=False):
rpc = vdiskrpc()
l = rpc.get_list(token=token, dir_id=dir_id) # FIXME: page
q = []
if isinstance(l.get('data', None), list):
l = l.get('data', [])
for i in l:
try:
if 'sha1' in i:
print i['sha1'], int(i['id']), int(
i['length']), to_console(i['name'])
else:
q.append(int(i['id']))
print 'DIR', int(i['id']), 0, to_console(i['name'])
except:
pass
if traverse:
for i in q:
print 'SUB_DIR', i
vdisk_ls_dirid(token, i, True)
def get_token(user, passwd, vdisk_user=False):
rpc = vdiskrpc()
return rpc.get_token(user, passwd,
vdisk_user=vdisk_user).get('data',
{}).get('token', None)
def continue_upload(token,
upload_key,
fp,
remote_filename,
filesize,
part_number,
split_size=DEFAULT_SPLITSIZE,
dir_id=0,
md5sum=None):
# get the md5 of the entire file (ignore if calculated)
if not isinstance(fp, RangeFile): raise Exception, 'fp is not RangeFile'
if not md5sum:
fp.limitrange(0, filesize - 1)
md5sum = filehash(fp)
rpc = vdiskrpc()
timedata = None
md5list = {}
ranges = rangesplit(filesize, split_size)
parts = len(ranges)
for i in range(parts):
save_resumedata(token, upload_key, i, md5sum)
range_left, range_right = ranges[i][0], ranges[i][1]
fp.limitrange(range_left, range_right)
current_part = i + 1
if i < part_number: # already finished, just add its md5 to the list
fp.seek(0)
md5sum_part = filehash(fp)
md5list[current_part] = md5sum_part
continue
# now begin to upload
for rpc_tries in range(RPC_RETRIES):
uripart = rpc.big_file_upload_part(token=token,
upload_key=upload_key,
part_number=current_part,
field='URI')
if not isinstance(uripart, dict): break
if isinstance(uripart, dict):
return {
'errcode': 9999,
'err_msg': 'upload RPC failed, please retry'
}
# time tick here
timedata = timetick(timedata, range_left - 1)
curtime_str = time.strftime("[%H:%M:%S]")
print '%s Uploading Part %d / %d (%.2f %%), Speed %s' % (
curtime_str, current_part, parts, float(range_left) / filesize *
100.0, speed_humanreadable(getspeed(timedata)))
uri = 'http://' + VDISK_S3HOST + uripart
for upload_tries in range(UPLOAD_RETRIES):
resultmd5 = None
try:
resultmd5 = vdisk_uploads3_put(fp, range_left, range_right,
uri) # headers?
if resultmd5: break
except KeyboardInterrupt:
print 'user cancelled'
raise
except Exception, e:
print str(e)
print 'part upload temporarily failed ... (tried: %d)' % (
upload_tries + 1)
if not resultmd5:
return {
'errcode': 9999,
'err_msg': 'part upload failed, please retry'
}
# time tick here
timedata = timetick(timedata, range_right)
# part upload ok :)
md5list[current_part] = resultmd5
def vdisk_dirid(token, path='/'):
if path[0] != '/': path = '/' + path
if path in ['/', '']: return 0 # FIXME: error handling
rpc = vdiskrpc()
return rpc.get_dirid_with_path(token=token, path=path, field='id')
print 'found dir_id for path %s: %s' % (path, dir_id)
remote_filename = get_remote_filename(filename, remote_filename)
try:
# get filesize, 0 & return
filesize = os.path.getsize(filename)
if filesize == 0: return {'errcode': 9999, 'err_msg': 'filesize is 0'}
# open the file, calculate md5 / sha1
fp = RangeFile(filename, 'rb')
md5sum, sha1sum = filemd5sha1(fp)
except OSError, e:
return {'errcode': 9999, 'err_msg': 'file error: ' + str(e)}
print 'File: %s\nMD5 : %s\nSHA1: %s\nSize: %d' % (
to_console(remote_filename), md5sum, sha1sum, filesize)
rpc = vdiskrpc()
# upload by sha1.
for rpc_tries in range(RPC_RETRIES):
match_result = rpc.upload_with_sha1(token=token,
sha1=sha1sum,
dir_id=dir_id,
file_name=remote_filename,
field='fid')
if not isinstance(match_result, dict):
# successed!
print 'Matched by SHA1, fid: ', match_result
return {'errcode': 0, 'fid': match_result}
elif match_result.get('errcode') == 1: # Not found
break
def get_token(user, passwd, vdisk_user=False):
rpc = vdiskrpc()
return rpc.get_token(user, passwd, vdisk_user=vdisk_user).get('data',{}).get('token', None)
def vdisk_dirid(token, path='/'): if path[0] != '/': path = '/' + path if path in ['/', '']: return 0 # FIXME: error handling rpc = vdiskrpc() return rpc.get_dirid_with_path(token=token, path=path, field='id')
dir_id = dir_id_new
print 'found dir_id for path %s: %s' % (path, dir_id)
remote_filename = get_remote_filename(filename, remote_filename)
try:
# get filesize, 0 & return
filesize = os.path.getsize(filename)
if filesize == 0: return {'errcode': 9999, 'err_msg': 'filesize is 0'}
# open the file, calculate md5 / sha1
fp = RangeFile(filename, 'rb')
md5sum, sha1sum = filemd5sha1(fp)
except OSError, e:
return {'errcode': 9999, 'err_msg': 'file error: ' + str(e)}
print 'File: %s\nMD5 : %s\nSHA1: %s\nSize: %d' % (to_console(remote_filename), md5sum, sha1sum, filesize)
rpc = vdiskrpc()
# upload by sha1.
for rpc_tries in range(RPC_RETRIES):
match_result = rpc.upload_with_sha1(token=token, sha1=sha1sum, dir_id=dir_id, file_name=remote_filename, field='fid')
if not isinstance(match_result, dict):
# successed!
print 'Matched by SHA1, fid: ', match_result
return {'errcode': 0, 'fid': match_result}
elif match_result.get('errcode') == 1: # Not found
break
# TODO: read the resume data by md5sum, remove upload_resume function
resumedata = load_resumedata(md5sum)
# (token, upload_key, current_part)
if resumedata:
