def upload(self, local, local_stat, path, callback_dict=None, max_upload_size=-1):
"""
Upload a file to the server.
:param local: file path
:param local_stat: stat of the file
:param path: target path on the server
:param callback_dict: an dict that can be fed with progress data
:param max_upload_size: a known or arbitrary upload max size. If the file file is bigger, it will be
chunked into many POST requests
:return: Server response
"""
if not local_stat:
raise PydioSdkException('upload', path, _('Local file to upload not found!'))
if local_stat['size'] == 0:
self.mkfile(path)
new = self.stat(path)
if not new or not (new['size'] == local_stat['size']):
raise PydioSdkException('upload', path, _('File not correct after upload (expected size was 0 bytes)'))
return True
existing_part = False
if (self.upload_max_size - 4096) < local_stat['size']:
self.has_disk_space_for_upload(path, local_stat['size'])
existing_part = self.stat(path+'.dlpart', True)
dirpath = os.path.dirname(path)
if dirpath and dirpath != '/':
folder = self.stat(dirpath)
if not folder:
self.mkdir(os.path.dirname(path))
url = self.url + '/upload/put' + self.urlencode_normalized((self.remote_folder + os.path.dirname(path)))
files = {
'userfile_0': local
}
if existing_part:
files['existing_dlpart'] = existing_part
data = {
'force_post': 'true',
'xhr_uploader': 'true',
'urlencoded_filename': self.urlencode_normalized(os.path.basename(path))
}
try:
self.perform_request(url=url, type='post', data=data, files=files, with_progress=callback_dict)
except PydioSdkDefaultException as e:
if e.message == '507':
usage, total = self.quota_usage()
raise PydioSdkQuotaException(path, local_stat['size'], usage, total)
if e.message == '412':
raise PydioSdkPermissionException('Cannot upload '+os.path.basename(path)+' in directory '+os.path.dirname(path))
else:
raise e
except RequestException as ce:
raise PydioSdkException("upload", path, 'RequestException: ' + ce.message)
new = self.stat(path)
if not new or not (new['size'] == local_stat['size']):
raise PydioSdkException('upload', path, _('File is incorrect after upload'))
return True
def stat(self, path, with_hash=False, partial_hash=None):
"""
Equivalent of the local fstat() on the remote server.
:param path: path of node from the workspace root
:param with_hash: stat result can be enriched with the node hash
:return:dict a list of key like
{
dev: 16777218,
ino: 4062280,
mode: 16895,
nlink: 15,
uid: 70,
gid: 20,
rdev: 0,
size: 510,
atime: 1401915891,
mtime: 1399883020,
ctime: 1399883020,
blksize: 4096,
blocks: 0
}
"""
if self.interrupt_tasks:
raise PydioSdkException("stat", path=path, detail=_('Task interrupted by user'))
path = self.remote_folder + path
action = '/stat_hash' if with_hash else '/stat'
try:
url = self.url + action + self.urlencode_normalized(path)
if partial_hash:
h = {'range': 'bytes=%i-%i' % (partial_hash[0], partial_hash[1])}
resp = self.perform_request(url, headers=h)
else:
resp = self.perform_request(url)
try:
data = json.loads(resp.content)
except ValueError as ve:
return False
logging.debug("data: %s" % data)
if not data:
return False
if len(data) > 0 and 'size' in data:
return data
else:
return False
except requests.exceptions.ConnectionError:
raise
except requests.exceptions.Timeout:
raise
except Exception, ex:
logging.warning("Stat failed", exc_info=ex)
return False
def basic_authenticate(self):
"""
Use basic-http authenticate to get a key/pair token instead of passing the
users credentials at each requests
:return:dict()
"""
url = self.base_url + 'pydio/keystore_generate_auth_token/' + self.device_id
resp = requests.get(url=url, auth=self.auth, verify=self.verify_ssl, proxies=self.proxies)
if resp.status_code == 401:
raise PydioSdkBasicAuthException(_('Authentication Error'))
# If content is empty (but not error status code), the token based auth may not be active
# We should switch to basic
if resp.content == '':
raise PydioSdkTokenAuthNotSupportedException("token_auth")
try:
tokens = json.loads(resp.content)
except ValueError as v:
raise PydioSdkException("basic_auth", "", "Cannot parse JSON result: " + resp.content + "")
#return False
self.set_tokens(tokens)
return tokens
def upload_file_with_progress(self, url, fields, files, stream, with_progress, max_size=0):
"""
Upload a file with progress, file chunking if necessary, and stream content directly from file.
:param url: url to post
:param fields: dict() query parameters
:param files: dict() {'fieldname' : '/path/to/file'}
:param stream: whether to get response as stream or not
:param with_progress: dict() updatable dict with progress data
:param max_size: upload max size
:return: response of the last requests if there were many of them
"""
if with_progress:
def cb(size=0, progress=0, delta=0, rate=0):
with_progress['total_size'] = size
with_progress['bytes_sent'] = delta
with_progress['total_bytes_sent'] = progress
dispatcher.send(signal=TRANSFER_CALLBACK_SIGNAL, change=with_progress)
else:
def cb(size=0, progress=0, delta=0, rate=0):
logging.debug('Current transfer rate ' + str(rate))
def parse_upload_rep(http_response):
if http_response.headers.get('content-type') != 'application/octet-stream':
if unicode(http_response.text).count('message type="ERROR"'):
if unicode(http_response.text).lower().count("(507)"):
raise PydioSdkDefaultException('507')
if unicode(http_response.text).lower().count("(412)"):
raise PydioSdkDefaultException('412')
import re
# Remove XML tags
text = re.sub('<[^<]+>', '', unicode(http_response.text))
raise PydioSdkDefaultException(text)
if unicode(http_response.text).lower().count("(507)"):
raise PydioSdkDefaultException('507')
if unicode(http_response.text).lower().count("(412)"):
raise PydioSdkDefaultException('412')
if unicode(http_response.text).lower().count("(410)") or unicode(http_response.text).lower().count("(411)"):
raise PydioSdkDefaultException(unicode(http_response.text))
filesize = os.stat(files['userfile_0']).st_size
if max_size:
# Reduce max size to leave some room for data header
max_size -= 4096
existing_pieces_number = 0
if max_size and filesize > max_size:
fields['partial_upload'] = 'true'
fields['partial_target_bytesize'] = str(filesize)
# Check if there is already a .dlpart on the server.
# If it's the case, maybe it's already the beginning of this?
if 'existing_dlpart' in files:
existing_dlpart = files['existing_dlpart']
existing_dlpart_size = existing_dlpart['size']
if filesize > existing_dlpart_size and \
file_start_hash_match(files['userfile_0'], existing_dlpart_size, existing_dlpart['hash']):
logging.info('Found the beggining of this file on the other file, skipping the first pieces')
existing_pieces_number = existing_dlpart_size / max_size
cb(filesize, existing_dlpart_size, existing_dlpart_size, 0)
if not existing_pieces_number:
(header_body, close_body, content_type) = encode_multiparts(fields)
body = BytesIOWithFile(header_body, close_body, files['userfile_0'], callback=cb, chunk_size=max_size, file_part=0)
resp = requests.post(
url,
data=body,
headers={'Content-Type': content_type},
stream=True,
timeout=20,
verify=self.verify_ssl
)
existing_pieces_number = 1
parse_upload_rep(resp)
if resp.status_code == 401:
return resp
if max_size and filesize > max_size:
fields['appendto_urlencoded_part'] = fields['urlencoded_filename']
del fields['urlencoded_filename']
(header_body, close_body, content_type) = encode_multiparts(fields)
for i in range(existing_pieces_number, int(math.ceil(filesize / max_size)) + 1):
if self.interrupt_tasks:
raise PydioSdkException("upload", path=os.path.basename(files['userfile_0']), detail=_('Task interrupted by user'))
before = time.time()
body = BytesIOWithFile(header_body, close_body, files['userfile_0'],
callback=cb, chunk_size=max_size, file_part=i)
resp = requests.post(
url,
data=body,
headers={'Content-Type': content_type},
stream=True,
verify=self.verify_ssl
)
parse_upload_rep(resp)
if resp.status_code == 401:
return resp
duration = time.time() - before
logging.info('Uploaded '+str(max_size)+' bytes of data in about %'+str(duration)+' s')
return resp
def download(self, path, local, callback_dict=None):
"""
Download the content of a server file to a local file.
:param path: node path on the server
:param local: local path on filesystem
:param callback_dict: a dict() than can be updated by with progress data
:return: Server response
"""
orig = self.stat(path)
if not orig:
raise PydioSdkException('download', path, _('Original file was not found on server'))
url = self.url + '/download' + self.urlencode_normalized((self.remote_folder + path))
local_tmp = local + '.pydio_dl'
headers = None
write_mode = 'wb'
dl = 0
if not os.path.exists(os.path.dirname(local)):
os.makedirs(os.path.dirname(local))
elif os.path.exists(local_tmp):
# A .pydio_dl already exists, maybe it's a chunk of the original?
# Try to get an md5 of the corresponding chunk
current_size = os.path.getsize(local_tmp)
chunk_local_hash = hashfile(open(local_tmp, 'rb'), hashlib.md5())
chunk_remote_stat = self.stat(path, True, partial_hash=[0, current_size])
if chunk_local_hash == chunk_remote_stat['hash']:
headers = {'range':'bytes=%i-%i' % (current_size, chunk_remote_stat['size'])}
write_mode = 'a+'
dl = current_size
if callback_dict:
callback_dict['bytes_sent'] = float(current_size)
callback_dict['total_bytes_sent'] = float(current_size)
callback_dict['total_size'] = float(chunk_remote_stat['size'])
callback_dict['transfer_rate'] = 0
dispatcher.send(signal=TRANSFER_CALLBACK_SIGNAL, send=self, change=callback_dict)
else:
os.unlink(local_tmp)
try:
with open(local_tmp, write_mode) as fd:
start = time.clock()
r = self.perform_request(url=url, stream=True, headers=headers)
total_length = r.headers.get('content-length')
if total_length is None: # no content length header
fd.write(r.content)
else:
previous_done = 0
for chunk in r.iter_content(1024 * 8):
if self.interrupt_tasks:
raise PydioSdkException("interrupt", path=path, detail=_('Task interrupted by user'))
dl += len(chunk)
fd.write(chunk)
done = int(50 * dl / int(total_length))
if done != previous_done:
transfer_rate = dl // (time.clock() - start)
logging.debug("\r[%s%s] %s bps" % ('=' * done, ' ' * (50 - done), transfer_rate))
dispatcher.send(signal=TRANSFER_RATE_SIGNAL, send=self, transfer_rate=transfer_rate)
if callback_dict:
callback_dict['bytes_sent'] = float(len(chunk))
callback_dict['total_bytes_sent'] = float(dl)
callback_dict['total_size'] = float(total_length)
callback_dict['transfer_rate'] = transfer_rate
dispatcher.send(signal=TRANSFER_CALLBACK_SIGNAL, send=self, change=callback_dict)
previous_done = done
if not os.path.exists(local_tmp):
raise PydioSdkException('download', local, _('File not found after download'))
else:
stat_result = os.stat(local_tmp)
if not orig['size'] == stat_result.st_size:
os.unlink(local_tmp)
raise PydioSdkException('download', path, _('File is not correct after download'))
else:
is_system_windows = platform.system().lower().startswith('win')
if is_system_windows and os.path.exists(local):
os.unlink(local)
os.rename(local_tmp, local)
return True
except PydioSdkException as pe:
if pe.operation == 'interrupt':
raise pe
else:
if os.path.exists(local_tmp):
os.unlink(local_tmp)
raise pe
except Exception as e:
if os.path.exists(local_tmp):
os.unlink(local_tmp)
raise PydioSdkException('download', path, _('Error while downloading file: %s') % e.message)
def bulk_stat(self, pathes, result=None, with_hash=False):
"""
Perform a stat operation (see self.stat()) but on a set of nodes. Very important to use that method instead
of sending tons of small stat requests to server. To keep POST content reasonable, pathes will be sent 200 by
200.
:param pathes: list() of node pathes
:param result: dict() an accumulator for the results
:param with_hash: bool whether to ask for files hash or not (md5)
:return:
"""
# NORMALIZE PATHES FROM START
pathes = map(lambda p: self.normalize(p), pathes)
action = '/stat_hash' if with_hash else '/stat'
data = dict()
maxlen = min(len(pathes), 200)
clean_pathes = map(lambda t: self.remote_folder + t.replace('\\', '/'),
filter(lambda x: x != '', pathes[:maxlen]))
data['nodes[]'] = map(lambda p: self.normalize(p), clean_pathes)
url = self.url + action + self.urlencode_normalized(clean_pathes[0])
resp = self.perform_request(url, type='post', data=data)
try:
data = json.loads(resp.content)
except ValueError:
logging.debug("url: %s" % url)
logging.debug("resp.content: %s" % resp.content)
raise
if len(pathes) == 1:
englob = dict()
englob[self.remote_folder + pathes[0]] = data
data = englob
if result:
replaced = result
else:
replaced = dict()
for (p, stat) in data.items():
if self.remote_folder:
p = p[len(self.remote_folder):]
#replaced[os.path.normpath(p)] = stat
p1 = os.path.normpath(p)
p2 = os.path.normpath(self.normalize_reverse(p))
p3 = p
p4 = self.normalize_reverse(p)
if p2 in pathes:
replaced[p2] = stat
pathes.remove(p2)
elif p1 in pathes:
replaced[p1] = stat
pathes.remove(p1)
elif p3 in pathes:
replaced[p3] = stat
pathes.remove(p3)
elif p4 in pathes:
replaced[p4] = stat
pathes.remove(p4)
else:
#pass
logging.info('Fatal charset error, cannot find files (%s, %s, %s, %s) in %s' % (repr(p1), repr(p2), repr(p3), repr(p4), repr(pathes),))
raise PydioSdkException('bulk_stat', p1, "Encoding problem, failed emptying bulk_stat, "
"exiting to avoid infinite loop")
if len(pathes):
self.bulk_stat(pathes, result=replaced, with_hash=with_hash)
return replaced