def upload(self, local, local_stat, path, callback_dict=None, max_upload_size=-1): """ Upload a file to the server. :param local: file path :param local_stat: stat of the file :param path: target path on the server :param callback_dict: an dict that can be fed with progress data :param max_upload_size: a known or arbitrary upload max size. If the file file is bigger, it will be chunked into many POST requests :return: Server response """ if not local_stat: raise PydioSdkException('upload', path, _('Local file to upload not found!')) if local_stat['size'] == 0: self.mkfile(path) new = self.stat(path) if not new or not (new['size'] == local_stat['size']): raise PydioSdkException('upload', path, _('File not correct after upload (expected size was 0 bytes)')) return True existing_part = False if (self.upload_max_size - 4096) < local_stat['size']: self.has_disk_space_for_upload(path, local_stat['size']) existing_part = self.stat(path+'.dlpart', True) dirpath = os.path.dirname(path) if dirpath and dirpath != '/': folder = self.stat(dirpath) if not folder: self.mkdir(os.path.dirname(path)) url = self.url + '/upload/put' + self.urlencode_normalized((self.remote_folder + os.path.dirname(path))) files = { 'userfile_0': local } if existing_part: files['existing_dlpart'] = existing_part data = { 'force_post': 'true', 'xhr_uploader': 'true', 'urlencoded_filename': self.urlencode_normalized(os.path.basename(path)) } try: self.perform_request(url=url, type='post', data=data, files=files, with_progress=callback_dict) except PydioSdkDefaultException as e: if e.message == '507': usage, total = self.quota_usage() raise PydioSdkQuotaException(path, local_stat['size'], usage, total) if e.message == '412': raise PydioSdkPermissionException('Cannot upload '+os.path.basename(path)+' in directory '+os.path.dirname(path)) else: raise e except RequestException as ce: raise PydioSdkException("upload", path, 'RequestException: ' + ce.message) new = self.stat(path) if not new or not (new['size'] == local_stat['size']): raise PydioSdkException('upload', path, _('File is incorrect after upload')) return True
def stat(self, path, with_hash=False, partial_hash=None): """ Equivalent of the local fstat() on the remote server. :param path: path of node from the workspace root :param with_hash: stat result can be enriched with the node hash :return:dict a list of key like { dev: 16777218, ino: 4062280, mode: 16895, nlink: 15, uid: 70, gid: 20, rdev: 0, size: 510, atime: 1401915891, mtime: 1399883020, ctime: 1399883020, blksize: 4096, blocks: 0 } """ if self.interrupt_tasks: raise PydioSdkException("stat", path=path, detail=_('Task interrupted by user')) path = self.remote_folder + path action = '/stat_hash' if with_hash else '/stat' try: url = self.url + action + self.urlencode_normalized(path) if partial_hash: h = {'range': 'bytes=%i-%i' % (partial_hash[0], partial_hash[1])} resp = self.perform_request(url, headers=h) else: resp = self.perform_request(url) try: data = json.loads(resp.content) except ValueError as ve: return False logging.debug("data: %s" % data) if not data: return False if len(data) > 0 and 'size' in data: return data else: return False except requests.exceptions.ConnectionError: raise except requests.exceptions.Timeout: raise except Exception, ex: logging.warning("Stat failed", exc_info=ex) return False
def basic_authenticate(self): """ Use basic-http authenticate to get a key/pair token instead of passing the users credentials at each requests :return:dict() """ url = self.base_url + 'pydio/keystore_generate_auth_token/' + self.device_id resp = requests.get(url=url, auth=self.auth, verify=self.verify_ssl, proxies=self.proxies) if resp.status_code == 401: raise PydioSdkBasicAuthException(_('Authentication Error')) # If content is empty (but not error status code), the token based auth may not be active # We should switch to basic if resp.content == '': raise PydioSdkTokenAuthNotSupportedException("token_auth") try: tokens = json.loads(resp.content) except ValueError as v: raise PydioSdkException("basic_auth", "", "Cannot parse JSON result: " + resp.content + "") #return False self.set_tokens(tokens) return tokens
def upload_file_with_progress(self, url, fields, files, stream, with_progress, max_size=0): """ Upload a file with progress, file chunking if necessary, and stream content directly from file. :param url: url to post :param fields: dict() query parameters :param files: dict() {'fieldname' : '/path/to/file'} :param stream: whether to get response as stream or not :param with_progress: dict() updatable dict with progress data :param max_size: upload max size :return: response of the last requests if there were many of them """ if with_progress: def cb(size=0, progress=0, delta=0, rate=0): with_progress['total_size'] = size with_progress['bytes_sent'] = delta with_progress['total_bytes_sent'] = progress dispatcher.send(signal=TRANSFER_CALLBACK_SIGNAL, change=with_progress) else: def cb(size=0, progress=0, delta=0, rate=0): logging.debug('Current transfer rate ' + str(rate)) def parse_upload_rep(http_response): if http_response.headers.get('content-type') != 'application/octet-stream': if unicode(http_response.text).count('message type="ERROR"'): if unicode(http_response.text).lower().count("(507)"): raise PydioSdkDefaultException('507') if unicode(http_response.text).lower().count("(412)"): raise PydioSdkDefaultException('412') import re # Remove XML tags text = re.sub('<[^<]+>', '', unicode(http_response.text)) raise PydioSdkDefaultException(text) if unicode(http_response.text).lower().count("(507)"): raise PydioSdkDefaultException('507') if unicode(http_response.text).lower().count("(412)"): raise PydioSdkDefaultException('412') if unicode(http_response.text).lower().count("(410)") or unicode(http_response.text).lower().count("(411)"): raise PydioSdkDefaultException(unicode(http_response.text)) filesize = os.stat(files['userfile_0']).st_size if max_size: # Reduce max size to leave some room for data header max_size -= 4096 existing_pieces_number = 0 if max_size and filesize > max_size: fields['partial_upload'] = 'true' fields['partial_target_bytesize'] = str(filesize) # Check if there is already a .dlpart on the server. # If it's the case, maybe it's already the beginning of this? if 'existing_dlpart' in files: existing_dlpart = files['existing_dlpart'] existing_dlpart_size = existing_dlpart['size'] if filesize > existing_dlpart_size and \ file_start_hash_match(files['userfile_0'], existing_dlpart_size, existing_dlpart['hash']): logging.info('Found the beggining of this file on the other file, skipping the first pieces') existing_pieces_number = existing_dlpart_size / max_size cb(filesize, existing_dlpart_size, existing_dlpart_size, 0) if not existing_pieces_number: (header_body, close_body, content_type) = encode_multiparts(fields) body = BytesIOWithFile(header_body, close_body, files['userfile_0'], callback=cb, chunk_size=max_size, file_part=0) resp = requests.post( url, data=body, headers={'Content-Type': content_type}, stream=True, timeout=20, verify=self.verify_ssl ) existing_pieces_number = 1 parse_upload_rep(resp) if resp.status_code == 401: return resp if max_size and filesize > max_size: fields['appendto_urlencoded_part'] = fields['urlencoded_filename'] del fields['urlencoded_filename'] (header_body, close_body, content_type) = encode_multiparts(fields) for i in range(existing_pieces_number, int(math.ceil(filesize / max_size)) + 1): if self.interrupt_tasks: raise PydioSdkException("upload", path=os.path.basename(files['userfile_0']), detail=_('Task interrupted by user')) before = time.time() body = BytesIOWithFile(header_body, close_body, files['userfile_0'], callback=cb, chunk_size=max_size, file_part=i) resp = requests.post( url, data=body, headers={'Content-Type': content_type}, stream=True, verify=self.verify_ssl ) parse_upload_rep(resp) if resp.status_code == 401: return resp duration = time.time() - before logging.info('Uploaded '+str(max_size)+' bytes of data in about %'+str(duration)+' s') return resp
def download(self, path, local, callback_dict=None): """ Download the content of a server file to a local file. :param path: node path on the server :param local: local path on filesystem :param callback_dict: a dict() than can be updated by with progress data :return: Server response """ orig = self.stat(path) if not orig: raise PydioSdkException('download', path, _('Original file was not found on server')) url = self.url + '/download' + self.urlencode_normalized((self.remote_folder + path)) local_tmp = local + '.pydio_dl' headers = None write_mode = 'wb' dl = 0 if not os.path.exists(os.path.dirname(local)): os.makedirs(os.path.dirname(local)) elif os.path.exists(local_tmp): # A .pydio_dl already exists, maybe it's a chunk of the original? # Try to get an md5 of the corresponding chunk current_size = os.path.getsize(local_tmp) chunk_local_hash = hashfile(open(local_tmp, 'rb'), hashlib.md5()) chunk_remote_stat = self.stat(path, True, partial_hash=[0, current_size]) if chunk_local_hash == chunk_remote_stat['hash']: headers = {'range':'bytes=%i-%i' % (current_size, chunk_remote_stat['size'])} write_mode = 'a+' dl = current_size if callback_dict: callback_dict['bytes_sent'] = float(current_size) callback_dict['total_bytes_sent'] = float(current_size) callback_dict['total_size'] = float(chunk_remote_stat['size']) callback_dict['transfer_rate'] = 0 dispatcher.send(signal=TRANSFER_CALLBACK_SIGNAL, send=self, change=callback_dict) else: os.unlink(local_tmp) try: with open(local_tmp, write_mode) as fd: start = time.clock() r = self.perform_request(url=url, stream=True, headers=headers) total_length = r.headers.get('content-length') if total_length is None: # no content length header fd.write(r.content) else: previous_done = 0 for chunk in r.iter_content(1024 * 8): if self.interrupt_tasks: raise PydioSdkException("interrupt", path=path, detail=_('Task interrupted by user')) dl += len(chunk) fd.write(chunk) done = int(50 * dl / int(total_length)) if done != previous_done: transfer_rate = dl // (time.clock() - start) logging.debug("\r[%s%s] %s bps" % ('=' * done, ' ' * (50 - done), transfer_rate)) dispatcher.send(signal=TRANSFER_RATE_SIGNAL, send=self, transfer_rate=transfer_rate) if callback_dict: callback_dict['bytes_sent'] = float(len(chunk)) callback_dict['total_bytes_sent'] = float(dl) callback_dict['total_size'] = float(total_length) callback_dict['transfer_rate'] = transfer_rate dispatcher.send(signal=TRANSFER_CALLBACK_SIGNAL, send=self, change=callback_dict) previous_done = done if not os.path.exists(local_tmp): raise PydioSdkException('download', local, _('File not found after download')) else: stat_result = os.stat(local_tmp) if not orig['size'] == stat_result.st_size: os.unlink(local_tmp) raise PydioSdkException('download', path, _('File is not correct after download')) else: is_system_windows = platform.system().lower().startswith('win') if is_system_windows and os.path.exists(local): os.unlink(local) os.rename(local_tmp, local) return True except PydioSdkException as pe: if pe.operation == 'interrupt': raise pe else: if os.path.exists(local_tmp): os.unlink(local_tmp) raise pe except Exception as e: if os.path.exists(local_tmp): os.unlink(local_tmp) raise PydioSdkException('download', path, _('Error while downloading file: %s') % e.message)
def bulk_stat(self, pathes, result=None, with_hash=False): """ Perform a stat operation (see self.stat()) but on a set of nodes. Very important to use that method instead of sending tons of small stat requests to server. To keep POST content reasonable, pathes will be sent 200 by 200. :param pathes: list() of node pathes :param result: dict() an accumulator for the results :param with_hash: bool whether to ask for files hash or not (md5) :return: """ # NORMALIZE PATHES FROM START pathes = map(lambda p: self.normalize(p), pathes) action = '/stat_hash' if with_hash else '/stat' data = dict() maxlen = min(len(pathes), 200) clean_pathes = map(lambda t: self.remote_folder + t.replace('\\', '/'), filter(lambda x: x != '', pathes[:maxlen])) data['nodes[]'] = map(lambda p: self.normalize(p), clean_pathes) url = self.url + action + self.urlencode_normalized(clean_pathes[0]) resp = self.perform_request(url, type='post', data=data) try: data = json.loads(resp.content) except ValueError: logging.debug("url: %s" % url) logging.debug("resp.content: %s" % resp.content) raise if len(pathes) == 1: englob = dict() englob[self.remote_folder + pathes[0]] = data data = englob if result: replaced = result else: replaced = dict() for (p, stat) in data.items(): if self.remote_folder: p = p[len(self.remote_folder):] #replaced[os.path.normpath(p)] = stat p1 = os.path.normpath(p) p2 = os.path.normpath(self.normalize_reverse(p)) p3 = p p4 = self.normalize_reverse(p) if p2 in pathes: replaced[p2] = stat pathes.remove(p2) elif p1 in pathes: replaced[p1] = stat pathes.remove(p1) elif p3 in pathes: replaced[p3] = stat pathes.remove(p3) elif p4 in pathes: replaced[p4] = stat pathes.remove(p4) else: #pass logging.info('Fatal charset error, cannot find files (%s, %s, %s, %s) in %s' % (repr(p1), repr(p2), repr(p3), repr(p4), repr(pathes),)) raise PydioSdkException('bulk_stat', p1, "Encoding problem, failed emptying bulk_stat, " "exiting to avoid infinite loop") if len(pathes): self.bulk_stat(pathes, result=replaced, with_hash=with_hash) return replaced