def __init__(self, hostname, path, username, password, additional_options): if not hostname: raise ValueError("access_token missing") self.access_token = hostname self.app_path = path self.quota = (0, (0, 0)) self.destroyed = False self.dry_run = 'ro' in additional_options # blob control self.dict_files_buffer = {} self.set_new_files = set() # upload control self.mutex = Lock() self.all_jobs_done = Event() self.new_job = Event() self.queue_pending_files = OrderedSet() upload_thread = Thread(target=self._upload) upload_thread.start()
class StorageOperations: def __init__(self, hostname, path, username, password, additional_options): if not hostname: raise ValueError("access_token missing") self.access_token = hostname self.app_path = path self.quota = (0, (0, 0)) self.destroyed = False self.dry_run = 'ro' in additional_options # blob control self.dict_files_buffer = {} self.set_new_files = set() # upload control self.mutex = Lock() self.all_jobs_done = Event() self.new_job = Event() self.queue_pending_files = OrderedSet() upload_thread = Thread(target=self._upload) upload_thread.start() def _get(self, base_url, parameters, headers=None): logger.debug( 'bpan: get(base_url={}, parameters={}, headers={})'.format( base_url, parameters, headers)) parameters['access_token'] = self.access_token req = Request('?'.join((base_url, urlencode(parameters)))) if headers: req.headers.update(headers) try: return urlopen(req).read() except HTTPError as e: return e.read() def _path(self, name): return '/'.join((self.app_path, name)) @staticmethod def _json(result): return json.loads(result.decode()) def _post(self, base_url, parameters, data=b'', headers=None): logger.debug( 'bpan: post(base_url={}, parameters={}, headers={})'.format( base_url, parameters, headers)) if self.dry_run: logger.debug('bpan: dry_run') return b'' parameters['access_token'] = self.access_token if data: data, boundary = encode_multipart(data) req = Request( '?'.join((base_url, urlencode(parameters))), data.encode('ISO-8859-1')) req.add_header( 'Content-Type', 'multipart/form-data; boundary=%s' % boundary) else: req = Request( '?'.join((base_url, urlencode(parameters))), data) if headers: req.headers.update(headers) try: return urlopen(req).read() except HTTPError as e: return e.read() def _read_factory(self, name): path = self._path(name) def read_factory(offset, length): return self._get( 'https://d.pcs.baidu.com/rest/2.0/pcs/file', {'method': 'download', 'path': path}, headers={ 'Range': 'bytes={}-{}'.format(offset, offset + length - 1) }) return read_factory def _upload(self): last_wake = time() while True: if not self.queue_pending_files: while time() < last_wake + 600 and not self.destroyed: self.new_job.wait() last_wake = time() self.new_job.clear() while self.queue_pending_files: with self.mutex: name = self.queue_pending_files.pop(False) self.dict_files_buffer[name].mutex.acquire() logger.debug(self._post( 'https://c.pcs.baidu.com/rest/2.0/pcs/file', { 'method': 'upload', 'path': self._path(name), 'ondup': 'overwrite'}, {'file': self.dict_files_buffer[name].read( 0, self.size(name))})) self.dict_files_buffer[name].dirty = False self.set_new_files.discard(name) self.dict_files_buffer[name].mutex.release() if self.destroyed: break self.all_jobs_done.set() def close(self, name): if self.dict_files_buffer[name].dirty: if len(self.dict_files_buffer[name]): with self.mutex: self.queue_pending_files.discard(name) self.queue_pending_files.add(name) self.new_job.set() else: self.remove(name) def create(self, name): self.set_new_files.add(name) def destory(self): self.destroyed = True self.new_job.set() self.all_jobs_done.wait() def flush(self, name): pass def open(self, name, attr=None): with self.mutex: if name in self.queue_pending_files: self.queue_pending_files.discard(name) if name not in self.dict_files_buffer: self.dict_files_buffer[name] = FragmentCache( self._read_factory(name)) if name in self.set_new_files: self.dict_files_buffer[name].dirty = True self.dict_files_buffer[name].length = 0 elif attr: self.dict_files_buffer[name].length = attr.st_size else: self.dict_files_buffer[name].length = self._json(self._get( 'https://pcs.baidu.com/rest/2.0/pcs/file', {'method': 'meta', 'path': self._path(name)} ))['list'][0]['size'] def read(self, name, offset, length): return self.dict_files_buffer[name].read(offset, length) def remove(self, name): with self.dict_files_buffer[name].mutex: with self.mutex: if name in self.queue_pending_files: self.queue_pending_files.discard(name) if name in self.set_new_files: self.set_new_files.discard(name) else: self._post( 'https://pcs.baidu.com/rest/2.0/pcs/file', {'method': 'delete', 'path': self._path(name)}) del self.dict_files_buffer[name] def statfs(self): if time() > self.quota[0] + 600: result = self._json(self._get( 'https://pcs.baidu.com/rest/2.0/pcs/quota', {'method': 'info'})) self.quota = (time(), (result['used'], result['quota'])) return self.quota[1] def size(self, name): return len(self.dict_files_buffer[name]) def truncate(self, name, length): return self.dict_files_buffer[name].truncate(length) def write(self, name, offset, buf): return self.dict_files_buffer[name].write(offset, buf)