def rmdir(self, path): path = util.format_path(path) metadata = self._path_to_metadata(path, isfolder=True) file_id = metadata['id'] self.service.files().delete(fileId=file_id).execute() self._decache_metadata(path, metadata)
def update(self, path, content): """Update the file Args and returns same as put """ dbg.dbg(path) path = util.format_path(path) metadata = self._path_to_metadata(path) file_id = metadata['id'] uri = GoogleAPI.UPLOAD_URL + '/files/%s?uploadType=media' % file_id headers = { 'Content-Type': 'text/plain', 'Content-Length': len(content), } for retry_num in xrange(self._num_retries + 1): resp, data = self.service._http.request(uri, method='PUT', body=content, headers=headers) if resp.status < 500: break if resp.status >= 300: raise errors.HttpError(resp, data, uri=uri) if resp.status == 200: drive_file = json.loads(data) self._cache_metadata(path, drive_file) return True else: return False
def putdir(self, path): uri = GoogleAPI.UPLOAD_URL + '/files?uploadType=multipart&alt=json' path = util.format_path(path) name = os.path.basename(path) folder = os.path.dirname(path) parent_id = None if folder != '/': # if it's not at the root folder, find out the parent id parent = self._path_to_metadata(folder, isfolder=True) if not parent: # if the parent folder doesn't exist, then create one self.putdir(folder) parent = self._path_to_metadata(folder, isfolder=True) parent_id = parent['id'] body = { 'title': name, 'mimeType': 'application/vnd.google-apps.folder' } if parent_id: # if not at root folder body['parents'] = [{'id': parent_id}] try: drive_file = self.service.files().insert(body=body).execute() self._cache_metadata(path, drive_file) return True except errors.HttpError, error: print 'An error occurred: %s' % error return False
def listdir(self, path): result = [] path = util.format_path(path) if path == '/': folder_id = 'root' else: folder_id = self._path_to_metadata(path)['id'] param = {} param['q'] = '"%s" in parents and trashed=false' % (folder_id) param['maxResults'] = 500 page_token = None while True: try: if page_token: param['pageToken'] = page_token files = self.service.files().list(**param).execute() result.extend(map(lambda x: x['title'], files['items'])) page_token = files.get('nextPageToken', None) if not page_token: break except errors.HttpError, error: print 'An error occurred: %s' % error return None
def metadata(self, path): path = util.format_path(path) _md = self.search(path) md = {} md['size'] = eval(_md['fileSize']) md['mtime'] = util.convert_time(_md['modifiedDate']) return md
def listdir_old(self, path): result = [] page_token = None path = util.format_path(path) folder_metadata = self.search(path) if folder_metadata: folder_id = folder_metadata['id'] else: folder_id = 'root' while True: try: param = {} if page_token: param['pageToken'] = page_token children = self.service.children().list(folderId=folder_id, **param).execute() for child in children.get('items', []): request = self.service.files().get(fileId=child['id']) metadata = request.execute() self._cache_metadata(os.path.join(path, metadata['title']), metadata) result.append(metadata['title']) page_token = children.get('nextPageToken') if not page_token: break except errors.HttpError, error: print 'An error occurred: %s' % error break
def get(self, path): path = BaiduAPI.ROOT_DIR + util.format_path(path) url = BaiduAPI.BASE_URL + '/file' params = {'method': 'download', 'path': path} resp = self._request('GET', url, params=params, raw=True, stream=True) return resp.raw.read()
def putdir(self, path): """ Args: path: string Returns: None """ path = util.format_path(path) name = os.path.basename(path) parent_folder = os.path.dirname(path) if parent_folder == '/': parent_id = BoxAPI.ROOT_FOLDER_ID else: parent = self._path_to_metadata(parent_folder, isfolder=True) if not parent: # if the parent folder doesn't exist, then create one self.putdir(parent_folder) parent = self._path_to_metadata(parent_folder, isfolder=True) parent_id = parent['id'] url = BoxAPI.BASE_URL + '/folders' data = '{"name":"%s", "parent":{"id":"%s"}}' % (name, parent_id) resp = self._request('POST', url, data=data) self._cache_metadata(path, resp)
def put(self, path, content): """ Args: path: string content: string Returns: None """ path = util.format_path(path) name = os.path.basename(path) parent_folder = os.path.dirname(path) if parent_folder == '/': parent_id = BoxAPI.ROOT_FOLDER_ID else: parent = self._path_to_metadata(parent_folder, isfolder=True) if not parent: # if the parent folder doesn't exist, then create one self.putdir(parent_folder) parent = self._path_to_metadata(parent_folder, isfolder=True) parent_id = parent['id'] url = BoxAPI.UPLOAD_URL + '/files/content' form = {"parent_id": parent_id} strobj = StringIO(content) resp = self._request('POST', url, data=form, files={name: strobj}) metadata = resp['entries'][0] self._cache_metadata(path, metadata) return True
def listdir(self, path): """ Args: path: string Returns: list of file/dir names """ path = util.format_path(path) if path == '/': folder_id = BoxAPI.ROOT_FOLDER_ID else: folder = self.search(path) folder_id = folder['id'] result = [] offset = 0 while True: metalist = self._listdir(folder_id, offset) for metadata in metalist: self._cache_metadata(os.path.join(path, metadata['name']), metadata) result.append(metadata['name']) if (len(metalist) < 1000): break offset += 1000 return result
def get_logs2(self, path, last_clock): beg = time.time() path = '/Public' + util.format_path(path) length = 5 offset = 0 # latest comment comes first #comments = self.get_comments(path, length, offset) revisions = self.get_revisions(path) if not revisions: return [], None new_logs = [] new_clock = revisions[0]['id'] end = False # while True: for revision in revisions: if last_clock and revision['id'] == last_clock: break msg = self.get_revision(path, revision['id']) if len(msg) > 0: new_logs.insert(0, msg) # if end: break # if len(revisions) < length: break # if haven't reached to end, read next batch # offset += length # comments = self.get_comments(path, length, offset) end = time.time() dbg.paxos_time("get_log %s", end - beg) return new_logs, new_clock
def get_logs(self, path, last_clock): beg = time.time() path = '/Public' + util.format_path(path) length = 5 offset = 0 # latest comment comes first comments = self.get_comments(path, length, offset) if not comments: return [], None new_logs = [] new_clock = comments[0]['id'] end = False while True: for comment in comments: if last_clock and comment['id'] == last_clock: end = True break new_logs.insert(0, comment['message']) if end: break if len(comments) < length: break # if haven't reached to end, read next batch offset += length comments = self.get_comments(path, length, offset) end = time.time() dbg.paxos_time("get_log %s", end-beg) return new_logs, new_clock
def listdir(self, path): """ Args: path: string Returns: list of file/dir names """ path = util.format_path(path) if path == '/': folder_id = BoxAPI.ROOT_FOLDER_ID else: folder = self.search(path) folder_id = folder['id'] result = [] offset = 0 while True: metalist = self._listdir(folder_id, offset) for metadata in metalist: self._cache_metadata(os.path.join(path, metadata['name']), metadata) result.append(metadata['name']) if(len(metalist) < 1000): break offset += 1000 return result
def get_logs(self, path, last_clock): path = util.format_path(path) file_id = self._path_to_metadata(path)['id'] # latest comment comes first comments = self.service.comments().list(fileId=file_id, maxResults=5).execute() if not comments['items']: return [], None new_logs = [] new_clock = comments['items'][0]['commentId'] end = False while True: for comment in comments['items']: if last_clock and comment['commentId'] == last_clock: end = True break new_logs.insert(0, comment['content']) if end: break if 'nextPageToken' not in comments: break # get a new batch (5) comments comments = self.service.comments().list( fileId=file_id, maxResults=5, pageToken=comments['nextPageToken']).execute() return new_logs, new_clock
def put(self, path, content): """ Args: path: string content: string Returns: None """ path = util.format_path(path) name = os.path.basename(path) parent_folder = os.path.dirname(path) parent = self._path_to_metadata(parent_folder, isfolder=True) if not parent: # if the parent folder doesn't exist, then create one self.putdir(parent_folder) parent = self._path_to_metadata(parent_folder, isfolder=True) parent_id = parent['id'] url = OneDriveAPI.BASE_URL + '/%s/files/%s' % (parent_id, name) strobj = StringIO(content) params = { 'overwrite': 'false' } metadata = self._request('PUT', url, params=params, data=strobj) metadata[u'type'] = u'file' self._cache_metadata(path, metadata) return True
def rmdir(self, path): path = util.format_path(path) metadata = self._path_to_metadata(path, isfolder=True) file_id = metadata['id'] url = OneDriveAPI.BASE_URL + '/%s' % file_id self._request('DELETE', url, raw=True)
def rm(self, path): path = util.format_path(path) metadata = self._path_to_metadata(path) file_id = metadata['id'] url = OneDriveAPI.BASE_URL + '/me/drive/items/%s' % file_id self._request('DELETE', url, raw=True)
def get_logs(self, path, last_clock): path = util.format_path(path) file_id = self._path_to_metadata(path)['id'] # latest comment comes first comments = self.service.comments().list(fileId=file_id, maxResults=5).execute() if not comments['items']: return [], None new_logs = [] new_clock = comments['items'][0]['commentId'] end = False while True: for comment in comments['items']: if last_clock and comment['commentId'] == last_clock: end = True break new_logs.insert(0, comment['content']) if end: break if 'nextPageToken' not in comments: break # get a new batch (5) comments comments = self.service.comments().list(fileId=file_id, maxResults=5, pageToken=comments['nextPageToken']).execute() return new_logs, new_clock
def get_revisions(self, path): path = '/Public' + util.format_path(path) metadata = self._path_to_metadata(path) file_id = metadata['id'] url = OneDriveAPI.BASE_URL + '/me/drive/items/%s/versions' % (file_id) resp = self._request('GET', url) return resp['value']
def get_con(s_name, auth_path=None, db=None): """For MySQL, one can specify a default database to connect to""" con = None auth_path = os.path.dirname(os.path.abspath( __file__)) + '/db.csv' if auth_path is None else auth_path csv_path = util.format_path(auth_path) t_db = pd.read_csv(csv_path) t_db.fillna('', inplace=True) t_db = t_db[t_db['ID'] == s_name] if len(t_db) == 0: util.error_msg('Database %s is not defined!' % s_name) one = t_db.iloc[0] s_db = db or one['DB'] if one['TYPE'] == 'MYSQL': import MySQLdb as mysql #print one['HOST'], one['USR'], one['PWD'], s_db con = mysql.connect(one['HOST'], one['USR'], one['PWD'], s_db) elif one['TYPE'] == 'POSTGRES': import pgdb # make sure you do: #module load postgresql/9.2.4 #export LD_LIBRARY_PATH=.:/tools/GNU/postgresql/9.2.4/lib/ con = pgdb.connect(one['CONNECT']) elif one['TYPE'] == 'ORACLE': import cx_Oracle as oracle con = oracle.connect(one['CONNECT']) else: util.error_msg('Unsupported database engine: %s' % one['TYPE']) return con
def putdir(self, path): """ Args: path: string Returns: None """ path = util.format_path(path) name = os.path.basename(path) parent_folder = os.path.dirname(path) parent = self._path_to_metadata(parent_folder, isfolder=True) if not parent: # if the parent folder doesn't exist, then create one self.putdir(parent_folder) parent = self._path_to_metadata(parent_folder, isfolder=True) parent_id = parent['id'] url = OneDriveAPI.BASE_URL + '/%s' % parent_id headers = { "Authorization": "Bearer " + self.token.access_token, "Content-Type": "application/json" } data = '{"name": "%s"}' % name resp = self._request('POST', url, headers=headers, data=data) self._cache_metadata(path, resp)
def metadata(self, path): path = util.format_path(path) _md = self.search(path) md = {} md['size'] = _md['size'] md['mtime'] = util.convert_time(_md['updated_time']) return md
def get_logs(self, path, last_clock): beg = time.time() path = '/Public' + util.format_path(path) length = 5 offset = 0 # latest comment comes first comments = self.get_comments(path, length, offset) if not comments: return [], None new_logs = [] new_clock = comments[0]['id'] end = False while True: for comment in comments: if last_clock and comment['id'] == last_clock: end = True break new_logs.insert(0, comment['message']) if end: break if len(comments) < length: break # if haven't reached to end, read next batch offset += length comments = self.get_comments(path, length, offset) end = time.time() dbg.paxos_time("get_log %s", end - beg) return new_logs, new_clock
def put(self, path, content): """ Args: path: string content: string Returns: None """ path = util.format_path(path) name = os.path.basename(path) parent_folder = os.path.dirname(path) parent = self._path_to_metadata(parent_folder, isfolder=True) if not parent: # if the parent folder doesn't exist, then create one self.putdir(parent_folder) parent = self._path_to_metadata(parent_folder, isfolder=True) parent_id = parent['id'] url = OneDriveAPI.BASE_URL + '/%s/files/%s' % (parent_id, name) strobj = StringIO(content) params = {'overwrite': 'false'} metadata = self._request('PUT', url, params=params, data=strobj) metadata[u'type'] = u'file' self._cache_metadata(path, metadata) return True
def rm(self, path): # remove file only path = util.format_path(path) metadata = self._path_to_metadata(path) file_id = metadata['id'] url = BoxAPI.BASE_URL + '/files/%s' % file_id self._request('DELETE', url, raw=True)
def rmdir(self, path): # remove directory only path = util.format_path(path) metadata = self._path_to_metadata(path, isfolder=True) dir_id = metadata['id'] url = BoxAPI.BASE_URL + '/folders/%s?recursive=true' % dir_id self._request('DELETE', url, raw=True)
def get_revision(self, path, rev_id): path = '/Public' + util.format_path(path) metadata = self._path_to_metadata(path) file_id = metadata['id'] url = OneDriveAPI.BASE_URL + '/me/drive/items/{0}/versions/{1}/content' % ( file_id) resp = self._request('GET', url, raw=True, stream=True) return resp.raw.read()
def get_comments(self, path): path = util.format_path(path) metadata = self._path_to_metadata(path) file_id = metadata['id'] url = BoxAPI.BASE_URL + '/files/%s/comments' % file_id resp = self._request('GET', url) return resp['entries']
def get(self, path): path = util.format_path(path) metadata = self._path_to_metadata(path) file_id = metadata['id'] url = OneDriveAPI.BASE_URL + '/%s/content' % file_id resp = self._request('GET', url, raw=True, stream=True) return resp.raw.read()
def get_comments(self, path): path = util.format_path(path) metadata = self._path_to_metadata(path) file_id = metadata['id'] url = BoxAPI.BASE_URL + '/files/%s/comments' % file_id resp = self._request('GET', url); return resp['entries']
def share(self, path, target_email): path = util.format_path(path) metadata = self._path_to_metadata(path) #print(metadata) #perm = self.service.permissions().list(fileId=metadata['id']).execute() body = {"role": "writer", "type": "user", "value": target_email} self.service.permissions().insert(fileId=metadata['id'], emailMessage=True, body=body).execute()
def rm(self, path): path = BaiduAPI.ROOT_DIR + util.format_path(path) url = BaiduAPI.BASE_URL + '/file' params = { 'method': 'delete', 'path': path } self._request('POST', url, params=params)
def putdir(self, path): path = BaiduAPI.ROOT_DIR + util.format_path(path) url = BaiduAPI.BASE_URL + '/file' params = { 'method': 'mkdir', 'path': path } resp = self._request('POST', url, params=params)
def get(self, path): path = BaiduAPI.ROOT_DIR + util.format_path(path) url = BaiduAPI.BASE_URL + '/file' params = { 'method': 'download', 'path': path } resp = self._request('GET', url, params=params, raw=True, stream=True) return resp.raw.read()
def post_comment(self, path, comment): path = util.format_path(path) metadata = self._path_to_metadata(path) file_id = metadata['id'] url = BoxAPI.BASE_URL + '/comments' data = '{"item": {"type": "file", "id": "%s"}, "message": "%s"}' % (file_id, comment) resp = self._request('POST', url, data = data)
def listdir(self, path): path = BaiduAPI.ROOT_DIR + util.format_path(path) url = BaiduAPI.BASE_URL + '/file' params = { 'method': 'list', 'path': path, } resp = self._request('GET', url, params=params) files = [os.path.basename(x['path']) for x in resp['list']] return files
def metadata(self, path): path = util.format_path(path) file_id = self._path_to_metadata(path)['id'] url = BoxAPI.BASE_URL + '/files/%s' % file_id _md = self._request('GET', url) md = {} md['size'] = _md['size'] md['mtime'] = util.convert_time(_md['modified_at']) return md
def share(self, path, target_email): path = util.format_path(path) metadata = self._path_to_metadata(path) #print(metadata) #perm = self.service.permissions().list(fileId=metadata['id']).execute() body = { "role" : "writer", "type" : "user", "value": target_email } self.service.permissions().insert(fileId=metadata['id'], emailMessage=True, body=body).execute()
def exists(self, path): """ Args: path: string Returns: exist: boolean """ path = util.format_path(path) metadata = self.search(path) return (metadata != None)
def get_con_info(s_name, auth_path=None): auth_path = os.path.dirname(os.path.abspath( __file__)) + '/db.csv' if auth_path is None else auth_path csv_path = util.format_path(auth_path) t_db = pd.read_csv(csv_path) t_db.fillna('', inplace=True) t_db = t_db[t_db['ID'] == s_name] if len(t_db) == 0: util.error_msg('Database %s is not defined!' % s_name) return None return t_db.iloc[0]
def put(self, path, content): path = BaiduAPI.ROOT_DIR + util.format_path(path) url = BaiduAPI.BASE_URL + '/file' params = { 'method': 'upload', 'path': path, } if len(content) == 0: content += '\0' strobj = StringIO(content) resp = self._request('POST', url, params=params, files={'file': strobj}) return True
def post_comment(self, path, comment): path = util.format_path(path) metadata = self._path_to_metadata(path) file_id = metadata['id'] url = OneDriveAPI.BASE_URL + '/%s/comments' % file_id headers = { "Authorization": "Bearer " + self.token.access_token, "Content-Type": "application/json" } data = '{"message": "%s"}' % comment resp = self._request('POST', url, headers=headers, data=data)
def exists(self, path): path = BaiduAPI.ROOT_DIR + util.format_path(path) url = BaiduAPI.BASE_URL + '/file' params = { 'method': 'meta', 'path': path, } try: resp = self._request('GET', url, params=params) return True except ItemDoesNotExist: return False