def add_metadata(self, size=0, rev=None, client_modified=None, is_dir=0, content_hash=None):
in_dic = {}
in_dic['poolid'] = self.poolid
in_dic['path'] = self.db_path
in_dic['rev'] = rev
in_dic['size'] = size
if client_modified is None:
client_modified = utils.get_timestamp()
in_dic['client_modified'] = client_modified
in_dic['is_dir'] = is_dir
in_dic['content_hash'] = content_hash
in_dic['editor'] = self.account
in_dic['owner'] = self.account
check_metadata = self.get_path()
if not check_metadata is None:
# [TODO]: handle special case: same path insert twice, it is conflict.
ret = self.delete_metadata(current_metadata=check_metadata)
ret, current_metadata, errorMessage = self.dbo_metadata.insert(in_dic)
if not current_metadata is None:
current_metadata = self.convert_for_dropboxlike_dict(current_metadata)
if current_metadata['type']=="file":
tornado.ioloop.IOLoop.instance().add_callback(self.add_thumbnail,current_metadata)
return ret, current_metadata, errorMessage
def delete_path(self, poolid=0, path='', account='', autocommit=True):
ret = False
errorMessage = ""
update_time = utils.get_timestamp()
try:
sql = "UPDATE delta set tag='deleted', update_time=?, account=? WHERE poolid=? and path=?"
cursor = self.conn.execute(sql, (
update_time,
account,
poolid,
path,
))
sql = "UPDATE delta set tag='deleted', update_time=?, account=? WHERE poolid=? and path like ?"
cursor = self.conn.execute(sql, (
update_time,
account,
poolid,
path + '/%',
))
if autocommit:
self.conn.commit()
ret = True
except Exception as error:
#except sqlite3.IntegrityError:
#except sqlite3.OperationalError, msg:
#print("Error: {}".format(error))
errorMessage = "{}".format(error)
logging.info("sqlite error: %s", errorMessage)
#raise
return ret, errorMessage
def create_folder(self, poolid, path, owner):
out_dic = {}
out_dic['poolid'] = poolid
out_dic['path'] = path
out_dic['content_hash'] = None
out_dic['rev'] = None
out_dic['size'] = 0
out_dic['client_modified'] = utils.get_timestamp()
out_dic['is_dir'] = 1
out_dic['editor'] = owner
out_dic['owner'] = owner
return self.insert(out_dic)
def post(self):
self.set_header('Content-Type', 'application/json')
is_pass_check = True
errorMessage = ""
errorCode = 0
is_pass_check = True
errorMessage = ""
errorCode = 0
if is_pass_check:
is_pass_check = False
try:
_body = json.loads(self.request.body)
is_pass_check = True
except Exception:
errorMessage = "wrong json format"
errorCode = 1001
pass
path = None
if is_pass_check:
is_pass_check = False
#logging.info('%s' % (str(_body)))
if _body:
try:
if 'path' in _body:
path = _body['path']
is_pass_check = True
except Exception:
errorMessage = "parse json fail"
errorCode = 1002
if is_pass_check:
ret, errorMessage = self.check_path(path)
if not ret:
is_pass_check = False
errorCode = 1010
#[TODO]:
# query delta for the last cursor on path.
query_result = {'cursor': utils.get_timestamp()}
if is_pass_check:
self.write(query_result)
else:
self.set_status(400)
self.write(dict(error=dict(message=errorMessage, code=errorCode)))
def _updateMtimeToFile(self, real_path, client_modified):
# update access and modify time.
#<Modified time#Created time#Accessed time>
is_pass_check = False
errorMessage = ""
try:
# update modify time.
#st = os.stat(real_path)
#atime = st[ST_ATIME]
atime = utils.get_timestamp()
if client_modified is None:
client_modified = atime
os.utime(real_path, (atime, client_modified))
is_pass_check = True
except Exception as error:
errorMessage = "{}".format(error)
logging.error(errorMessage)
pass
return is_pass_check, errorMessage
def add_path(self,
tag='file',
poolid=0,
path='',
doc_id=0,
account='',
autocommit=True):
update_time = utils.get_timestamp()
ret = False
errorMessage = ""
try:
sql = "DELETE FROM delta WHERE poolid=? and path=?"
cursor = self.conn.execute(sql, (
poolid,
path,
))
sql = "INSERT INTO delta (tag,poolid,path,doc_id,account,update_time) VALUES (?, ?, ?, ?, ?, ?)"
cursor = self.conn.execute(sql, (
tag,
poolid,
path,
doc_id,
account,
update_time,
))
if autocommit:
self.conn.commit()
ret = True
except Exception as error:
#except sqlite3.IntegrityError:
#except sqlite3.OperationalError, msg:
#print("Error: {}".format(error))
errorMessage = "{}".format(error)
logging.info("sqlite error: %s", errorMessage)
#raise
return ret, errorMessage
def get_delta(self, cursor):
# start to get metadata (owner)
poolid = self.current_user['poolid']
#self.open_metadata(poolid)
#query_result = self.metadata_manager.query(path)
query_result = None
if not query_result is None:
#self.write(query_result)
#self.write("[]")
pass
else:
# todo:
# real path exist, but database not exist.
# reture error or sync database from real path.
pass
dict_delta = {
'entries':[],
'cursor': utils.get_timestamp(),
'quota': 0,
'has_more': False
}
self.write(dict_delta)
def insert(self, in_dic, autocommit=True):
out_dic = {}
errorMessage = ""
ret = True
if ret:
for param in ['is_dir']:
if in_dic[param] != 0 and in_dic[param] != 1:
errorMessage = param + ' value must be 0 or 1 for boolean in integer.'
ret = False
if ret:
for param in ['size']:
if in_dic[param] == '':
errorMessage = param + ' value cannot be null.'
ret = False
if ret:
if in_dic.get('owner', '') == '':
errorMessage = 'owner value cannot be empty.'
ret = False
poolid = in_dic.get('poolid', 0)
if poolid < 1:
errorMessage = 'poolid value wrong.'
ret = False
path = in_dic.get('path', '')
if ret:
# skip parent's commit.
if self.last_transaction_path is None:
self.last_transaction_path = path
# get parent node by path
parent_node = None
item_name = path
if len(path) > 0:
parent_node, item_name = os.path.split(path)
#print "parent_node, item_name:", parent_node, item_name
if not parent_node is None:
if parent_node == "/":
parent_node = ""
#print "parent_node, check_and_create_parent:", parent_node
self.check_and_create_parent(poolid, parent_node,
in_dic['owner'])
#logging.info("start to insert '%s' to metadata database ...", item_name)
current_metadata = None
try:
sql = 'insert into metadata (poolid, path, content_hash, rev, size, is_dir, parent, name, client_modified, server_modified, editor, owner)'
sql = sql + ' values(?,?,?,?,?,?,?,?,?,?,?,?) '
#logging.info("sql: %s", sql)
cursor = self.conn.execute(sql, (
poolid,
path,
in_dic['content_hash'],
in_dic['rev'],
in_dic['size'],
in_dic['is_dir'],
parent_node,
item_name,
in_dic['client_modified'],
utils.get_timestamp(),
in_dic['editor'],
in_dic['owner'],
))
current_metadata = self.get_metadata(poolid, path)
if not current_metadata is None:
self.add_metadata_to_delta(current_metadata,
in_dic['editor'],
autocommit=False)
else:
ret = False
errorMessage = "data is lost after move metadata in database"
if self.last_transaction_path != path:
#logging.info("insert commit at path: {}".format(path))
#out_dic['lastrowid'] = cursor.lastrowid
# force skip commit.
autocommit = False
if autocommit:
self.conn.commit()
self.last_transaction_path = None
except sqlite3.IntegrityError:
ret = False
errorMessage = "insert metadata same path twice: {}".format(
in_dic['path'].encode('utf-8').strip())
logging.error(errorMessage)
except Exception as error:
ret = False
errorMessage = "insert metadata table Error: {}".format(error)
logging.error(errorMessage)
return ret, current_metadata, errorMessage
def copy(self, in_dic, autocommit=True):
errorMessage = ""
current_metadata = None
ret = True
poolid = ''
if ret:
if in_dic.get('poolid', 0) == 0:
errorMessage = 'poolid value cannot be empty.'
ret = False
else:
poolid = in_dic['poolid']
path = ''
if ret:
if in_dic.get('path', '') == '':
errorMessage = 'path value cannot be empty.'
ret = False
else:
path = in_dic['path']
old_poolid = ''
if ret:
if in_dic.get('old_poolid', '') == '':
errorMessage = 'old_poolid value cannot be empty.'
ret = False
else:
old_poolid = in_dic['old_poolid']
old_path = in_dic.get('old_path', '')
if ret:
if in_dic.get('editor', '') == '':
errorMessage = 'editor value cannot be empty.'
ret = False
if ret:
# skip parent's commit.
if self.last_transaction_path is None:
self.last_transaction_path = path
# get parent node by path
parent_node = None
item_name = path
if len(path) > 0:
# get parent node by path
parent_node, item_name = os.path.split(path)
if not parent_node is None:
if parent_node == "/":
parent_node = ""
self.check_and_create_parent(poolid, parent_node,
in_dic['editor'])
#logging.info("copy '%s' to metadata database ...", item_name)
try:
#[TODO]: generate new rev for new file.
sql = "insert into metadata (poolid, path, content_hash, rev, size, is_dir, parent, name, client_modified, server_modified, editor, owner) "
sql += "select ?, ?, content_hash, null, size, is_dir, ?, ?, client_modified, ?, ?, owner from metadata where poolid=? and path=?"
self.conn.execute(sql, (
poolid,
path,
parent_node,
item_name,
utils.get_timestamp(),
in_dic['editor'],
old_poolid,
old_path,
))
#logging.info("sql: %s", sql)
# copy child node path.
sql = "insert into metadata (poolid, path, content_hash, rev, size, is_dir, parent, name, client_modified, server_modified, editor, owner) "
sql += "select ?, ? || substr(path, length(?)+1), content_hash, null, size, is_dir, ? || substr(parent, length(?)+1), name, client_modified, ?, ?, owner from metadata where poolid=? and path like ?"
self.conn.execute(sql, (
poolid,
path,
old_path,
path,
old_path,
utils.get_timestamp(),
in_dic['editor'],
old_poolid,
old_path + '/%',
))
#logging.info("sql: %s", sql)
current_metadata = self.get_metadata(poolid, path)
if not current_metadata is None:
self.add_metadata_to_delta(current_metadata,
in_dic['editor'],
autocommit=False)
else:
ret = False
errorMessage = "data is lost after move metadata in database"
# must commit at finish!
if autocommit:
self.conn.commit()
self.last_transaction_path = None
except Exception as error:
ret = False
errorMessage = "copy metadata table Error: {}".format(error)
logging.error(errorMessage)
return ret, current_metadata, errorMessage
def update(self, in_dic, autocommit=True):
errorMessage = ""
current_metadata = None
ret = True
poolid = ''
if ret:
if in_dic.get('poolid', 0) == 0:
errorMessage = 'poolid value cannot be empty.'
ret = False
else:
poolid = in_dic['poolid']
path = in_dic.get('path', '')
old_poolid = ''
if ret:
if in_dic.get('old_poolid', '') == '':
errorMessage = 'old_poolid value cannot be empty.'
ret = False
else:
old_poolid = in_dic['old_poolid']
old_path = in_dic.get('old_path', '')
if ret:
if in_dic.get('editor', '') == '':
errorMessage = 'editor value cannot be empty.'
ret = False
#print "in_dic", in_dic
if ret:
# skip parent's commit.
if self.last_transaction_path is None:
self.last_transaction_path = path
# get parent node by path
parent_node = None
item_name = path
if len(path) > 0:
# get parent node by path
parent_node, item_name = os.path.split(path)
if not parent_node is None:
if parent_node == "/":
parent_node = ""
self.check_and_create_parent(poolid, parent_node,
in_dic['editor'])
#logging.info("update '%s' to metadata database ...", item_name)
try:
sql = 'update metadata set poolid=?, path = ?'
if 'content_hash' in in_dic:
sql += ' , content_hash = \'%s\'' % str(
in_dic['content_hash']).replace("'", "''")
#[TODO]: Revision
#sql += ' , rev = ?'
if 'size' in in_dic:
sql += ' , size = %d' % int(in_dic['size'])
if 'client_modified' in in_dic:
sql += ' , client_modified = %d' % int(
in_dic['client_modified'])
if 'is_dir' in in_dic:
sql += ' , is_dir = %d' % int(in_dic['is_dir'])
sql += ' , parent=?, name = ? , server_modified = ?, editor = ? where poolid=? and path = ? '
#logging.info('update sql:%s' % (sql))
self.conn.execute(sql, (
poolid,
path,
parent_node,
item_name,
utils.get_timestamp(),
in_dic['editor'],
old_poolid,
old_path,
))
# update child node path.
# question: is need to update the editor/owner account?
if not (old_poolid == poolid and old_path == path):
# skip for file revision.
sql = 'update metadata set poolid=?, path = ? || substr(path, length(?)+1) , parent = ? || substr(parent, length(?)+1), editor=? where poolid=? and path like ? '
#logging.info('update subfolder sql:%s' % (sql))
self.conn.execute(sql, (
poolid,
path,
old_path,
path,
old_path,
in_dic['editor'],
old_poolid,
old_path + '/%',
))
#self.conn.commit()
current_metadata = self.get_metadata(poolid, path)
if not current_metadata is None:
self.dbo_delta.delete_path(poolid=old_poolid,
path=old_path,
account=in_dic['editor'],
autocommit=False)
self.add_metadata_to_delta(current_metadata,
in_dic['editor'],
autocommit=False)
else:
ret = False
errorMessage = "data is lost after move metadata in database"
# must commit at finish!
if autocommit:
self.conn.commit()
self.last_transaction_path = None
except Exception as error:
ret = False
errorMessage = "update metadata table Error: {}".format(error)
logging.error(errorMessage)
return ret, current_metadata, errorMessage
def list_folder(self, poolid=None, db_path=None, show_share_folder=False):
if poolid is None:
poolid = self.poolid
if db_path is None:
db_path = self.db_path
if db_path is None:
# unable found this path in database.
db_path = self.query_path
dic_children = None
if not poolid is None:
if not self.dbo_metadata is None:
dic_children = self.dbo_metadata.list_folder(poolid, db_path)
metadata_dic = {}
contents = []
# for small case used, use append for each item.
if not dic_children is None:
for item in dic_children:
contents.append(self.convert_for_dropboxlike_dict(item))
# add shared folder
if show_share_folder and (self.poolname == "" or self.poolname is None):
share_folder_array = self.dbo_pool_subscriber.list_share_poolid(self.account, db_path)
for pool_dict in share_folder_array:
doc_id = 0
new_poolid = pool_dict['poolid']
metadata_conn = self.open_metadata_db(new_poolid)
dbo_share_folder_metadata = DboMetadata(metadata_conn)
if not dbo_share_folder_metadata is None:
current_metadata = dbo_share_folder_metadata.get_metadata(new_poolid, "")
if not current_metadata is None:
doc_id = current_metadata['doc_id']
share_folder_dict = {}
share_folder_dict['id'] = doc_id
share_folder_dict['path'] = pool_dict['poolname']
parent_node, item_name = os.path.split(pool_dict['poolname'])
share_folder_dict['name'] = item_name
#out_dic['permission'] = "{"write": True}"ll
share_folder_dict['permission'] = 'r'
if pool_dict['can_edit'] == 1:
share_folder_dict['permission'] = 'rw'
if pool_dict['status'] == dbconst.POOL_STATUS_SHARED:
share_folder_dict['permission'] = 'rwx'
share_folder_dict['type'] = "folder"
share_folder_dict['shared_folder'] = True
contents.append(share_folder_dict)
metadata_dic['entries']=contents
metadata_dic['cursor']=utils.get_timestamp()
#[TOOD]: paging metadata
metadata_dic['has_more']=False
#print 'dic_current:%s' % (metadata_dic)
#self.write(metadata_dic)
# for big data used. (but, seems speed the same.)
'''
metadata_dic['contents']=contents
delimiter = '\"contents\": ['
#body = "{}".format(metadata_dic)
body = json.dumps(metadata_dic)
body_item = body.split(delimiter)
self.write(body_item[0]+delimiter)
dic_children_count = len(dic_children)
if dic_children_count > 0:
iPos = 0
for item in dic_children:
iPos += 1
self.write(json.dumps(self.convert_for_dropboxlike_dict(item)))
if iPos < dic_children_count:
self.write(",")
self.write(body_item[1])
'''
return metadata_dic