コード例 #1
0
ファイル: tasks.py プロジェクト: velsa/mdbox
def save_node(node_id, content, use_ust=True, callback=None):
	"""
	Save content into node (must be a file !)
	Arguments:
		node_id:		node to update
		content: 		content to write to file
		use_ust: 		if True, process save in User Synchronous mode
	"""
	node, db_sess = _get_node(node_id)
	l.info("save_node: %s" % (node.db_path,  ))

	if use_ust:
		if django_settings.ASYNC_DROPBOX:
			yield gen.Task(async_ust_start, node.user)
		else:
			ust_start(node.user)

	try:
		# First save to dropbox
		u_content = force_unicode((smart_unicode(content)))
		if django_settings.ASYNC_DROPBOX:
			yield_key = object()
			db_sess.write_file(encode_if_unicode(node.db_path), u_content,
				callback=(yield gen.Callback(yield_key)))
			ret_dict = yield gen.Wait(yield_key)
		else:
			ret_dict = db_sess.write_file(encode_if_unicode(node.db_path), u_content)

		# Then to local storage
		_save_and_convert(node, content)
	except Exception, e:
		err_msg = "save_node: can't save %s (%s)" %\
				  (node.db_path, e, )
		l.error(err_msg)
		raise MDBException("ERROR: "+err_msg)
コード例 #2
0
ファイル: tasks.py プロジェクト: velsa/mdbox
def sync_file_from_dropbox(node_id, use_ust=True, callback=None):
	"""
	Sync file which was updated on dropbox
	Arguments:
		node_id: 			updated file
		use_ust: 			if True, process remove in User Synchronous mode
	"""
	node, db_sess = _get_node(node_id)

	if use_ust:
		if django_settings.ASYNC_DROPBOX:
			yield gen.Task(async_ust_start, node.user)
		else:
			ust_start(node.user)

	l.info("sync_file_FROM_dropbox: '%s'" % (node.db_path, ))
	try:
		# Read file content from dropbox
		if django_settings.ASYNC_DROPBOX:
			yield_key = object()
			db_sess.read_file(encode_if_unicode(node.db_path),
				callback=(yield gen.Callback(yield_key)))
			content = yield gen.Wait(yield_key)
		else:
			content = db_sess.read_file(encode_if_unicode(node.db_path))
		# And save it to local storage
		_save_and_convert(node, content)
		# User may want to know that this file was synced
		append_to_changes_list(node.user, node.db_path)
	except Exception, e:
		err_msg = "SYNC_file_from_dropbox: '%s' (%s)" %\
				  (node.db_path, e,)
		l.error(err_msg)
		raise MDBException("ERROR: "+err_msg)
コード例 #3
0
ファイル: tasks.py プロジェクト: velsa/mdbox
def rename_node(node_id, new_filename, use_ust=True, callback=None):
	"""
	Rename node to new_filename
	Arguments:
		node_id:		node to rename
		new_filename:	filename to remove to (can be both directory and file)
		use_ust: 		if True, process remove in User Synchronous mode
	"""
	node, db_sess = _get_node(node_id)
	l.info("rename_node: '%s' -> '%s'" % (node.db_path, new_filename, ))

	if use_ust:
		if django_settings.ASYNC_DROPBOX:
			yield gen.Task(async_ust_start, node.user)
		else:
			ust_start(node.user)

	# TODO: Dropbox exception's text contains HTML, which can't be parsed
	# TODO: correctly by frontend, use Reason (part of exception dict)
	try:
		# First rename on dropbox
		new_db_path = get_dropbox_path(os.path.split(node.db_path)[0], new_filename)
		from_path = encode_if_unicode(node.db_path)
		to_path = encode_if_unicode(new_db_path)

		if django_settings.ASYNC_DROPBOX:
			yield_key = object()
			db_sess.move_file(from_path, to_path,
				callback=(yield gen.Callback(yield_key)))
			ret_dict = yield gen.Wait(yield_key)
		else:
			db_sess.db_client.file_move(from_path, to_path)

		# Then on local storage
		new_src = get_server_src_file_path(node.user, new_db_path)
		forced_rename(node.srv_src, new_src)
		new_html = get_server_html_file_path(node.user, new_db_path)
		if new_html != "":
			forced_rename(node.srv_html, new_html)

		# And the object itself
		MDBTree.objects.filter(pk=node.pk).update(
			db_path 	= new_db_path,
			srv_src 	= new_src,
			srv_html 	= new_html,
		)
	except Exception, e:
		err_msg = "rename_node: '%s' -> '%s' (%s)" %\
				  (node.db_path, new_filename, e,)
		l.error(err_msg)
		raise MDBException("ERROR: "+err_msg)
コード例 #4
0
ファイル: tasks.py プロジェクト: velsa/mdbox
def create_new_file(parent_node_id, filename, content, use_ust=True, callback=None):
	"""
	Add new file TO dropbox and also to our database and to server
	NOTE: filename should be just the name of the file, without any
	path elements (it will be added as file in parent_node)
	Arguments:
		parent_node_id: 	dir where the file should be created
		filename:			name of new file
		content:			initial content
		use_ust: 			if True, process remove in User Synchronous mode
	"""
	parent_node, db_sess = _get_node(parent_node_id)
	l.info("create_new_file: '%s' in '%s' (%d bytes)" %\
		   (filename, parent_node.db_path, len(content), ))

	if use_ust:
		if django_settings.ASYNC_DROPBOX:
			yield gen.Task(async_ust_start, parent_node.user)
		else:
			ust_start(parent_node.user)

	try:
		# We first create file ON dropbox and then sync it back
		# to our database and server to keep the exec logic consistent
		db_file_path = get_dropbox_path(parent_node.db_path, filename)
		if django_settings.ASYNC_DROPBOX:
			yield_key = object()
			db_sess.write_file(encode_if_unicode(db_file_path), content,
				callback=(yield gen.Callback(yield_key)))
			ret_dict = yield gen.Wait(yield_key)
		else:
			ret_dict = db_sess.write_file(encode_if_unicode(db_file_path), content)
		l.info("create_new_file: '%s' in '%s' DONE. Now syncing dir..." %\
			   (filename, parent_node.db_path,  ))

		if django_settings.ASYNC_DROPBOX:
			yield_key = object()
			add_file_from_dropbox(parent_node.id, db_file_path, datetime.utcnow(), use_ust=False,
				callback=(yield gen.Callback(yield_key)))
			ret_dict = yield gen.Wait(yield_key)
		else:
			add_file_from_dropbox(parent_node.id, db_file_path, datetime.utcnow(), use_ust=False)
	except Exception, e:
		err_msg = "create_new_file: '%s' in '%s' (%s)" %\
				  (filename, parent_node.db_path, e,)
		l.error(err_msg)
		raise MDBException("ERROR: "+err_msg)
コード例 #5
0
ファイル: tasks.py プロジェクト: velsa/mdbox
def remove_node_as_file(node_id, skip_dropbox=False, use_ust=True, callback=None):
	"""
	Remove node (must be a file !)
	Arguments:
		node_id:		node to remove
		skip_dropbox: 	if True, only remove from local storage (used in sync)
		use_ust: 		if True, process remove in User Synchronous mode
	"""
	node, db_sess = _get_node(node_id)

	if use_ust:
		if django_settings.ASYNC_DROPBOX:
			yield gen.Task(async_ust_start, node.user)
		else:
			ust_start(node.user)

	try:
		# First remove from dropbox
		if not skip_dropbox:
			if django_settings.ASYNC_DROPBOX:
				yield_key = object()
				db_sess.delete_file(encode_if_unicode(node.db_path),
					callback=(yield gen.Callback(yield_key)))
				yield gen.Wait(yield_key)
			else:
				db_sess.db_client.file_delete(encode_if_unicode(node.db_path))
		else:
			l.info("remove_node_as_file: '%s' skipping dropbox removal" %\
				   (node.db_path, ))
		# Then from local storage
		for file in (node.srv_html, node.srv_src):
			l.info("remove_node_as_file: removing '%s'" %\
				   (file, ))
			if not os.path.exists(file):
				l.info("remove_node_as_file: '%s' does not exists ?!" %\
					   (file, ))
			else:
				os.remove(file)

		# Also delete treebeard node
		node.delete()
	except Exception, e:
		err_msg = "remove_node_as_file: '%s' (%s)" %\
				  (node.db_path, e,)
		l.error(err_msg)
		raise MDBException("ERROR: "+err_msg)
コード例 #6
0
ファイル: tasks.py プロジェクト: velsa/mdbox
def create_new_dir(parent_node_id, dir_name, use_ust=True, callback=None):
	"""
	Add new directory TO dropbox and also to our database and to server
	NOTE: directory should be just the name of the directory, without any
	path elements (it will be added in parent_node)
	Arguments:
		parent_node_id: 	dir where the new dir should be created
		filename:			name of new dir
		use_ust: 			if True, process remove in User Synchronous mode
	"""
	parent_node, db_sess = _get_node(parent_node_id)
	l.info("create_new_dir: '%s' in '%s'" %\
		   (dir_name, parent_node.db_path, ))

	if use_ust:
		if django_settings.ASYNC_DROPBOX:
			yield gen.Task(async_ust_start, parent_node.user)
		else:
			ust_start(parent_node.user)

	try:
		# First save to dropbox
		db_dir_path = get_dropbox_path(parent_node.db_path, dir_name)
		if django_settings.ASYNC_DROPBOX:
			yield_key = object()
			db_sess.create_folder(encode_if_unicode(db_dir_path),
				callback=(yield gen.Callback(yield_key)))
			ret_dict = yield gen.Wait(yield_key)
		else:
			ret_dict = db_sess.db_client.file_create_folder(encode_if_unicode(db_dir_path))
		l.info("create_new_dir: '%s' in '%s' DONE. Now syncing parent..." %\
			   (dir_name, parent_node.db_path, ))

		if django_settings.ASYNC_DROPBOX:
			yield_key = object()
			add_dir_from_dropbox(parent_node.id, db_dir_path, use_ust=False,
				callback=(yield gen.Callback(yield_key)))
			yield gen.Wait(yield_key)
		else:
			add_dir_from_dropbox(parent_node.id, db_dir_path, use_ust=False)
	except Exception, e:
		err_msg = "create_new_dir: '%s' -> '%s' (%s)" %\
				  (dir_name, parent_node.db_path, e,)
		l.error(err_msg)
		raise MDBException("ERROR: "+err_msg)
コード例 #7
0
ファイル: tasks.py プロジェクト: velsa/mdbox
def remove_node_as_dir(node_id, skip_dropbox=False, use_ust=True, callback=None):
	"""
	Remove node (must be a directory !)
	Arguments:
		node_id:		node to remove
		skip_dropbox: 	if True, only remove from local storage (used in sync)
		use_ust: 		if True, process remove in User Synchronous mode
	"""
	node, db_sess = _get_node(node_id)

	if use_ust:
		if django_settings.ASYNC_DROPBOX:
			yield gen.Task(async_ust_start, node.user)
		else:
			ust_start(node.user)

	try:
		# Remove dir from dropbox if asked
		if not skip_dropbox:
			# Removes dir from dropbox (recursively)
			if django_settings.ASYNC_DROPBOX:
				yield_key = object()
				db_sess.delete_file(encode_if_unicode(node.db_path),
					callback=(yield gen.Callback(yield_key)))
				yield gen.Wait(yield_key)
			else:
				db_sess.db_client.file_delete(encode_if_unicode(node.db_path))
		else:
			l.info("remove_node_as_dir: '%s' skipping dropbox removal" %\
				   (node.db_path, ))

		# Then from local storage
		remove_server_dir(node.user, node.db_path)

		# Go through treebeard to find and delete node's descendants
		for child in node.get_children():
			child.delete()

		# And finally delete treebeard node
		node.delete()
	except Exception, e:
		err_msg = "remove_node_as_dir: '%s' (%s)" %\
				  (node.db_path, e,)
		l.error(err_msg)
		raise MDBException("ERROR: "+err_msg)
コード例 #8
0
ファイル: tasks.py プロジェクト: velsa/mdbox
def add_dir_from_dropbox(parent_node_id, db_dir_path, use_ust=True, callback=None):
	"""
	Add new dir from dropbox to our database and sync it to server
	Arguments:
		parent_node_id: 	dir where the new dir should be created
		db_file_path:		full path to dropbox dir
		use_ust: 			if True, process remove in User Synchronous mode
	"""
	parent_node, db_sess = _get_node(parent_node_id, need_db_sess=False)

	if use_ust:
		if django_settings.ASYNC_DROPBOX:
			yield gen.Task(async_ust_start, parent_node.user)
		else:
			ust_start(parent_node.user)


	# Add new node to treebeard
	node = parent_node.add_child (
		db_path		= db_dir_path,
		is_file		= False,
		last_modified = datetime.utcnow(),
		srv_src		= get_server_dir(parent_node.user, db_dir_path),
		srv_html	= get_server_dir(parent_node.user, db_dir_path),
		user		= parent_node.user
	)

	try:
		# And dir to local storage
		create_server_dir(parent_node.user, db_dir_path)

		# Sync creates the dir on server if it does not exists
		if django_settings.ASYNC_DROPBOX:
			yield_key = object()
			sync_dir_from_dropbox(node.id, use_ust=False,
				callback=(yield gen.Callback(yield_key)))
			yield gen.Wait(yield_key)
		else:
			sync_dir_from_dropbox(node.id, use_ust=False)
	except Exception, e:
		l.error("add_DIR_FROM_dropbox: '%s':'%s' FAILED" %\
				(node.user.username, node.db_path, ))
		raise MDBException("ERROR: "+e.message)
コード例 #9
0
ファイル: tasks.py プロジェクト: velsa/mdbox
def add_file_from_dropbox(parent_node_id, db_file_path, db_last_modified, use_ust=True, callback=None):
	"""
	Add new file from dropbox to our database and sync it to server
	Arguments:
		parent_node_id: 	dir where the new file should be created
		db_file_path:		full path to dropbox file
		db_last_modified:	taken from dropbox to remember last mod date for next sync
		use_ust: 			if True, process remove in User Synchronous mode
	"""
	parent_node, db_sess = _get_node(parent_node_id, need_db_sess=False)

	if use_ust:
		if django_settings.ASYNC_DROPBOX:
			yield gen.Task(async_ust_start, parent_node.user)
		else:
			ust_start(parent_node.user)

	# Add new node to treebeard
	node = parent_node.add_child (
		db_path		= db_file_path,
		is_file		= True,
		last_modified = db_last_modified,
		srv_src		= get_server_src_file_path(parent_node.user, db_file_path),
		srv_html	= get_server_html_file_path(parent_node.user, db_file_path),
		user		= parent_node.user
	)

	try:
		# Sync also creates the file on server if it does not exists
		if django_settings.ASYNC_DROPBOX:
			yield_key = object()
			sync_file_from_dropbox(node.id, use_ust=False,
				callback=(yield gen.Callback(yield_key)))
			yield gen.Wait(yield_key)
		else:
			sync_file_from_dropbox(node.id, use_ust=False)
	except Exception, e:
		err_msg = "ADD_file_from_dropbox: '%s' (%s)" %\
				  (node.db_path, e,)
		l.error(err_msg)
		raise MDBException("ERROR: "+err_msg)
コード例 #10
0
ファイル: tasks.py プロジェクト: velsa/mdbox
def sync_dir_from_dropbox(node_id, use_ust=True, callback=None):
	"""
	Sync dir which was updated on dropbox
	The whole dir is checked for added/removed files and dirs and each entry
	is updated using appropriate method
	Arguments:
		node_id: 			updated dir
		use_ust: 			if True, process remove in User Synchronous mode
	"""
	node, db_sess = _get_node(node_id)

	if use_ust:
		if django_settings.ASYNC_DROPBOX:
			yield gen.Task(async_ust_start, node.user)
		else:
			ust_start(node.user)

	# We need to try/except the whole block to make sure
	# that we call ust_end() before leaving the method
	try:
		if django_settings.ASYNC_DROPBOX:
			resp = yield gen.Task(db_sess.dir_changed,
				node.db_path, node.hash)
		else:
			resp = db_sess.dir_changed(node.db_path, node.hash)
		if not resp:
			l.info("sync_DIR_FROM_dropbox: '%s':'%s' skipping (same hash)" %\
				   (node.user.username, node.db_path, ))
		else:
			# User may want to know that this dir was updated
			append_to_changes_list(node.user, node.db_path)

			# Retrieve list of files and dirs from dropbox for given directory
			if django_settings.ASYNC_DROPBOX:
				file_dicts = yield gen.Task(db_sess.get_files_as_dicts,
					node.db_path)
			else:
				file_dicts = db_sess.get_files_as_dicts(node.db_path)
			if file_dicts:
				yield_keys = []
				# Check each file in directory for updates
				# We go through OUR file list and compare each file's
				# .last_modified date with the one in matching dropbox file
				for child in node.get_children():
					if not child.is_file:
						continue
					file_dict = dicts_find(file_dicts, child.db_path)
					if file_dict and dict_last_modified(file_dict) > child.last_modified:
						# File in dropbox was changed: remember new mod date and sync it
						MDBTree.objects.filter(pk=child.pk).update(
							last_modified = dict_last_modified(file_dict),
						)
						if django_settings.ASYNC_DROPBOX:
							yield_key = object()
							yield_keys.append(yield_key)
							local_callback = (yield gen.Callback(yield_key))
						else:
							local_callback = None
						sync_file_from_dropbox(child.id, use_ust=False, callback=local_callback)
				# Wait for all add_file_from_dropbox() to complete
				if django_settings.ASYNC_DROPBOX and yield_keys:
					yield gen.WaitAll(yield_keys)

			# Get only list of files as a set
			# And see what's new/missing
			db_files = set(dicts_to_paths(file_dicts, files_only=True))
			if db_files:
				# Get list of files in dir from database
				known_files = set(node.get_paths('FILES'))
				yield_keys = []
				# Add new files from dropbox to our database
				for new_file in db_files - known_files:
					if new_file != '':
						db_last_modified =\
						dict_last_modified(dicts_find(file_dicts, new_file))
						if django_settings.ASYNC_DROPBOX:
							yield_key = object()
							yield_keys.append(yield_key)
							local_callback = (yield gen.Callback(yield_key))
						else:
							local_callback = None
						add_file_from_dropbox(node.id, new_file, db_last_modified, use_ust=False,
							callback=local_callback)
				# Wait for all add_file_from_dropbox() to complete
				if django_settings.ASYNC_DROPBOX and yield_keys:
					yield gen.WaitAll(yield_keys)

				# Remove files deleted from dropbox from our database
				for missing_file in known_files - db_files:
					if missing_file != '':
						missing_node = get_node_by_path(node.user, missing_file)
						# Only remove it from our database and server
						if django_settings.ASYNC_DROPBOX:
							yield_key = object()
							remove_node_as_file(missing_node.id, skip_dropbox=True, use_ust=False,
								callback=(yield gen.Callback(yield_key)))
							yield gen.Wait(yield_key)
						else:
							remove_node_as_file(missing_node.id, skip_dropbox=True, use_ust=False)

			# Get only list of dirs as a set
			# And see what's new/missing
			db_dirs = set(dicts_to_paths(file_dicts, dirs_only=True))
			yield_keys = []
			if db_dirs:
				# Get list of dirs in dir from database
				known_dirs = set(node.get_paths('DIRS'))
				# Add new dirs from dropbox to our database
				for new_dir in db_dirs - known_dirs:
					if new_dir != '':
						if django_settings.ASYNC_DROPBOX:
							yield_key = object()
							yield_keys.append(yield_key)
							local_callback = (yield gen.Callback(yield_key))
						else:
							local_callback = None
						add_dir_from_dropbox(node.id, new_dir, use_ust=False, callback=local_callback)
					#ust_store_id(async_res.task_id, node.user)
				# Wait for all add_dir_from_dropbox() to complete
				if django_settings.ASYNC_DROPBOX and yield_keys:
					yield gen.WaitAll(yield_keys)

				# Remove dirs, which are missing from dropbox from our database
				for missing_dir in known_dirs - db_dirs:
					if missing_dir != '' and missing_dir != '/' :
						missing_node = get_node_by_path(node.user, missing_dir)
						# Only remove it from our database and server
						if django_settings.ASYNC_DROPBOX:
							yield_key = object()
							remove_node_as_dir(missing_node.id, skip_dropbox=True, use_ust=False,
								callback=(yield gen.Callback(yield_key)))
							yield gen.Wait(yield_key)
						else:
							remove_node_as_dir(missing_node.id, skip_dropbox=True, use_ust=False)

			# Save new hash for future checks
			MDBTree.objects.filter(pk=node.pk).update(
				hash 		= resp['hash'],
				last_synced = datetime.utcnow(),
			)
	except Exception, e:
		l.error("sync_DIR_FROM_dropbox: '%s':'%s' (%s)" %\
				(node.user.username, node.db_path, e, ))
		raise MDBException("ERROR: "+e.message)
コード例 #11
0
ファイル: tasks.py プロジェクト: velsa/mdbox
		use_ust: 			if True, process remove in User Synchronous mode
	"""
	# Get MDBUser object
	try:
		user = MDBUser.objects.get(pk=user_id)
	except Exception, e:
		err_msg = "sync_all_dirs_from_dropbox(): can't retrieve user %d from database (%s)" %\
				  (user_id, e, )
		l.error(err_msg)
		raise MDBException("ERROR: "+err_msg)

	if use_ust:
		if django_settings.ASYNC_DROPBOX:
			yield gen.Task(async_ust_start, user)
		else:
			ust_start(user)

	try:
		yield_keys = []
		# Sync root node
		root_node = MDBTree.objects.get(user=user, db_path='/')
		if django_settings.ASYNC_DROPBOX:
			yield_key = object()
			yield_keys.append(yield_key)
			local_callback = (yield gen.Callback(yield_key))
		else:
			local_callback = None
		sync_dir_from_dropbox(root_node.id, use_ust=False, callback=local_callback)
		# And all subdirs
		for node in root_node.get_descendants():
			if node and not node.is_file: